mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-16 07:31:58 +00:00
合入期 0 运行时底座:发起运行的规则抽成 launch_run,运行记下选项快照和定时来源,往停下或已封存的运行里追加事件统一走 append_events,质询用量计进运行用量
This commit is contained in:
+28
-220
@@ -2,7 +2,6 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
from urllib.parse import unquote
|
||||
@@ -12,24 +11,17 @@ from pydantic import BaseModel, Field
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from sqlalchemy import select as _select
|
||||
|
||||
from app.api.coded import CodedHTTPException
|
||||
from app.core.bus import bus
|
||||
from app.core import artifact_store
|
||||
from app.core.events import EventType
|
||||
from app.data.catalog_impact import catalog_drift
|
||||
from app.db.base import SessionLocal, get_session
|
||||
from app.db.models import Approval, Artifact, Run, RunEvent, Workflow, WorkflowVersion
|
||||
from app.engine.governance import governance_note, unresolved_caliber_upgrades
|
||||
from app.db.models import Approval, Artifact, Run, RunEvent
|
||||
# 发起规则和发起前检查绑定的工具搬到了 engine/launch.py。这几个名字以前从这里导入,照旧可用
|
||||
from app.engine.launch import ( # noqa: F401
|
||||
TOOL_MISSING, LaunchRefused, launch_run, missing_tools, node_setting, tool_bindings,
|
||||
)
|
||||
from app.engine.runner import run_manager
|
||||
from app.engine.schema import GraphSpec, NodeType, type_label
|
||||
|
||||
router = APIRouter(prefix="/api/runs", tags=["runs"])
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
#: 发起时发现绑定的工具在本机不存在(见 missing_tools)
|
||||
TOOL_MISSING = "run_tool_missing"
|
||||
|
||||
|
||||
def actor_of(header: str | None) -> str | None:
|
||||
@@ -76,6 +68,8 @@ class RunOut(BaseModel):
|
||||
#: 发起时实际生效的记忆域和知识库
|
||||
memory_scope: str | None = None
|
||||
collection: str | None = None
|
||||
#: 发起它的定时任务。为空 = 手动发起
|
||||
schedule_id: str | None = None
|
||||
# 时间一律带时区(db.base.UTCDateTime 读出来就是 UTC aware)
|
||||
created_at: Any = None
|
||||
started_at: Any = None
|
||||
@@ -90,218 +84,29 @@ async def start_run(
|
||||
session: AsyncSession = Depends(get_session),
|
||||
x_actor: str | None = Header(default=None),
|
||||
) -> Run:
|
||||
actor = actor_of(x_actor)
|
||||
# 没挂在工作流上(画布没保存就跑、问数据当场搭的)。老数据里是「临时图」,前端两种都认
|
||||
name = "未保存的工作流"
|
||||
version: int | None = None
|
||||
version_hash: str | None = None
|
||||
governed = False
|
||||
|
||||
if payload.run_class == "formal":
|
||||
# 正式运行的全部语义就这一条规则:必须引用一个不可变的已发布版本。
|
||||
# 传裸 graph、或工作流还没发布过,都进不了 formal。
|
||||
if not payload.workflow_id:
|
||||
raise HTTPException(400, "正式运行需要从已保存的工作流发起:请求中没有指定工作流")
|
||||
if payload.graph is not None:
|
||||
raise HTTPException(400, "正式运行只能使用已发布的版本,画布上未发布的修改不会生效。请先保存并发布")
|
||||
workflow = await session.get(Workflow, payload.workflow_id)
|
||||
if not workflow:
|
||||
raise HTTPException(404, "工作流不存在,可能已被删除")
|
||||
# 只看 published_version:status 反映的是当前画布(改过图就退回 draft),
|
||||
# 而 formal 跑的是已发布的那一版,两者本来就可以不一致
|
||||
if not workflow.published_version:
|
||||
raise HTTPException(409, f"「{workflow.name}」还没有发布版本,请先在编排页发布")
|
||||
version = payload.version or workflow.published_version
|
||||
# "不可变"和"过过闸"是两件事。快照确实不可变,但 PATCH 保存出来的
|
||||
# 草稿版本同样有快照——只认 version 存不存在的话,工作流只要发布过
|
||||
# 任意一个版本,之后所有草稿都能以 formal 身份启动,发布期的
|
||||
# validate_graph + lint_for_publish 全部绕过。
|
||||
if version != workflow.published_version:
|
||||
raise HTTPException(
|
||||
409,
|
||||
f"v{version} 不是当前发布版本(v{workflow.published_version})。"
|
||||
"正式运行只能使用当前发布版本;如需运行这一版,请先发布它。",
|
||||
)
|
||||
snapshot = (
|
||||
await session.execute(
|
||||
_select(WorkflowVersion).where(
|
||||
WorkflowVersion.workflow_id == workflow.id,
|
||||
WorkflowVersion.version == version,
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if not snapshot:
|
||||
raise HTTPException(404, f"版本 v{version} 不存在")
|
||||
graph = snapshot.graph
|
||||
version_hash = snapshot.graph_hash or artifact_store.graph_hash(graph)
|
||||
name = workflow.name
|
||||
# 按这一版发布时记下的级别;记级别之前发布的老版本没有,照旧看工作流现在的 status
|
||||
governed = (snapshot.level == "governed") if snapshot.level else workflow.status == "governed"
|
||||
|
||||
# 口径升版是被迫处置的事件:钉住的方法卡有新版本而未声明策略,拒绝启动
|
||||
spec = GraphSpec.model_validate(graph)
|
||||
errors, upgrade_events = await unresolved_caliber_upgrades(session, spec)
|
||||
if errors:
|
||||
raise HTTPException(409, ";".join(errors))
|
||||
# 数据目录和口径卡不同:目录是给人和助手看的说明,不是运行的输入。发布之后变了只提醒、不拦——
|
||||
# 拦下来等于让一次文字修订卡住正式出具
|
||||
catalog_changes = await catalog_drift(session, snapshot.catalog_versions)
|
||||
else:
|
||||
graph = payload.graph
|
||||
upgrade_events = []
|
||||
catalog_changes = []
|
||||
if payload.workflow_id:
|
||||
workflow = await session.get(Workflow, payload.workflow_id)
|
||||
if not workflow:
|
||||
raise HTTPException(404, "工作流不存在,可能已被删除")
|
||||
graph = graph or workflow.graph
|
||||
name = workflow.name
|
||||
if not graph:
|
||||
raise HTTPException(400, "没有要运行的内容:请求中既没有工作流 ID,也没有工作流内容")
|
||||
await _refuse_missing_tools(session, graph)
|
||||
|
||||
from app.api.settings import resolve_run_scope
|
||||
|
||||
memory_scope, collection = await resolve_run_scope(
|
||||
session, payload.memory_scope, payload.collection
|
||||
)
|
||||
"""发起规则全在 engine/launch.py(定时任务到点发起也用它),这里只把拒绝换回 HTTP 响应。"""
|
||||
try:
|
||||
run = await run_manager.start(
|
||||
graph=graph,
|
||||
input_payload=payload.input,
|
||||
return await launch_run(
|
||||
session,
|
||||
workflow_id=payload.workflow_id,
|
||||
workflow_name=name,
|
||||
memory_scope=memory_scope,
|
||||
collection=collection,
|
||||
run_class="formal" if payload.run_class == "formal" else "exploratory",
|
||||
version=version,
|
||||
version_hash=version_hash,
|
||||
started_by=actor,
|
||||
graph=payload.graph,
|
||||
input_payload=payload.input,
|
||||
memory_scope=payload.memory_scope,
|
||||
collection=payload.collection,
|
||||
run_class=payload.run_class,
|
||||
version=payload.version,
|
||||
actor=actor_of(x_actor),
|
||||
)
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
|
||||
if payload.run_class == "formal":
|
||||
# 按不按受管出具在发起这一刻定下来、记进封存范围:status 反映的是当前画布,中途改图、
|
||||
# 审批停下再恢复时再去读它,报告节点和出口复核可能各读到不同的值
|
||||
await run_manager.note(run.id, "log", **governance_note(
|
||||
governed=governed, workflow_name=name, version=version))
|
||||
if catalog_changes:
|
||||
# 落在封存范围内:之后回看这次正式出具,能知道当时的目录和发布时不一样、差在哪几张表。
|
||||
# 每张表带 impact:direct 写死的 SQL 用到的,possible Agent 可能查询的(界面分开说)
|
||||
await run_manager.note(run.id, EventType.CATALOG_DRIFT, version=version, count=len(catalog_changes),
|
||||
tables=catalog_changes)
|
||||
# 已声明策略的升版记入事件流,出具物上能看到"这期换口径了、怎么处置的"
|
||||
for ev in upgrade_events:
|
||||
payload_ev = dict(ev)
|
||||
node = payload_ev.pop("node_id", None)
|
||||
await run_manager.note(run.id, "caliber.upgrade", node_id=node, **payload_ev)
|
||||
return run
|
||||
except LaunchRefused as e:
|
||||
raise _refused(e) from e
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 发起前检查绑定的工具
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def node_setting(spec: GraphSpec, node: Any, key: str) -> Any:
|
||||
"""节点配置的实际取值,和运行时 NodeContext.cfg 同一个规则:节点上没写(None、空串)就取图级 defaults。"""
|
||||
value = node.config.get(key)
|
||||
return spec.defaults.get(key) if value in (None, "") else value
|
||||
|
||||
|
||||
def tool_bindings(spec: GraphSpec) -> list[tuple[str, str]]:
|
||||
"""图里每一处绑定的工具:(谁绑的, 工具名)。调用工具节点的 tool、Agent 的 tools、协作成员的 tools。
|
||||
|
||||
取值和运行时一致(node_setting):Agent 节点上没写 tools、协作节点上没写成员,运行时用的是
|
||||
图级 defaults 里的,这里也算它们——不然发起前检查放过了缺的工具,发起时固定数据版本也漏了源。
|
||||
|
||||
模板拼出来的名字({{ vars.tool }})也在内:发起前检查(_bindings)跳过它们,发起时固定数据
|
||||
版本(engine/runner.py)要知道有没有这种名字——有的话说不准会用到哪个数据源。
|
||||
"""
|
||||
out: list[tuple[str, str]] = []
|
||||
for node in spec.nodes:
|
||||
who = f"「{node.title}」({type_label(node.type)})"
|
||||
if node.type == NodeType.TOOL:
|
||||
names = [node_setting(spec, node, "tool")]
|
||||
out.extend((who, n) for n in names if isinstance(n, str) and n.strip())
|
||||
elif node.type == NodeType.AGENT:
|
||||
out.extend((who, n) for n in node_setting(spec, node, "tools") or []
|
||||
if isinstance(n, str) and n.strip())
|
||||
elif node.type == NodeType.SUPERVISOR:
|
||||
for member in node_setting(spec, node, "agents") or []:
|
||||
if not isinstance(member, dict):
|
||||
continue
|
||||
label = f"「{node.title}」的成员「{member.get('name') or '?'}」"
|
||||
out.extend((label, n) for n in member.get("tools") or [] if isinstance(n, str) and n.strip())
|
||||
return [(who, name.strip()) for who, name in out]
|
||||
|
||||
|
||||
def _bindings(spec: GraphSpec) -> list[tuple[str, str]]:
|
||||
"""发起前要检查的绑定。模板拼出来的工具名({{ vars.tool }})发起时还不知道是谁,交给运行时。"""
|
||||
return [(who, name) for who, name in tool_bindings(spec) if "{{" not in name]
|
||||
|
||||
|
||||
async def missing_tools(session: AsyncSession, spec: GraphSpec) -> list[tuple[str, str, str]]:
|
||||
"""本机不存在的绑定:(谁绑的, 工具名, 去哪接入:data / tools)。
|
||||
|
||||
解析和运行时 build_tools 一致:内置(静态注册)、mcp:<服务>/<工具>、db_query__ / db_schema__
|
||||
<数据源>、其余按自定义工具。数据源和自定义工具只认启用着的——停用的运行时同样拿不到。
|
||||
MCP 只看服务登记着、启用着没有:连不连得上、工具还在不在,要连一次才知道,发起时不去等它,
|
||||
上次没连上或者工具清单里没有的只在日志里提一句。
|
||||
"""
|
||||
from app.db.models import CustomTool, DataSource, McpServer
|
||||
from app.tools.datasource import QUERY_PREFIX, SCHEMA_PREFIX
|
||||
from app.tools.registry import all_specs
|
||||
|
||||
bindings = _bindings(spec)
|
||||
if not bindings:
|
||||
return []
|
||||
builtin = all_specs()
|
||||
sources = {n for (n,) in await session.execute(select(DataSource.name).where(DataSource.enabled.is_(True)))}
|
||||
custom = {n for (n,) in await session.execute(select(CustomTool.name).where(CustomTool.enabled.is_(True)))}
|
||||
servers = {r.name: r for r in (await session.execute(select(McpServer))).scalars()}
|
||||
missing: list[tuple[str, str, str]] = []
|
||||
for who, name in bindings:
|
||||
if name.startswith("mcp:"):
|
||||
server, _, tool = name[4:].partition("/")
|
||||
row = servers.get(server)
|
||||
if row is None or not row.enabled:
|
||||
missing.append((who, name, "tools"))
|
||||
elif row.status == "error":
|
||||
logger.warning("%s 绑的 %s:MCP 服务 %s 上次没连上,运行时可能拿不到这个工具", who, name, server)
|
||||
elif tool not in ("", "*") and row.tools_cache and tool not in row.tools_cache:
|
||||
logger.warning("%s 绑的 %s:MCP 服务 %s 上次交回的工具清单里没有 %s", who, name, server, tool)
|
||||
elif name in builtin:
|
||||
continue
|
||||
elif name.startswith((QUERY_PREFIX, SCHEMA_PREFIX)):
|
||||
source = name[len(QUERY_PREFIX):] if name.startswith(QUERY_PREFIX) else name[len(SCHEMA_PREFIX):]
|
||||
if source not in sources:
|
||||
missing.append((who, name, "data"))
|
||||
elif name not in custom:
|
||||
missing.append((who, name, "tools"))
|
||||
return missing
|
||||
|
||||
|
||||
async def _refuse_missing_tools(session: AsyncSession, graph: dict[str, Any]) -> None:
|
||||
"""绑定的工具在本机不存在就不发起:以前要跑到那一步才失败,前面的步骤白跑、钱白花。"""
|
||||
try:
|
||||
spec = GraphSpec.model_validate(graph)
|
||||
except Exception: # noqa: BLE001 - 图本身不成形交给 run_manager.start 报
|
||||
return
|
||||
missing = await missing_tools(session, spec)
|
||||
if not missing:
|
||||
return
|
||||
lines = ";".join(f"{who}绑定的 {name}" for who, name, _ in missing[:8])
|
||||
more = f"等 {len(missing)} 处" if len(missing) > 8 else ""
|
||||
places = {where for _, _, where in missing}
|
||||
if places == {"data"}:
|
||||
fix = "请到「数据」页接入,或在节点中重新选择"
|
||||
elif places == {"tools"}:
|
||||
fix = "请到「工具」页接入,或在节点中重新选择"
|
||||
else:
|
||||
fix = "数据源工具请到「数据」页接入,其他工具请到「工具」页接入,或在节点中重新选择"
|
||||
raise CodedHTTPException(422, f"绑定的工具不存在:{lines}{more}。{fix}", TOOL_MISSING)
|
||||
def _refused(e: LaunchRefused) -> HTTPException:
|
||||
"""发起被拒的 HTTP 响应,和规则搬去 engine/launch.py 之前逐字相同:带机器码的回 {detail, code},
|
||||
不带的回 {detail}。"""
|
||||
if e.code:
|
||||
return CodedHTTPException(e.status_code, e.message, e.code)
|
||||
return HTTPException(e.status_code, e.message)
|
||||
|
||||
|
||||
#: workflow_id 取这个值时只要没挂在任何工作流上的运行:画布上没保存就跑的临时图,
|
||||
@@ -315,6 +120,7 @@ async def list_runs(
|
||||
status: str | None = Query(default=None, description="逗号分隔,如 failed,cancelled"),
|
||||
run_class: str | None = None,
|
||||
q: str | None = Query(default=None, description="按工作流名模糊匹配"),
|
||||
schedule_id: str | None = Query(default=None, description="只要这个定时任务发起的运行"),
|
||||
limit: int = Query(default=50, ge=1, le=200),
|
||||
before: datetime | None = Query(default=None, description="翻页游标:只要 created_at 早于它的"),
|
||||
session: AsyncSession = Depends(get_session),
|
||||
@@ -335,6 +141,8 @@ async def list_runs(
|
||||
stmt = stmt.where(Run.run_class.in_(classes))
|
||||
if q and q.strip():
|
||||
stmt = stmt.where(Run.workflow_name.ilike(f"%{_like(q.strip())}%", escape="\\"))
|
||||
if schedule_id:
|
||||
stmt = stmt.where(Run.schedule_id == schedule_id)
|
||||
if before is not None:
|
||||
stmt = stmt.where(Run.created_at < before)
|
||||
return list((await session.execute(stmt)).scalars())
|
||||
|
||||
@@ -65,6 +65,10 @@ class EventType(StrEnum):
|
||||
# 人工介入
|
||||
HUMAN_REQUESTED = "human.requested"
|
||||
HUMAN_RESOLVED = "human.resolved"
|
||||
# 审批质询:运行停在审批时,审批人就待审的报告问的一个问题和得到的回答(engine/run_append.py 追加,
|
||||
# node_id 为空)。这次问答的用量写在事件数据的 usage 里,计进运行用量(runner._usage_of_events)。
|
||||
# 停在审批时追加的随运行结束一起封存;封存之后追加的带 post_seal: true,是批注不是证据
|
||||
APPROVAL_INQUIRY = "approval.inquiry"
|
||||
|
||||
# 出具与口径治理
|
||||
ISSUANCE = "issuance" # 出具契约的判定结果(档位 / 缺数据 / 未回指数字)
|
||||
|
||||
@@ -200,6 +200,17 @@ _COLUMN_MIGRATIONS: list[tuple[str, str, str]] = [
|
||||
("import_stagings", "edits", "JSON"),
|
||||
("import_stagings", "answers_dropped", "JSON"),
|
||||
("import_stagings", "redraft_sha256", "VARCHAR(64)"),
|
||||
# 运行选项(发起时的快照,整次运行沿用)和发起它的定时任务。存量运行回填 {} 和 NULL:
|
||||
# 没有选项,也不是定时任务发起的
|
||||
("runs", "options", "JSON DEFAULT '{}'"),
|
||||
("runs", "schedule_id", "VARCHAR(32)"),
|
||||
]
|
||||
|
||||
#: 迁移补上的列要建的索引:(索引名, 表, 列)。create_all 不碰已有的表,表上新列的索引也就不会建。
|
||||
#: 索引名照 NAMING_CONVENTION 取,和新库 create_all 建出来的同名,IF NOT EXISTS 才不会重复建
|
||||
_INDEX_MIGRATIONS: list[tuple[str, str, str]] = [
|
||||
# 定时任务的触发历史按它找这个任务发起的运行
|
||||
("ix_runs_schedule_id", "runs", "schedule_id"),
|
||||
]
|
||||
|
||||
|
||||
@@ -210,6 +221,10 @@ def _migrate(conn) -> None: # pragma: no cover - 同步回调
|
||||
cols = {row[1] for row in conn.execute(text(f"PRAGMA table_info({table})"))}
|
||||
if cols and column not in cols:
|
||||
conn.execute(text(f"ALTER TABLE {table} ADD COLUMN {column} {ddl}"))
|
||||
for name, table, column in _INDEX_MIGRATIONS:
|
||||
cols = {row[1] for row in conn.execute(text(f"PRAGMA table_info({table})"))}
|
||||
if column in cols:
|
||||
conn.execute(text(f"CREATE INDEX IF NOT EXISTS {name} ON {table} ({column})"))
|
||||
|
||||
|
||||
async def init_db() -> None:
|
||||
|
||||
@@ -491,6 +491,11 @@ class Run(Base, TimestampMixin):
|
||||
# 发起时钉住的上传表格版本:{source_id: {"snapshot": 快照 id, "name": 源名}}。续跑、恢复沿用,
|
||||
# 回收(table_versions.gc)不删任何运行引用的快照。None 是升级前发起的运行
|
||||
data_versions: Mapped[dict[str, Any] | None] = mapped_column(default=None)
|
||||
# 发起时定下、整次运行沿用的运行选项(比如核查设置的快照)。审批停下期间改了设置,恢复和重放
|
||||
# 仍按发起时的这份——同 approval_default 的道理。非空时写进 run.started,进封存范围。存量运行是 {}
|
||||
options: Mapped[dict[str, Any]] = mapped_column(default=dict)
|
||||
# 发起它的定时任务,手动发起的为空。不设外键:定时任务删掉之后,运行记录照旧是追溯凭证,来源 id 留着
|
||||
schedule_id: Mapped[str | None] = mapped_column(String(32), default=None, index=True)
|
||||
|
||||
events: Mapped[list["RunEvent"]] = relationship(
|
||||
back_populates="run", cascade="all, delete-orphan"
|
||||
|
||||
@@ -0,0 +1,283 @@
|
||||
"""发起一次运行:HTTP 发起(api/runs.py 的 start_run)和定时任务到点发起共用的那套规则。
|
||||
|
||||
以前这些规则全写在 start_run 里、只认 HTTP 请求:定时任务要发起正式运行,要么另抄一份(两份规则迟早
|
||||
走样,正式运行「只能从当前发布版本发起」这条最要紧的规则就可能在一边被绕过),要么假装成一次 HTTP 请求。
|
||||
现在两边都调 launch_run。拒绝时抛 LaunchRefused:start_run 把它换回以前的 HTTP 响应(状态码、detail、
|
||||
code 逐字不变),调度器据此记一次未能发起。别的异常(库出错之类)也说明运行没有开始执行:发起时定下的事件
|
||||
(受管级别、目录变化、口径升版处置)和运行记录同一个事务提交,执行任务是最后一步才建的。
|
||||
|
||||
发起前检查绑定的工具(missing_tools),以及它和 runner 固定数据版本共用的取绑定的办法(tool_bindings、
|
||||
node_setting)也放在这里:runner 以前要从接口层导入它们。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core import artifact_store
|
||||
from app.core.events import EventType
|
||||
from app.data.catalog_impact import catalog_drift
|
||||
from app.db.models import Run, Workflow, WorkflowVersion
|
||||
from app.engine.governance import governance_note, unresolved_caliber_upgrades
|
||||
from app.engine.runner import run_manager
|
||||
from app.engine.schema import GraphSpec, NodeType, type_label
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
#: 发起时发现绑定的工具在本机不存在(见 missing_tools)
|
||||
TOOL_MISSING = "run_tool_missing"
|
||||
|
||||
|
||||
class LaunchRefused(Exception):
|
||||
"""不发起这次运行。status_code 和 message 就是 HTTP 发起时回的状态码和 detail;code 是给程序认的机器码,
|
||||
没有的是 None(HTTP 响应里也就没有 code 字段)。"""
|
||||
|
||||
def __init__(self, status_code: int, message: str, code: str | None = None) -> None:
|
||||
super().__init__(message)
|
||||
self.status_code = status_code
|
||||
self.message = message
|
||||
self.code = code
|
||||
|
||||
|
||||
async def launch_run(
|
||||
session: AsyncSession,
|
||||
*,
|
||||
workflow_id: str | None = None,
|
||||
graph: dict[str, Any] | None = None,
|
||||
input_payload: dict[str, Any] | None = None,
|
||||
memory_scope: str | None = None,
|
||||
collection: str | None = None,
|
||||
run_class: str = "exploratory",
|
||||
version: int | None = None,
|
||||
actor: str | None,
|
||||
schedule_id: str | None = None,
|
||||
options: dict[str, Any] | None = None,
|
||||
) -> Run:
|
||||
"""发起一次运行,返回刚建好的运行记录(执行已经交给 run_manager)。不发起就抛 LaunchRefused;抛别的异常时
|
||||
运行也没有开始执行。
|
||||
|
||||
参数和 POST /api/runs 的请求体一一对应(input 叫 input_payload);actor 是记在运行上的发起人。
|
||||
schedule_id 是发起它的定时任务,options 是整次运行沿用的选项快照,都原样交给 run_manager.start。
|
||||
"""
|
||||
# 没挂在工作流上(画布没保存就发起、问数据当场搭的)。老数据里是「临时图」,前端两种都认
|
||||
name = "未保存的工作流"
|
||||
run_version: int | None = None
|
||||
version_hash: str | None = None
|
||||
governed = False
|
||||
formal = run_class == "formal"
|
||||
|
||||
if formal:
|
||||
# 正式运行的全部语义就这一条规则:必须引用一个不可变的已发布版本。
|
||||
# 传裸 graph、或工作流还没发布过,都进不了 formal。
|
||||
if not workflow_id:
|
||||
raise LaunchRefused(400, "正式运行需要从已保存的工作流发起:请求中没有指定工作流")
|
||||
if graph is not None:
|
||||
raise LaunchRefused(400, "正式运行只能使用已发布的版本,画布上未发布的修改不会生效。请先保存并发布")
|
||||
workflow = await session.get(Workflow, workflow_id)
|
||||
if not workflow:
|
||||
raise LaunchRefused(404, "工作流不存在,可能已被删除")
|
||||
# 只看 published_version:status 反映的是当前画布(改过图就退回 draft),
|
||||
# 而 formal 用的是已发布的那一版,两者本来就可以不一致
|
||||
if not workflow.published_version:
|
||||
raise LaunchRefused(409, f"「{workflow.name}」还没有发布版本,请先在编排页发布")
|
||||
run_version = version or workflow.published_version
|
||||
# "不可变"和"过过闸"是两件事。快照确实不可变,但 PATCH 保存出来的
|
||||
# 草稿版本同样有快照——只认 version 存不存在的话,工作流只要发布过
|
||||
# 任意一个版本,之后所有草稿都能以 formal 身份启动,发布期的
|
||||
# validate_graph + lint_for_publish 全部绕过。
|
||||
if run_version != workflow.published_version:
|
||||
raise LaunchRefused(
|
||||
409,
|
||||
f"v{run_version} 不是当前发布版本(v{workflow.published_version})。"
|
||||
"正式运行只能使用当前发布版本;如需运行这一版,请先发布它。",
|
||||
)
|
||||
snapshot = (
|
||||
await session.execute(
|
||||
select(WorkflowVersion).where(
|
||||
WorkflowVersion.workflow_id == workflow.id,
|
||||
WorkflowVersion.version == run_version,
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if not snapshot:
|
||||
raise LaunchRefused(404, f"版本 v{run_version} 不存在")
|
||||
graph = snapshot.graph
|
||||
version_hash = snapshot.graph_hash or artifact_store.graph_hash(graph)
|
||||
name = workflow.name
|
||||
# 按这一版发布时记下的级别;记级别之前发布的老版本没有,照旧看工作流现在的 status
|
||||
governed = (snapshot.level == "governed") if snapshot.level else workflow.status == "governed"
|
||||
|
||||
try:
|
||||
spec = GraphSpec.model_validate(graph)
|
||||
except ValueError as e:
|
||||
# 快照里的图不成形。以前这里没接,由 main.py 的 ValueError 兜底回 400、detail 是校验原话;
|
||||
# 原样换成拒绝,HTTP 响应不变,调度器也拿得到一个明确的拒绝
|
||||
raise LaunchRefused(400, str(e)) from e
|
||||
# 口径升版是被迫处置的事件:引用的方法卡有新版本而未声明策略,拒绝启动
|
||||
errors, upgrade_events = await unresolved_caliber_upgrades(session, spec)
|
||||
if errors:
|
||||
raise LaunchRefused(409, ";".join(errors))
|
||||
# 数据目录和口径卡不同:目录是给人和助手看的说明,不是运行的输入。发布之后变了只提醒、不拦——
|
||||
# 拦下来等于让一次文字修订卡住正式出具
|
||||
catalog_changes = await catalog_drift(session, snapshot.catalog_versions)
|
||||
else:
|
||||
upgrade_events = []
|
||||
catalog_changes = []
|
||||
if workflow_id:
|
||||
workflow = await session.get(Workflow, workflow_id)
|
||||
if not workflow:
|
||||
raise LaunchRefused(404, "工作流不存在,可能已被删除")
|
||||
graph = graph or workflow.graph
|
||||
name = workflow.name
|
||||
if not graph:
|
||||
raise LaunchRefused(400, "没有要运行的内容:请求中既没有工作流 ID,也没有工作流内容")
|
||||
await _refuse_missing_tools(session, graph)
|
||||
|
||||
from app.api.settings import resolve_run_scope
|
||||
|
||||
memory_scope, collection = await resolve_run_scope(session, memory_scope, collection)
|
||||
# 发起时就定下的事件交给 run_manager.start,和运行记录同一个事务写进事件流(在封存范围内、排在 run.started
|
||||
# 前面)。以前运行建好、开始执行之后才另外写:写失败时运行已经在执行,这里却抛出异常,定时任务会把一次
|
||||
# 正在执行的正式运行记成未能发起
|
||||
notes: list[tuple[EventType | str, str | None, dict[str, Any]]] = []
|
||||
if formal:
|
||||
# 按不按受管出具在发起这一刻定下来:status 反映的是当前画布,中途改图、审批停下再恢复时再去读它,
|
||||
# 报告节点和出口复核可能各读到不同的值
|
||||
notes.append(("log", None, governance_note(governed=governed, workflow_name=name, version=run_version)))
|
||||
if catalog_changes:
|
||||
# 之后回看这次正式出具,能知道当时的目录和发布时不一样、差在哪几张表。每张表带 impact:direct 写死的
|
||||
# SQL 用到的,possible Agent 可能查询的(界面分开说)
|
||||
notes.append((EventType.CATALOG_DRIFT, None,
|
||||
{"version": run_version, "count": len(catalog_changes), "tables": catalog_changes}))
|
||||
# 已声明策略的升版记入事件流,出具物上能看到"这期换口径了、怎么处置的"
|
||||
for ev in upgrade_events:
|
||||
payload_ev = dict(ev)
|
||||
node = payload_ev.pop("node_id", None)
|
||||
notes.append(("caliber.upgrade", node, payload_ev))
|
||||
try:
|
||||
run = await run_manager.start(
|
||||
graph=graph,
|
||||
input_payload=input_payload if input_payload is not None else {},
|
||||
workflow_id=workflow_id,
|
||||
workflow_name=name,
|
||||
memory_scope=memory_scope,
|
||||
collection=collection,
|
||||
run_class="formal" if formal else "exploratory",
|
||||
version=run_version,
|
||||
version_hash=version_hash,
|
||||
started_by=actor,
|
||||
options=options,
|
||||
schedule_id=schedule_id,
|
||||
notes=notes,
|
||||
)
|
||||
except ValueError as e:
|
||||
raise LaunchRefused(400, str(e)) from e
|
||||
return run
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 发起前检查绑定的工具
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def node_setting(spec: GraphSpec, node: Any, key: str) -> Any:
|
||||
"""节点配置的实际取值,和运行时 NodeContext.cfg 同一个规则:节点上没写(None、空串)就取图级 defaults。"""
|
||||
value = node.config.get(key)
|
||||
return spec.defaults.get(key) if value in (None, "") else value
|
||||
|
||||
|
||||
def tool_bindings(spec: GraphSpec) -> list[tuple[str, str]]:
|
||||
"""图里每一处绑定的工具:(谁绑的, 工具名)。调用工具节点的 tool、Agent 的 tools、协作成员的 tools。
|
||||
|
||||
取值和运行时一致(node_setting):Agent 节点上没写 tools、协作节点上没写成员,运行时用的是
|
||||
图级 defaults 里的,这里也算它们——不然发起前检查放过了缺的工具,发起时固定数据版本也漏了源。
|
||||
|
||||
模板拼出来的名字({{ vars.tool }})也在内:发起前检查(_bindings)跳过它们,发起时固定数据
|
||||
版本(engine/runner.py)要知道有没有这种名字——有的话说不准会用到哪个数据源。
|
||||
"""
|
||||
out: list[tuple[str, str]] = []
|
||||
for node in spec.nodes:
|
||||
who = f"「{node.title}」({type_label(node.type)})"
|
||||
if node.type == NodeType.TOOL:
|
||||
names = [node_setting(spec, node, "tool")]
|
||||
out.extend((who, n) for n in names if isinstance(n, str) and n.strip())
|
||||
elif node.type == NodeType.AGENT:
|
||||
out.extend((who, n) for n in node_setting(spec, node, "tools") or []
|
||||
if isinstance(n, str) and n.strip())
|
||||
elif node.type == NodeType.SUPERVISOR:
|
||||
for member in node_setting(spec, node, "agents") or []:
|
||||
if not isinstance(member, dict):
|
||||
continue
|
||||
label = f"「{node.title}」的成员「{member.get('name') or '?'}」"
|
||||
out.extend((label, n) for n in member.get("tools") or [] if isinstance(n, str) and n.strip())
|
||||
return [(who, name.strip()) for who, name in out]
|
||||
|
||||
|
||||
def _bindings(spec: GraphSpec) -> list[tuple[str, str]]:
|
||||
"""发起前要检查的绑定。模板拼出来的工具名({{ vars.tool }})发起时还不知道是谁,交给运行时。"""
|
||||
return [(who, name) for who, name in tool_bindings(spec) if "{{" not in name]
|
||||
|
||||
|
||||
async def missing_tools(session: AsyncSession, spec: GraphSpec) -> list[tuple[str, str, str]]:
|
||||
"""本机不存在的绑定:(谁绑的, 工具名, 去哪接入:data / tools)。
|
||||
|
||||
解析和运行时 build_tools 一致:内置(静态注册)、mcp:<服务>/<工具>、db_query__ / db_schema__
|
||||
<数据源>、其余按自定义工具。数据源和自定义工具只认启用着的——停用的运行时同样拿不到。
|
||||
MCP 只看服务登记着、启用着没有:连不连得上、工具还在不在,要连一次才知道,发起时不去等它,
|
||||
上次没连上或者工具清单里没有的只在日志里提一句。
|
||||
"""
|
||||
from app.db.models import CustomTool, DataSource, McpServer
|
||||
from app.tools.datasource import QUERY_PREFIX, SCHEMA_PREFIX
|
||||
from app.tools.registry import all_specs
|
||||
|
||||
bindings = _bindings(spec)
|
||||
if not bindings:
|
||||
return []
|
||||
builtin = all_specs()
|
||||
sources = {n for (n,) in await session.execute(select(DataSource.name).where(DataSource.enabled.is_(True)))}
|
||||
custom = {n for (n,) in await session.execute(select(CustomTool.name).where(CustomTool.enabled.is_(True)))}
|
||||
servers = {r.name: r for r in (await session.execute(select(McpServer))).scalars()}
|
||||
missing: list[tuple[str, str, str]] = []
|
||||
for who, name in bindings:
|
||||
if name.startswith("mcp:"):
|
||||
server, _, tool = name[4:].partition("/")
|
||||
row = servers.get(server)
|
||||
if row is None or not row.enabled:
|
||||
missing.append((who, name, "tools"))
|
||||
elif row.status == "error":
|
||||
logger.warning("%s 绑的 %s:MCP 服务 %s 上次没连上,运行时可能拿不到这个工具", who, name, server)
|
||||
elif tool not in ("", "*") and row.tools_cache and tool not in row.tools_cache:
|
||||
logger.warning("%s 绑的 %s:MCP 服务 %s 上次交回的工具清单里没有 %s", who, name, server, tool)
|
||||
elif name in builtin:
|
||||
continue
|
||||
elif name.startswith((QUERY_PREFIX, SCHEMA_PREFIX)):
|
||||
source = name[len(QUERY_PREFIX):] if name.startswith(QUERY_PREFIX) else name[len(SCHEMA_PREFIX):]
|
||||
if source not in sources:
|
||||
missing.append((who, name, "data"))
|
||||
elif name not in custom:
|
||||
missing.append((who, name, "tools"))
|
||||
return missing
|
||||
|
||||
|
||||
async def _refuse_missing_tools(session: AsyncSession, graph: dict[str, Any]) -> None:
|
||||
"""绑定的工具在本机不存在就不发起:以前要执行到那一步才失败,前面的步骤白做、钱白花。"""
|
||||
try:
|
||||
spec = GraphSpec.model_validate(graph)
|
||||
except Exception: # noqa: BLE001 - 图本身不成形交给 run_manager.start 报
|
||||
return
|
||||
missing = await missing_tools(session, spec)
|
||||
if not missing:
|
||||
return
|
||||
lines = ";".join(f"{who}绑定的 {name}" for who, name, _ in missing[:8])
|
||||
more = f"等 {len(missing)} 处" if len(missing) > 8 else ""
|
||||
places = {where for _, _, where in missing}
|
||||
if places == {"data"}:
|
||||
fix = "请到「数据」页接入,或在节点中重新选择"
|
||||
elif places == {"tools"}:
|
||||
fix = "请到「工具」页接入,或在节点中重新选择"
|
||||
else:
|
||||
fix = "数据源工具请到「数据」页接入,其他工具请到「工具」页接入,或在节点中重新选择"
|
||||
raise LaunchRefused(422, f"绑定的工具不存在:{lines}{more}。{fix}", TOOL_MISSING)
|
||||
@@ -0,0 +1,160 @@
|
||||
"""往一次停下的运行里追加事件:运行停在审批时的审批质询、封存之后的批注。
|
||||
|
||||
事件流是封存的依据:清单按 seq 排序算哈希,而 run_events 的 (run_id, seq) 没有唯一约束——两条事件撞了号,
|
||||
排序就说不准,封存核对时对时不对。所以追加和 runner 发事件守同一套序号:接在库里最后一条后面,从 bus 取号
|
||||
(同一进程里和执行任务共用一个计数器),写事件和条件更新 last_seq 在同一个事务里,谁先提交算谁的。做法同
|
||||
api/evidence.py 的 _append_judged(按需裁判的判定)。
|
||||
|
||||
require 说明往哪种运行里追加:
|
||||
- "interrupted":运行停在审批、还没封存。事件落在以后的封存范围里,随运行结束一起封存。
|
||||
- "sealed":已经封存的终态运行。事件带 post_seal: true,落在 manifest_seq 之后,不进封存范围,verify 照旧一致。
|
||||
|
||||
没抢到(读完运行之后它被恢复、被放弃、继续运行后重新封存了,问的那张审批卡已经批过了,或者别处刚往里写过
|
||||
事件)就一条都不写、返回 None,调用方据此说明「未写入运行记录」。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
from collections.abc import AsyncIterator, Sequence
|
||||
from typing import Any, Literal
|
||||
|
||||
from sqlalchemy import exists, func, select, update
|
||||
|
||||
from app.core.bus import bus
|
||||
from app.core.events import EventType, RunEventModel
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import Approval, Run, RunEvent
|
||||
from app.engine.runner import _seq_floor, _stage, run_manager
|
||||
|
||||
#: 运行的终态。封存过的运行只会停在这几种上
|
||||
_TERMINAL = ("succeeded", "failed", "cancelled")
|
||||
#: 宣布运行结束的事件。封存盖住了最后一条这种事件,才是这次结束之后封的(见 _current_seal)
|
||||
_ENDINGS = (EventType.RUN_FINISHED, EventType.RUN_FAILED, EventType.RUN_CANCELLED)
|
||||
|
||||
#: 一条要追加的事件:(事件类型, node_id, 数据)
|
||||
Item = tuple[EventType | str, str | None, dict[str, Any] | None]
|
||||
|
||||
|
||||
class _RunLocks:
|
||||
"""每次运行一把锁,没人用了就扔:同一次运行的追加在本进程里排队,读 last_seq 到条件更新之间不会被
|
||||
另一次追加插进来——插进来的话后提交的那次条件更新落空,明明没有冲突也成了「未写入」。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._locks: dict[str, asyncio.Lock] = {}
|
||||
self._users: dict[str, int] = {}
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def hold(self, key: str) -> AsyncIterator[None]:
|
||||
lock = self._locks.setdefault(key, asyncio.Lock())
|
||||
self._users[key] = self._users.get(key, 0) + 1
|
||||
try:
|
||||
async with lock:
|
||||
yield
|
||||
finally:
|
||||
self._users[key] -= 1
|
||||
if not self._users[key]:
|
||||
del self._users[key], self._locks[key]
|
||||
|
||||
|
||||
_APPENDING = _RunLocks()
|
||||
|
||||
|
||||
async def append_events(
|
||||
run_id: str,
|
||||
items: Sequence[Item],
|
||||
*,
|
||||
require: Literal["interrupted", "sealed"],
|
||||
seal: tuple[int | None, str | None] | None = None,
|
||||
approval_id: str | None = None,
|
||||
) -> list[RunEventModel] | None:
|
||||
"""把 items 依次接在运行记录的最后一条后面,返回写入的事件(seq 严格递增,id 和落库的相同);没写入返回 None。
|
||||
|
||||
调用方开始做事时看到的状态可以钉住,做的事要花几秒(问模型),这期间变了就不追加:
|
||||
- seal 只用于 "sealed":当时的 (manifest_seq, manifest_hash)。继续运行、重新封存了的话,结果依据的已经
|
||||
不是现在的封存。不传就按这次读到的封存。
|
||||
- approval_id 只用于 "interrupted":问的是哪张审批卡。那张卡仍待审批才写——批过了,运行可能已经停到了
|
||||
下一张卡上,仍是「停在审批」,关于上一张的问答却会写进这一次停下里。
|
||||
items 为空时什么都不做,返回空列表。
|
||||
"""
|
||||
if require not in ("interrupted", "sealed"):
|
||||
raise ValueError(f"require 只能是 interrupted 或 sealed,收到的是 {require!r}")
|
||||
if approval_id is not None and require != "interrupted":
|
||||
raise ValueError("approval_id 只用于 interrupted:封存之后没有待审批的卡")
|
||||
# 事件类型先验过:写错了在取号之前就报出来
|
||||
entries = [(EventType(str(kind)), node_id, dict(data or {})) for kind, node_id, data in items]
|
||||
if not entries:
|
||||
return []
|
||||
sealed = require == "sealed"
|
||||
for _, _, data in entries:
|
||||
if sealed:
|
||||
data["post_seal"] = True
|
||||
else:
|
||||
# 停在审批时追加的事件要进封存范围,不能自称在封存之后
|
||||
data.pop("post_seal", None)
|
||||
|
||||
async with _APPENDING.hold(run_id):
|
||||
async with SessionLocal() as session:
|
||||
run = await session.get(Run, run_id)
|
||||
if run is None:
|
||||
return None
|
||||
if sealed:
|
||||
if run.status not in _TERMINAL or not run.manifest_hash \
|
||||
or (seal is not None and (run.manifest_seq, run.manifest_hash) != tuple(seal)) \
|
||||
or not await _current_seal(session, run):
|
||||
return None
|
||||
guard = (Run.status.in_(_TERMINAL), _same(Run.manifest_seq, run.manifest_seq),
|
||||
_same(Run.manifest_hash, run.manifest_hash))
|
||||
else:
|
||||
if run.status != "interrupted":
|
||||
return None
|
||||
guard = (Run.status == "interrupted",)
|
||||
if approval_id is not None:
|
||||
guard += (exists().where(Approval.id == approval_id, Approval.run_id == run_id,
|
||||
Approval.status == "pending"),)
|
||||
last = run.last_seq
|
||||
# 进程重启过的话内存里的序号从 0 起:接在库里最后一条后面。last_seq 按说就是最后一条,_seq_floor
|
||||
# 另取一次实际的最大序号兜底——别处的追加把 last_seq 写小了,也不至于在这里撞号
|
||||
bus.set_seq(run_id, await _seq_floor(session, run_id, last))
|
||||
events = [run_manager._event(run_id, kind, node_id=node_id, data=data)
|
||||
for kind, node_id, data in entries]
|
||||
# 核对和写入是同一行上的条件更新,和恢复、放弃、继续运行(runner._claim / _abandon)抢的是同一行。
|
||||
# 不同步会话里的 run(下面 _stage 会把它的 last_seq 改成同一个值):不写明的话,条件里带了 EXISTS 时
|
||||
# SQLAlchemy 会悄悄换一种同步办法、给这条语句加上 RETURNING
|
||||
claimed = await session.execute(
|
||||
update(Run).where(Run.id == run_id, _same(Run.last_seq, last), *guard)
|
||||
.values(last_seq=events[-1].seq).execution_options(synchronize_session=False))
|
||||
if claimed.rowcount != 1:
|
||||
# 读完运行之后它变了。烧掉几个序号不要紧:不落库的事件(流式增量)本来就占序号,
|
||||
# 序号只要求递增、不撞号
|
||||
await session.rollback()
|
||||
return None
|
||||
for event in events:
|
||||
await _stage(session, event, run)
|
||||
await session.commit()
|
||||
# 先落库再广播,理由同 RunManager._emit
|
||||
for event in events:
|
||||
await bus.publish(event)
|
||||
return events
|
||||
|
||||
|
||||
def _same(column: Any, value: Any) -> Any:
|
||||
return column.is_(None) if value is None else column == value
|
||||
|
||||
|
||||
async def _current_seal(session: Any, run: Run) -> bool:
|
||||
"""运行记录上的封存是不是这次结束之后封的:manifest_seq 盖住了最后一条宣布结束的事件。
|
||||
|
||||
继续运行之后再次结束时,终态和终态事件先提交、封存后算。这中间运行记录上还是上一次结束时的封存,
|
||||
当成现在的封存去追加,事件会被接下来的封存收进去,post_seal 就成了假话。
|
||||
|
||||
manifest_seq 为空的是加这一列之前封存的老运行,封到哪一条是 verify_manifest 按老口径推出来的:有终态事件的
|
||||
封到最后一条终态事件,追加在后面不碍事;没有的封到最后一行,追加一条,范围就挪到它身上,我们自己写的
|
||||
批注被当成篡改。所以老运行要有终态事件才追加。
|
||||
"""
|
||||
ended = await session.scalar(
|
||||
select(func.max(RunEvent.seq)).where(RunEvent.run_id == run.id,
|
||||
RunEvent.type.in_([str(t) for t in _ENDINGS])))
|
||||
if run.manifest_seq is None:
|
||||
return ended is not None
|
||||
return ended is None or run.manifest_seq >= ended
|
||||
+138
-17
@@ -2,17 +2,18 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import copy
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Callable, Sequence
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
|
||||
from langgraph.errors import GraphRecursionError
|
||||
from langgraph.types import Command
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy import case, func, select, update
|
||||
|
||||
from app.core.bus import bus
|
||||
from app.core.config import settings
|
||||
@@ -33,6 +34,10 @@ logger = logging.getLogger(__name__)
|
||||
# 轨迹里正文靠 node.finished 的 preview,思考靠调用结束时的单条 llm.thinking 汇总
|
||||
_EPHEMERAL = {EventType.LLM_TOKEN, EventType.LLM_THINKING_DELTA}
|
||||
|
||||
#: RunContext.extra 里放这次运行的选项(Run.options)的键。节点从这里读发起时定下的快照、不读设置的
|
||||
#: 当前值:审批停下期间改了设置,恢复后的那一段也按发起时的,重放和正式运行才可复现
|
||||
RUN_OPTIONS_KEY = "run_options"
|
||||
|
||||
|
||||
class RunManager:
|
||||
"""负责一次运行的完整生命周期:启动、串流、中断、恢复、取消。"""
|
||||
@@ -61,6 +66,15 @@ class RunManager:
|
||||
# 一个 checkpoint 都没写过,"恢复"它只会从 START 用默认值重跑一遍,
|
||||
# 还丢掉 run.input——那不是恢复,是伪造一份结果。
|
||||
async with SessionLocal() as session:
|
||||
# 先把落后于库里实际最后一条的 last_seq 改正(为什么会落后见 _seq_floor)。内存里的序号计数刚清零,
|
||||
# 接下来各处都按 last_seq 接号:本模块的几处另外兜了底,按需裁判记判定(api/evidence.py 的
|
||||
# _append_judged)没有,不改正的话它重启后第一次追加就撞上已有的序号。有 (run_id, seq) 索引,
|
||||
# 每次运行一次索引查找。updated_at 不动:这是补记录,不是运行有了变化
|
||||
latest = select(func.max(RunEvent.seq)).where(RunEvent.run_id == Run.id).scalar_subquery()
|
||||
await session.execute(
|
||||
update(Run).where(func.coalesce(Run.last_seq, 0) < latest)
|
||||
.values(last_seq=latest, updated_at=Run.updated_at)
|
||||
.execution_options(synchronize_session=False))
|
||||
stranded = list((await session.execute(
|
||||
select(Run.id, Run.last_seq).where(Run.status == "running")
|
||||
)).all())
|
||||
@@ -158,7 +172,7 @@ class RunManager:
|
||||
self, run_id: str, event_type: EventType | str, *, node_id: str | None = None,
|
||||
**data: Any,
|
||||
) -> None:
|
||||
"""给 API 层用的事件入口(比如 formal 启动时记录口径升版处置)。"""
|
||||
"""给运行之外的代码往运行里记一条事件。发起时就定下的事件不走这里,交给 start 的 notes。"""
|
||||
await self._emit(run_id, event_type, node_id=node_id, data=data)
|
||||
|
||||
# ---------------- 启动 ----------------
|
||||
@@ -177,7 +191,16 @@ class RunManager:
|
||||
version_hash: str | None = None,
|
||||
started_by: str | None = None,
|
||||
approval_default: str | None = None,
|
||||
options: dict[str, Any] | None = None,
|
||||
schedule_id: str | None = None,
|
||||
notes: Sequence[tuple[EventType | str, str | None, dict[str, Any]]] = (),
|
||||
) -> Run:
|
||||
"""建运行记录、交给执行任务。options 是这次运行的选项快照,schedule_id 是发起它的定时任务(可空)。
|
||||
|
||||
notes 是发起时就定下的事件 (类型, node_id, 数据),比如正式运行按什么级别出具、口径升版怎么处置。它们和
|
||||
运行记录同一个事务提交,排在 run.started 前面:写不进去就不建运行,不会出现运行已经在执行、调用方却
|
||||
只拿到一个异常(定时任务会把它记成未能发起);也不会和执行抢先后,运行结束得快时落到封存之后。
|
||||
"""
|
||||
spec = GraphSpec.model_validate(graph)
|
||||
report = validate_graph(spec)
|
||||
if not report.ok:
|
||||
@@ -187,6 +210,8 @@ class RunManager:
|
||||
)
|
||||
if approval_default is None:
|
||||
approval_default = await _approval_default(run_class=run_class)
|
||||
# 拷一份:调用方手上的 dict 之后再改,不该改到这次运行的快照
|
||||
options = copy.deepcopy(options) if options else {}
|
||||
|
||||
async with SessionLocal() as session:
|
||||
from app.data.table_versions import pin_lock
|
||||
@@ -214,11 +239,23 @@ class RunManager:
|
||||
tool_trust=tool_trust,
|
||||
agent_limits=agent_limits,
|
||||
data_versions=data_versions,
|
||||
options=options,
|
||||
schedule_id=schedule_id,
|
||||
)
|
||||
run.thread_id = run.id # 一个 run 一条 checkpoint 线程
|
||||
session.add(run)
|
||||
staged: list[RunEventModel] = []
|
||||
if notes:
|
||||
await session.flush() # 事件要带上运行的 id
|
||||
staged = [self._event(run.id, kind, node_id=node_id, data=data)
|
||||
for kind, node_id, data in notes]
|
||||
for event in staged:
|
||||
await _stage(session, event, run)
|
||||
await session.commit()
|
||||
await session.refresh(run)
|
||||
# 先落库再广播,理由同 _emit;执行任务还没建,run.started 必定排在它们后面
|
||||
for event in staged:
|
||||
await bus.publish(event)
|
||||
|
||||
self._tasks[run.id] = asyncio.create_task(
|
||||
self._drive(
|
||||
@@ -232,6 +269,7 @@ class RunManager:
|
||||
tool_trust=tool_trust,
|
||||
agent_limits=agent_limits,
|
||||
data_versions=data_versions,
|
||||
options=options,
|
||||
)
|
||||
)
|
||||
return run
|
||||
@@ -379,7 +417,7 @@ class RunManager:
|
||||
.values(status="resolved")
|
||||
)
|
||||
run = await session.get(Run, run_id)
|
||||
bus.set_seq(run_id, run.last_seq)
|
||||
bus.set_seq(run_id, await _seq_floor(session, run_id, run.last_seq))
|
||||
await session.commit()
|
||||
await session.refresh(run)
|
||||
|
||||
@@ -479,7 +517,7 @@ class RunManager:
|
||||
**({"graph": spec.model_dump(mode="json")} if graph is not None else {}),
|
||||
**({"started_by": actor} if actor else {}))
|
||||
run = await session.get(Run, run_id)
|
||||
bus.set_seq(run_id, run.last_seq)
|
||||
bus.set_seq(run_id, await _seq_floor(session, run_id, run.last_seq))
|
||||
await session.commit()
|
||||
await session.refresh(run)
|
||||
|
||||
@@ -538,9 +576,14 @@ class RunManager:
|
||||
closed = await _close_approvals(session, run_id, actor, now)
|
||||
run = await session.get(Run, run_id)
|
||||
timing = await _timing(session, run, 0, now)
|
||||
run.usage = {**(run.usage or {}), "duration_ms": timing["active_ms"], **timing}
|
||||
usage = dict(run.usage or {})
|
||||
# 停下时记的用量里还没有这次停在审批期间问的质询,这里补上(上次停下时计过的不再计)
|
||||
inquiry = (await _usage_of_events(session, run_id)).get("inquiry")
|
||||
if inquiry or usage.get("inquiry"):
|
||||
usage = _with_inquiry(usage, inquiry)
|
||||
run.usage = {**usage, "duration_ms": timing["active_ms"], **timing}
|
||||
run.error = "用户取消:等待审批时放弃了这次运行" if closed else "用户取消:挂起期间放弃了这次运行"
|
||||
bus.set_seq(run_id, run.last_seq or 0)
|
||||
bus.set_seq(run_id, await _seq_floor(session, run_id, run.last_seq))
|
||||
# 和状态同一个事务提交,理由同 _finalize
|
||||
event = self._event(run_id, EventType.RUN_CANCELLED, data={
|
||||
"timing": timing, "actor": actor,
|
||||
@@ -585,6 +628,7 @@ class RunManager:
|
||||
tool_trust: dict[str, str] | None = None,
|
||||
agent_limits: dict[str, Any] | None = None,
|
||||
data_versions: dict[str, Any] | None = None,
|
||||
options: dict[str, Any] | None = None,
|
||||
resumed: list[str] | tuple[str, ...] = (),
|
||||
actors: dict[str, str | None] | None = None,
|
||||
) -> None:
|
||||
@@ -631,7 +675,10 @@ class RunManager:
|
||||
# 运行记录里的那份,含运行中途第一次用到时补上的;None 是升级前的运行
|
||||
"data_versions": data_versions,
|
||||
# 这一段是按哪一版重放协议跑的,恢复时据此认出升级前做下的工作
|
||||
"replay_protocol": PROTOCOL},
|
||||
"replay_protocol": PROTOCOL,
|
||||
# 发起时定下的运行选项(核查设置之类),每一段都记、记的是同一份。没有选项的
|
||||
# 运行不带这个键,和加选项之前的 run.started 逐字相同
|
||||
**({"options": options} if options else {})},
|
||||
)
|
||||
|
||||
run_ctx = RunContext(
|
||||
@@ -648,7 +695,9 @@ class RunManager:
|
||||
# 的 run_versions)。复制一份:运行中途补固定的源就地写进这个 dict,同一段执行里
|
||||
# 后面的节点都看得到,但不该碰到别处(运行记录、事件)手上的那个对象
|
||||
extra={"resumed": set(resumed), "actors": dict(actors or {}),
|
||||
RUN_VERSIONS_KEY: dict(data_versions) if data_versions is not None else None},
|
||||
RUN_VERSIONS_KEY: dict(data_versions) if data_versions is not None else None,
|
||||
# 同理也是一份拷贝:节点读它,不该改到运行记录那份
|
||||
RUN_OPTIONS_KEY: copy.deepcopy(options or {})},
|
||||
)
|
||||
app = compile_graph(spec, run_ctx).compile(checkpointer=self.checkpointer)
|
||||
# 循环按自己声明的轮数另记一份预算,全局上限只管没人把关的环
|
||||
@@ -838,6 +887,7 @@ class RunManager:
|
||||
if run:
|
||||
timing = await _timing(session, run, elapsed_ms, now)
|
||||
events_usage = await _usage_of_events(session, run_id)
|
||||
inquiry = events_usage.pop("inquiry", None)
|
||||
if status != "succeeded":
|
||||
# 状态里的用量只算跑完了的节点。失败、取消、停在审批上的那个节点已经
|
||||
# 花出去的调用(校验修复、agent 的前几步)只在 llm.end 里——以前这部分
|
||||
@@ -846,6 +896,9 @@ class RunManager:
|
||||
elif events_usage.get("unpriced_calls"):
|
||||
# 金额估不出来的调用只有 llm.end 数得出(状态里的用量只有金额,0 分不出是免费还是没价)
|
||||
usage = {**usage, "unpriced_calls": events_usage["unpriced_calls"]}
|
||||
if inquiry:
|
||||
# 审批质询不在图里执行,状态和 llm.end 两本账里都没有它:比完再加上
|
||||
usage = _with_inquiry(usage, inquiry)
|
||||
# duration_ms 是各段执行时长之和(= active_ms)。以前用这一段的耗时
|
||||
# 覆盖它:87 秒、中间等过一次审批的运行,列表上写着 22ms
|
||||
usage.update(duration_ms=timing["active_ms"], **timing)
|
||||
@@ -1025,6 +1078,8 @@ def _carried(run: Run) -> dict[str, Any]:
|
||||
"agent_limits": run.agent_limits,
|
||||
# 固定的数据版本照运行记录来,不重新读数据源的当前指针:中途重传过,续跑查的仍是发起时那一版
|
||||
"data_versions": run.data_versions,
|
||||
# 运行选项同理:照发起时的快照,不重新读设置
|
||||
"options": run.options or {},
|
||||
}
|
||||
|
||||
|
||||
@@ -1062,11 +1117,12 @@ async def _data_versions(session: Any, spec: GraphSpec) -> dict[str, Any]:
|
||||
async def _bound_sources(session: Any, spec: GraphSpec) -> tuple[set[str], bool]:
|
||||
"""图里(含子工作流)绑定的数据源工具指向的源名,以及有没有带模板、发起时定不下来的工具名。
|
||||
|
||||
绑定的取法和发起前检查同一个(api/runs.py 的 tool_bindings),子工作流照运行时的取法
|
||||
绑定的取法和发起前检查同一个(engine/launch.py 的 tool_bindings),子工作流照运行时的取法
|
||||
读出来接着找:固定了版本的读那一版,没固定的读工作流现在的图。读不到、图不成形的跳过——
|
||||
那个节点运行时自己会报错。同一个工作流(同一版)只读一次,互相引用也不会转圈。
|
||||
"""
|
||||
from app.api.runs import node_setting, tool_bindings
|
||||
# 在函数里导入:launch 在模块级导入了本模块(它要 run_manager)
|
||||
from app.engine.launch import node_setting, tool_bindings
|
||||
from app.db.models import WorkflowVersion
|
||||
from app.tools.datasource import source_of_tool
|
||||
|
||||
@@ -1223,17 +1279,36 @@ async def _claim(
|
||||
|
||||
|
||||
async def _stage(session: Any, event: RunEventModel, run: Run | None = None) -> None:
|
||||
"""把一条事件写进这个 session,随它一起提交。run 已经在 session 里时直接改它的 last_seq。"""
|
||||
session.add(RunEvent(run_id=event.run_id, seq=event.seq, type=str(event.type),
|
||||
"""把一条事件写进这个 session,随它一起提交。run 已经在 session 里时直接改它的 last_seq。
|
||||
|
||||
落库的 id 用事件自己的:广播出去的、交回给调用方的和库里的是同一个 id,回头按 id 找得到。
|
||||
"""
|
||||
session.add(RunEvent(id=event.id, run_id=event.run_id, seq=event.seq, type=str(event.type),
|
||||
node_id=event.node_id, ts=event.ts, data=event.data))
|
||||
if run is not None:
|
||||
run.last_seq = event.seq
|
||||
else:
|
||||
# 只往大里改:另开会话记的事件(note)和执行任务同时在写时,取号和提交的先后可以不一样,号小的后提交,
|
||||
# 无条件写会把 last_seq 写小(后果见 _seq_floor)
|
||||
await session.execute(
|
||||
update(Run).where(Run.id == event.run_id).values(last_seq=event.seq)
|
||||
update(Run).where(Run.id == event.run_id)
|
||||
.values(last_seq=case((Run.last_seq > event.seq, Run.last_seq), else_=event.seq))
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
|
||||
|
||||
async def _seq_floor(session: Any, run_id: str, last_seq: int | None) -> int:
|
||||
"""接着一次运行往下编号的起点:last_seq 和库里实际最大的序号取大的。
|
||||
|
||||
last_seq 按说就是最后一条,但有的写入会把它写小:按需裁判记判定(api/evidence.py 的 _append_judged)
|
||||
的条件更新不带 last_seq 条件,它取了号还没写、封存后追加(engine/run_append.py)抢先提交了,它再提交时
|
||||
把 last_seq 写回自己那个更小的号。同一进程里序号计数只增不减,不碍事;进程重启后计数从 0 起,只按
|
||||
last_seq 接号就会撞上已有的序号。清单按序号排序算哈希,撞了号封存核对就时对时不对。
|
||||
"""
|
||||
stored = await session.scalar(select(func.max(RunEvent.seq)).where(RunEvent.run_id == run_id))
|
||||
return max(last_seq or 0, stored or 0)
|
||||
|
||||
|
||||
async def _close_approvals(session: Any, run_id: str, actor: str | None, now: datetime) -> int:
|
||||
"""运行已经结束,它还没处理完的审批一并关掉。返回关掉了几条。"""
|
||||
rows = list((await session.execute(
|
||||
@@ -1248,18 +1323,39 @@ async def _close_approvals(session: Any, run_id: str, actor: str | None, now: da
|
||||
|
||||
|
||||
_USAGE_KEYS = ("input_tokens", "output_tokens", "cost_usd", "calls")
|
||||
#: 审批质询的用量里计进运行用量的键:unpriced_calls 是其中金额估不出来的调用次数(模型不在价格目录里)
|
||||
_INQUIRY_KEYS = (*_USAGE_KEYS, "unpriced_calls")
|
||||
|
||||
|
||||
async def _usage_of_events(session: Any, run_id: str) -> dict[str, Any]:
|
||||
"""按 llm.end 事件逐次加总的用量:每次模型调用都有一条,不管节点最后成没成。"""
|
||||
"""按事件加总的用量。
|
||||
|
||||
llm.end 逐次加总:每次模型调用都有一条,不管节点最后成没成。审批质询不在图里执行、不发 llm.end,
|
||||
它的用量写在 approval.inquiry 事件数据的 usage 里,单独加总放在 inquiry 下(没有质询就没有这个键):
|
||||
llm.end 这本账要和状态里的用量比大小(_spent),质询花的两本里都没有,混进去比会被状态那本盖掉。
|
||||
|
||||
封存之后追加的质询(数据带 post_seal)不算:运行用量是这次运行自己花的,结束时就定下了、写进了终态事件;
|
||||
事后的批注另有自己的账,同按需裁判的判定。继续运行之后它落进新的封存范围,照样不算——不然同一条质询
|
||||
计不计,全看之后有没有继续运行。
|
||||
"""
|
||||
rows = await session.execute(
|
||||
select(RunEvent.data).where(RunEvent.run_id == run_id, RunEvent.type == EventType.LLM_END)
|
||||
select(RunEvent.type, RunEvent.data).where(
|
||||
RunEvent.run_id == run_id, RunEvent.type.in_([EventType.LLM_END, EventType.APPROVAL_INQUIRY]))
|
||||
)
|
||||
from app.providers.catalog import unpriced_call
|
||||
|
||||
total: dict[str, Any] = dict.fromkeys(_USAGE_KEYS, 0)
|
||||
unpriced = 0
|
||||
for (data,) in rows:
|
||||
inquiry: dict[str, Any] = {}
|
||||
for kind, data in rows:
|
||||
if kind == EventType.APPROVAL_INQUIRY:
|
||||
spent = (data or {}).get("usage")
|
||||
if isinstance(spent, dict) and not (data or {}).get("post_seal"):
|
||||
for key in _INQUIRY_KEYS:
|
||||
value = spent.get(key)
|
||||
if isinstance(value, (int, float)) and not isinstance(value, bool):
|
||||
inquiry[key] = inquiry.get(key, 0) + value
|
||||
continue
|
||||
for key in _USAGE_KEYS:
|
||||
value = (data or {}).get(key, 1 if key == "calls" else 0)
|
||||
if isinstance(value, (int, float)) and not isinstance(value, bool):
|
||||
@@ -1269,9 +1365,34 @@ async def _usage_of_events(session: Any, run_id: str) -> dict[str, Any]:
|
||||
if unpriced:
|
||||
# 有几次调用的金额估不出来(模型不在价格目录里):cost_usd 只是估得出的那部分,界面据此不写 $0
|
||||
total["unpriced_calls"] = unpriced
|
||||
if inquiry:
|
||||
if "cost_usd" in inquiry:
|
||||
inquiry["cost_usd"] = round(inquiry["cost_usd"], 6)
|
||||
total["inquiry"] = inquiry
|
||||
return total
|
||||
|
||||
|
||||
def _with_inquiry(usage: dict[str, Any], inquiry: dict[str, Any] | None) -> dict[str, Any]:
|
||||
"""把审批质询的用量计进运行用量,明细另放在 inquiry 下(界面要说清其中多少是质询花的)。
|
||||
|
||||
usage 里已经计过一回的(上次停下时记下的 inquiry)先减掉再加:停过两次、两次都问过的运行,
|
||||
每次质询只计一遍。
|
||||
"""
|
||||
out = dict(usage)
|
||||
counted = out.pop("inquiry", None) or {}
|
||||
for key in _INQUIRY_KEYS:
|
||||
delta = (inquiry or {}).get(key, 0) - counted.get(key, 0)
|
||||
if delta:
|
||||
out[key] = (out.get(key) or 0) + delta
|
||||
if isinstance(out.get("cost_usd"), float):
|
||||
out["cost_usd"] = round(out["cost_usd"], 6)
|
||||
if "input_tokens" in out or "output_tokens" in out:
|
||||
out["total_tokens"] = int(out.get("input_tokens") or 0) + int(out.get("output_tokens") or 0)
|
||||
if inquiry:
|
||||
out["inquiry"] = dict(inquiry)
|
||||
return out
|
||||
|
||||
|
||||
def _spent(state_usage: dict[str, Any], events_usage: dict[str, Any]) -> dict[str, Any]:
|
||||
"""两本账取大的那本。新引擎每次调用都发 llm.end,事件那本不会更少;
|
||||
加这条事件之前的老运行(agent 节点不发 llm.end)状态那本更全。"""
|
||||
|
||||
@@ -0,0 +1,279 @@
|
||||
"""审批质询的用量计进运行用量。
|
||||
|
||||
质询不在图里执行、不发 llm.end,它花的 token 和金额写在 approval.inquiry 事件数据的 usage 里
|
||||
(input_tokens、output_tokens、cost_usd、calls,可带 unpriced_calls)。以前运行用量只认图的状态和 llm.end,
|
||||
质询花的钱在运行页上看不见。
|
||||
|
||||
- 运行结束(成功、失败)时:图的用量照旧(成功取状态、其余两本账取大),再加上质询的;明细另放在 inquiry 下
|
||||
- 停在审批时被放弃:停下时记的用量里还没有这次停下期间的质询,放弃时补上
|
||||
- 停过两次、两次都问过:每次质询只计一遍
|
||||
- 封存之后追加的质询(post_seal)不计进运行用量,继续运行、重新封存之后也不计
|
||||
- 没有质询的运行,用量和以前逐字相同
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.core.events import EventType
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import Run, RunEvent
|
||||
from app.engine import compiler
|
||||
from app.engine.context import NodeError
|
||||
from app.engine.runner import run_manager
|
||||
from app.engine.schema import NodeType
|
||||
|
||||
A = {"input_tokens": 1200, "output_tokens": 300, "cost_usd": 0.0123, "calls": 2}
|
||||
B = {"input_tokens": 800, "output_tokens": 100, "cost_usd": 0.004, "calls": 1, "unpriced_calls": 1}
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
async def engine_up():
|
||||
await run_manager.setup()
|
||||
yield
|
||||
await run_manager.shutdown()
|
||||
|
||||
|
||||
def node(nid, ntype, **config):
|
||||
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}}
|
||||
|
||||
|
||||
def chain(*nodes):
|
||||
return {"nodes": list(nodes), "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]}
|
||||
|
||||
|
||||
def gated(*middle, gates: int = 1) -> dict:
|
||||
nodes = [node("start", "input"), *middle]
|
||||
nodes += [node(f"gate{i}", "human", mode="approve", title=f"第 {i + 1} 道审批") for i in range(gates)]
|
||||
nodes.append(node("out", "output", fields=[{"name": "r", "value": "完成"}]))
|
||||
return chain(*nodes)
|
||||
|
||||
|
||||
def ask(question: str, usage: dict | None) -> tuple:
|
||||
data = {"question": question, "answer": "按订单表核对一致"}
|
||||
if usage is not None:
|
||||
data["usage"] = usage
|
||||
return (EventType.APPROVAL_INQUIRY, None, data)
|
||||
|
||||
|
||||
async def _row(run_id: str) -> Run:
|
||||
async with SessionLocal() as session:
|
||||
return await session.get(Run, run_id)
|
||||
|
||||
|
||||
async def _until(run_id: str, statuses: tuple[str, ...]) -> Run:
|
||||
for _ in range(500):
|
||||
row = await _row(run_id)
|
||||
if row.status in statuses and (row.status == "interrupted" or row.manifest_seq is not None):
|
||||
await run_manager.wait_idle(run_id)
|
||||
return await _row(run_id)
|
||||
await asyncio.sleep(0.03)
|
||||
raise AssertionError(f"运行 {run_id} 停在 {row.status}")
|
||||
|
||||
|
||||
async def _ask_while_paused(run_id: str, *items: tuple) -> None:
|
||||
from app.engine.run_append import append_events
|
||||
|
||||
assert await append_events(run_id, list(items), require="interrupted"), "质询没写进运行"
|
||||
|
||||
|
||||
def _tokens(usage: dict) -> tuple:
|
||||
return (usage.get("input_tokens"), usage.get("output_tokens"), usage.get("total_tokens"),
|
||||
usage.get("calls"), usage.get("unpriced_calls"))
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 按事件加总
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_usage_of_events_keeps_inquiries_apart():
|
||||
"""llm.end 照旧逐次加总;质询的另记在 inquiry 下(_finalize 比完两本账再加,见下面几条)。"""
|
||||
from app.engine.runner import _usage_of_events
|
||||
|
||||
async with SessionLocal() as session:
|
||||
run = Run(workflow_name="用量加总", status="succeeded", graph={}, input={})
|
||||
session.add(run)
|
||||
await session.flush()
|
||||
rows = [
|
||||
("llm.end", {"model": "claude-sonnet-4-5", "input_tokens": 100, "output_tokens": 20, "cost_usd": 0.001,
|
||||
"calls": 1}),
|
||||
("llm.end", {"model": "claude-sonnet-4-5", "input_tokens": 50, "output_tokens": 5, "cost_usd": 0.0005}),
|
||||
("approval.inquiry", {"question": "甲", "usage": A}),
|
||||
("approval.inquiry", {"question": "乙", "usage": B}),
|
||||
("approval.inquiry", {"question": "没问模型就答了"}),
|
||||
("approval.inquiry", {"question": "写坏的用量", "usage": "很多"}),
|
||||
# 别的事件里的金额不算:按需裁判的判定另有自己的账
|
||||
("evidence.judged", {"cost_usd": 9.9, "calls": 9}),
|
||||
]
|
||||
for seq, (kind, data) in enumerate(rows, start=1):
|
||||
session.add(RunEvent(run_id=run.id, seq=seq, type=kind, node_id=None, ts=float(seq), data=data))
|
||||
await session.commit()
|
||||
got = await _usage_of_events(session, run.id)
|
||||
assert {k: got[k] for k in ("input_tokens", "output_tokens", "calls")} == {
|
||||
"input_tokens": 150, "output_tokens": 25, "calls": 2}
|
||||
assert got["cost_usd"] == pytest.approx(0.0015)
|
||||
assert got["inquiry"] == {"input_tokens": 2000, "output_tokens": 400, "cost_usd": pytest.approx(0.0163),
|
||||
"calls": 3, "unpriced_calls": 1}
|
||||
|
||||
|
||||
async def test_without_inquiries_there_is_no_inquiry_key():
|
||||
from app.engine.runner import _usage_of_events
|
||||
|
||||
async with SessionLocal() as session:
|
||||
run = Run(workflow_name="没有质询", status="succeeded", graph={}, input={})
|
||||
session.add(run)
|
||||
await session.flush()
|
||||
session.add(RunEvent(run_id=run.id, seq=1, type="llm.end", node_id="ask", ts=1.0,
|
||||
data={"model": "claude-sonnet-4-5", "input_tokens": 10, "output_tokens": 2,
|
||||
"cost_usd": 0.0001, "calls": 1}))
|
||||
await session.commit()
|
||||
got = await _usage_of_events(session, run.id)
|
||||
assert set(got) == {"input_tokens", "output_tokens", "cost_usd", "calls"}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 运行用量
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_a_succeeded_run_counts_the_inquiry():
|
||||
run = await run_manager.start(graph=gated(), input_payload={})
|
||||
await _until(run.id, ("interrupted",))
|
||||
await _ask_while_paused(run.id, ask("销售额为什么涨了", A))
|
||||
await run_manager.resume(run.id, {"approved": True})
|
||||
row = await _until(run.id, ("succeeded",))
|
||||
# 图里没有模型调用,用量全是这次质询的
|
||||
assert _tokens(row.usage) == (1200, 300, 1500, 2, None)
|
||||
assert row.usage["cost_usd"] == pytest.approx(0.0123)
|
||||
assert row.usage["inquiry"] == A
|
||||
assert row.usage["wall_ms"] >= row.usage["active_ms"]
|
||||
|
||||
|
||||
async def test_the_inquiry_is_added_on_top_of_the_model_calls():
|
||||
"""图里有模型调用:成功的运行取状态里的用量,再加上质询的。"""
|
||||
graph = gated(node("ask", "llm", prompt="上周到店多少人"))
|
||||
run = await run_manager.start(graph=graph, input_payload={})
|
||||
await _until(run.id, ("interrupted",))
|
||||
await _ask_while_paused(run.id, ask("到店人数怎么算的", B))
|
||||
await run_manager.resume(run.id, {"approved": True})
|
||||
row = await _until(run.id, ("succeeded",))
|
||||
async with SessionLocal() as session:
|
||||
ends = [e.data for e in (await session.execute(select(RunEvent).where(
|
||||
RunEvent.run_id == run.id, RunEvent.type == "llm.end"))).scalars()]
|
||||
assert ends, "图里的模型调用没有记 llm.end"
|
||||
model_in = sum(e["input_tokens"] for e in ends)
|
||||
model_out = sum(e["output_tokens"] for e in ends)
|
||||
unpriced = sum(1 for e in ends if e.get("unpriced"))
|
||||
assert row.usage["input_tokens"] == model_in + B["input_tokens"]
|
||||
assert row.usage["output_tokens"] == model_out + B["output_tokens"]
|
||||
assert row.usage["total_tokens"] == model_in + model_out + B["input_tokens"] + B["output_tokens"]
|
||||
assert row.usage["calls"] == len(ends) + B["calls"]
|
||||
assert row.usage["unpriced_calls"] == unpriced + B["unpriced_calls"]
|
||||
assert row.usage["cost_usd"] == pytest.approx(sum(e["cost_usd"] for e in ends) + B["cost_usd"])
|
||||
|
||||
|
||||
async def test_a_failed_run_counts_the_inquiry_too(monkeypatch):
|
||||
original = compiler.RUNNERS[NodeType.TRANSFORM]
|
||||
|
||||
async def boom(state, ctx):
|
||||
if ctx.node.id == "boom":
|
||||
raise NodeError(ctx.node.id, "数据源暂时不可用")
|
||||
return await original(state, ctx)
|
||||
|
||||
monkeypatch.setitem(compiler.RUNNERS, NodeType.TRANSFORM, boom)
|
||||
graph = chain(node("start", "input"), node("gate", "human", mode="approve", title="放行吗"),
|
||||
node("boom", "transform", mode="template", template="x"),
|
||||
node("out", "output", fields=[{"name": "r", "value": "x"}]))
|
||||
run = await run_manager.start(graph=graph, input_payload={})
|
||||
await _until(run.id, ("interrupted",))
|
||||
await _ask_while_paused(run.id, ask("口径对吗", A))
|
||||
await run_manager.resume(run.id, {"approved": True})
|
||||
row = await _until(run.id, ("failed",))
|
||||
assert _tokens(row.usage) == (1200, 300, 1500, 2, None)
|
||||
assert row.usage["inquiry"] == A
|
||||
|
||||
|
||||
async def test_each_inquiry_counts_once_across_two_pauses():
|
||||
run = await run_manager.start(graph=gated(gates=2), input_payload={})
|
||||
await _until(run.id, ("interrupted",))
|
||||
await _ask_while_paused(run.id, ask("第一问", A))
|
||||
await run_manager.resume(run.id, {"approved": True})
|
||||
await asyncio.sleep(0.05)
|
||||
paused = await _until(run.id, ("interrupted",))
|
||||
# 第二次停下时记的用量已经有第一问
|
||||
assert paused.usage["input_tokens"] == A["input_tokens"] and paused.usage["inquiry"] == A
|
||||
await _ask_while_paused(run.id, ask("第二问", B))
|
||||
await run_manager.resume(run.id, {"approved": True})
|
||||
row = await _until(run.id, ("succeeded",))
|
||||
assert _tokens(row.usage) == (2000, 400, 2400, 3, 1)
|
||||
assert row.usage["cost_usd"] == pytest.approx(0.0163)
|
||||
assert row.usage["inquiry"] == {"input_tokens": 2000, "output_tokens": 400, "cost_usd": pytest.approx(0.0163),
|
||||
"calls": 3, "unpriced_calls": 1}
|
||||
|
||||
|
||||
async def test_post_seal_inquiries_stay_out_of_the_run_usage(monkeypatch):
|
||||
"""运行用量是这次运行自己花的,结束时就定下了、写进了封存范围里的终态事件。封存之后追加的质询是事后的批注,
|
||||
和按需裁判的判定一样另有自己的账,不计进来。以前平时不计,继续运行、重新算用量时又被计了进去:同一条质询
|
||||
计不计,全看之后有没有继续运行。"""
|
||||
from app.engine.run_append import append_events
|
||||
|
||||
original = compiler.RUNNERS[NodeType.TRANSFORM]
|
||||
state = {"fail": True}
|
||||
|
||||
async def flaky(st, ctx):
|
||||
if ctx.node.id == "boom" and state["fail"]:
|
||||
raise NodeError(ctx.node.id, "数据源暂时不可用")
|
||||
return await original(st, ctx)
|
||||
|
||||
monkeypatch.setitem(compiler.RUNNERS, NodeType.TRANSFORM, flaky)
|
||||
graph = chain(node("start", "input"), node("boom", "transform", mode="template", template="x"),
|
||||
node("out", "output", fields=[{"name": "r", "value": "x"}]))
|
||||
run = await run_manager.start(graph=graph, input_payload={})
|
||||
failed = await _until(run.id, ("failed",))
|
||||
written = await append_events(run.id, [ask("这次为什么失败", A)], require="sealed")
|
||||
assert written and written[0].data["post_seal"] is True
|
||||
assert set(failed.usage) == {"duration_ms", "wall_ms", "active_ms", "wait_ms"}
|
||||
|
||||
state["fail"] = False
|
||||
await run_manager.continue_failed(run.id)
|
||||
row = await _until(run.id, ("succeeded",))
|
||||
assert row.manifest_seq > written[0].seq, "继续运行之后重新封存,这条质询落进了新的封存范围"
|
||||
assert set(row.usage) == {"duration_ms", "wall_ms", "active_ms", "wait_ms"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("asked_before", [False, True], ids=["first-pause", "second-pause"])
|
||||
async def test_abandoning_a_paused_run_counts_what_was_asked(asked_before):
|
||||
"""停在审批时放弃:停下时记的用量里还没有这次停下期间的质询,放弃时补上;上一次停下时已经计过的不再计。"""
|
||||
run = await run_manager.start(graph=gated(gates=2 if asked_before else 1), input_payload={})
|
||||
await _until(run.id, ("interrupted",))
|
||||
if asked_before:
|
||||
await _ask_while_paused(run.id, ask("第一问", A))
|
||||
await run_manager.resume(run.id, {"approved": True})
|
||||
await asyncio.sleep(0.05)
|
||||
await _until(run.id, ("interrupted",))
|
||||
await _ask_while_paused(run.id, ask("最后一问", B))
|
||||
assert await run_manager.cancel(run.id) == "cancelled"
|
||||
row = await _until(run.id, ("cancelled",))
|
||||
if asked_before:
|
||||
assert _tokens(row.usage) == (2000, 400, 2400, 3, 1)
|
||||
assert row.usage["cost_usd"] == pytest.approx(0.0163)
|
||||
else:
|
||||
assert _tokens(row.usage) == (800, 100, 900, 1, 1) and row.usage["inquiry"] == B
|
||||
assert row.usage["wall_ms"] >= row.usage["active_ms"]
|
||||
|
||||
|
||||
async def test_runs_without_inquiries_keep_their_usage():
|
||||
run = await run_manager.start(graph=gated(), input_payload={})
|
||||
await _until(run.id, ("interrupted",))
|
||||
await run_manager.resume(run.id, {"approved": True})
|
||||
row = await _until(run.id, ("succeeded",))
|
||||
assert set(row.usage) == {"duration_ms", "wall_ms", "active_ms", "wait_ms"}
|
||||
|
||||
other = await run_manager.start(graph=gated(), input_payload={})
|
||||
await _until(other.id, ("interrupted",))
|
||||
assert await run_manager.cancel(other.id) == "cancelled"
|
||||
row = await _until(other.id, ("cancelled",))
|
||||
assert set(row.usage) == {"duration_ms", "wall_ms", "active_ms", "wait_ms"}
|
||||
@@ -0,0 +1,762 @@
|
||||
"""往停下的运行里追加事件(engine/run_append.py 的 append_events)。
|
||||
|
||||
- 停在审批时追加:事件接在最后一条后面、落进以后的封存范围。批准、运行结束封存之后 verify 照旧一致,
|
||||
改动追加的事件 verify 就对不上——它们确实在封存范围里
|
||||
- 追加途中运行被恢复(条件更新落空):什么都不写,返回 None,last_seq 不动
|
||||
- 质询指明问的是哪张审批卡(approval_id)时,那张卡仍待审批才写;返回的事件和落库的 id、序号一样
|
||||
- 先落库再广播,没写入的一条都不广播
|
||||
- 已封存的运行追加:带 post_seal、落在 manifest_seq 之后,verify 照旧一致;封存不是这次结束之后封的、
|
||||
封存和调用方开始时看到的不同、没有终态事件的老封存,都不追加
|
||||
- 连续追加、并发追加,序号严格递增,整条事件流没有重复的序号
|
||||
- last_seq 落后于库里实际的最后一条(按需裁判的追加和封存后追加交错提交时会这样):进程重启后恢复、
|
||||
继续运行、放弃、按需裁判都接在实际的最后一条后面,不撞号;runner 记事件不把 last_seq 写小
|
||||
- 恢复后的重放不受影响:前面的节点不重复执行,Agent 里批过的工具不重复执行(含升级前写下的断点)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import sqlite3
|
||||
from collections import Counter
|
||||
|
||||
import pytest
|
||||
from langchain_core.messages import AIMessage, ToolMessage
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.core.bus import bus
|
||||
from app.core.config import settings
|
||||
from app.core.events import EventType
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import Approval, Run, RunEvent
|
||||
from app.engine import compiler
|
||||
from app.engine.context import NodeError
|
||||
from app.engine.runner import RunManager, run_manager, verify_manifest
|
||||
from app.engine.schema import NodeType
|
||||
from app.tools.registry import register
|
||||
|
||||
#: Agent 场景里工具真正执行的次数。注册表是进程级的,工具只注册一次,计数按测试清零
|
||||
CALLS: list[str] = []
|
||||
|
||||
|
||||
class _TallyArgs(BaseModel):
|
||||
label: str
|
||||
|
||||
|
||||
@register("inquiry_tally", "测试用:每执行一次记一笔", "test", _TallyArgs, dangerous=True, needs_context=False)
|
||||
async def _tally(label: str) -> str:
|
||||
CALLS.append(label)
|
||||
return json.dumps({"label": label, "n": len(CALLS)}, ensure_ascii=False)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
async def engine_up():
|
||||
CALLS.clear()
|
||||
await run_manager.setup()
|
||||
yield
|
||||
await run_manager.shutdown()
|
||||
|
||||
|
||||
def node(nid, ntype, **config):
|
||||
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}}
|
||||
|
||||
|
||||
def chain(*nodes):
|
||||
return {"nodes": list(nodes), "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]}
|
||||
|
||||
|
||||
GATED = chain(node("start", "input"),
|
||||
node("work", "transform", mode="template", template="本周销售额 4321.5 元"),
|
||||
node("gate", "human", mode="approve", title="这份周报可以出具吗"),
|
||||
node("out", "output", fields=[{"name": "r", "value": "{{ nodes.work }}"}]))
|
||||
SIMPLE = chain(node("start", "input"), node("out", "output", fields=[{"name": "r", "value": "完成"}]))
|
||||
TWO_GATES = chain(node("start", "input"),
|
||||
node("g1", "human", mode="approve", title="第一道:口径对吗"),
|
||||
node("g2", "human", mode="approve", title="第二道:可以出具吗"),
|
||||
node("out", "output", fields=[{"name": "r", "value": "完成"}]))
|
||||
#: 「boom」节点按 flaky 的开关失败:先让运行失败、封存,再关掉开关继续运行
|
||||
FLAKY = chain(node("start", "input"), node("boom", "transform", mode="template", template="门店 17 家"),
|
||||
node("out", "output", fields=[{"name": "r", "value": "{{ nodes.boom }}"}]))
|
||||
|
||||
|
||||
def inquiry(question: str) -> tuple:
|
||||
return (EventType.APPROVAL_INQUIRY, None, {"question": question, "answer": f"关于「{question}」:按订单表核对一致"})
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def executed(monkeypatch) -> Counter:
|
||||
"""每个「数据整形」节点真正执行了几次。"""
|
||||
seen: Counter = Counter()
|
||||
original = compiler.RUNNERS[NodeType.TRANSFORM]
|
||||
|
||||
async def counting(state, ctx):
|
||||
seen[ctx.node.id] += 1
|
||||
return await original(state, ctx)
|
||||
|
||||
monkeypatch.setitem(compiler.RUNNERS, NodeType.TRANSFORM, counting)
|
||||
return seen
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def flaky(monkeypatch) -> dict:
|
||||
"""FLAKY 里的「boom」在 fail 为真时失败。"""
|
||||
state = {"fail": True}
|
||||
original = compiler.RUNNERS[NodeType.TRANSFORM]
|
||||
|
||||
async def boom(st, ctx):
|
||||
if ctx.node.id == "boom" and state["fail"]:
|
||||
raise NodeError(ctx.node.id, "数据源暂时不可用")
|
||||
return await original(st, ctx)
|
||||
|
||||
monkeypatch.setitem(compiler.RUNNERS, NodeType.TRANSFORM, boom)
|
||||
return state
|
||||
|
||||
|
||||
async def _row(run_id: str) -> Run:
|
||||
async with SessionLocal() as session:
|
||||
return await session.get(Run, run_id)
|
||||
|
||||
|
||||
async def _until(run_id: str, statuses: tuple[str, ...], sealed: bool = False) -> Run:
|
||||
for _ in range(500):
|
||||
row = await _row(run_id)
|
||||
if row.status in statuses and (not sealed or row.manifest_seq is not None):
|
||||
await run_manager.wait_idle(run_id)
|
||||
return await _row(run_id)
|
||||
await asyncio.sleep(0.03)
|
||||
raise AssertionError(f"运行 {run_id} 停在 {row.status}")
|
||||
|
||||
|
||||
async def _paused(graph=GATED) -> Run:
|
||||
run = await run_manager.start(graph=graph, input_payload={})
|
||||
return await _until(run.id, ("interrupted",))
|
||||
|
||||
|
||||
async def _finished(graph=SIMPLE) -> Run:
|
||||
run = await run_manager.start(graph=graph, input_payload={})
|
||||
return await _until(run.id, ("succeeded", "failed"), sealed=True)
|
||||
|
||||
|
||||
async def _events(run_id: str, etype: str | None = None) -> list[RunEvent]:
|
||||
async with SessionLocal() as session:
|
||||
query = select(RunEvent).where(RunEvent.run_id == run_id)
|
||||
if etype:
|
||||
query = query.where(RunEvent.type == etype)
|
||||
return list((await session.execute(query.order_by(RunEvent.seq))).scalars())
|
||||
|
||||
|
||||
def _strictly_increasing(seqs: list[int]) -> bool:
|
||||
return all(a < b for a, b in zip(seqs, seqs[1:]))
|
||||
|
||||
|
||||
async def _no_repeated_seq(run_id: str) -> None:
|
||||
seqs = [e.seq for e in await _events(run_id)]
|
||||
assert len(seqs) == len(set(seqs)), f"事件流里有重复的序号:{seqs}"
|
||||
|
||||
|
||||
async def _pending(run_id: str) -> list[Approval]:
|
||||
async with SessionLocal() as session:
|
||||
return list((await session.execute(
|
||||
select(Approval).where(Approval.run_id == run_id, Approval.status == "pending"))).scalars())
|
||||
|
||||
|
||||
def _sql(statement: str, *params) -> None:
|
||||
"""另开一条连接改库:模拟别处(恢复、继续运行、另一个写入者)抢在中间提交。"""
|
||||
db = sqlite3.connect(settings.db_path, timeout=5)
|
||||
try:
|
||||
db.execute(statement, params)
|
||||
db.commit()
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 停在审批时追加
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_appended_while_waiting_are_sealed_with_the_run(executed):
|
||||
from app.engine.run_append import append_events
|
||||
|
||||
paused = await _paused()
|
||||
subscription = bus.attach(paused.id)
|
||||
try:
|
||||
written = await append_events(paused.id, [inquiry("销售额为什么涨了"), inquiry("退款算进去了吗")],
|
||||
require="interrupted")
|
||||
assert written is not None and len(written) == 2
|
||||
assert written[0].seq > paused.last_seq and _strictly_increasing([e.seq for e in written])
|
||||
# 提交之后才广播:在线的客户端收到的就是落了库的这两条
|
||||
stream = subscription.stream()
|
||||
live = [await asyncio.wait_for(stream.__anext__(), 2) for _ in range(2)]
|
||||
assert [e.seq for e in live] == [e.seq for e in written]
|
||||
finally:
|
||||
subscription.close()
|
||||
row = await _row(paused.id)
|
||||
assert row.status == "interrupted" and row.last_seq == written[-1].seq
|
||||
stored = await _events(paused.id, "approval.inquiry")
|
||||
assert [(e.seq, e.node_id, e.data["question"]) for e in stored] == [
|
||||
(written[0].seq, None, "销售额为什么涨了"), (written[1].seq, None, "退款算进去了吗")]
|
||||
assert all("post_seal" not in e.data for e in stored), "进封存范围的事件不该标成封存之后"
|
||||
|
||||
await run_manager.resume(paused.id, {"approved": True})
|
||||
final = await _until(paused.id, ("succeeded", "failed"), sealed=True)
|
||||
assert final.status == "succeeded", final.error
|
||||
assert final.manifest_seq > written[-1].seq
|
||||
check = await verify_manifest(paused.id)
|
||||
assert check["ok"] is True and check["sealed_at"] == final.manifest_seq
|
||||
# 恢复之后前面的节点没有再执行一遍
|
||||
assert executed["work"] == 1
|
||||
assert len([e for e in await _events(paused.id, "node.started") if e.node_id == "work"]) == 1
|
||||
await _no_repeated_seq(paused.id)
|
||||
|
||||
# 追加的事件确实在封存范围里:改它一个字,封存核对就对不上
|
||||
async with SessionLocal() as session:
|
||||
event = await session.get(RunEvent, stored[0].id)
|
||||
event.data = {**event.data, "answer": "改过的回答"}
|
||||
await session.commit()
|
||||
assert (await verify_manifest(paused.id))["ok"] is False
|
||||
|
||||
|
||||
async def test_a_resume_that_wins_the_row_leaves_nothing_behind(monkeypatch):
|
||||
"""读完运行、写入之前,恢复先把运行占成了 running:条件更新落空,一条都不写、last_seq 不动。"""
|
||||
from app.engine.run_append import append_events
|
||||
|
||||
paused = await _paused()
|
||||
set_seq = bus.set_seq
|
||||
|
||||
def resumed_in_between(run_id, value):
|
||||
if run_id == paused.id:
|
||||
db = sqlite3.connect(settings.db_path, timeout=5)
|
||||
try:
|
||||
db.execute("UPDATE runs SET status = 'running' WHERE id = ?", (paused.id,))
|
||||
db.commit()
|
||||
finally:
|
||||
db.close()
|
||||
set_seq(run_id, value)
|
||||
|
||||
monkeypatch.setattr(bus, "set_seq", resumed_in_between)
|
||||
assert await append_events(paused.id, [inquiry("毛利率怎么算的")], require="interrupted") is None
|
||||
row = await _row(paused.id)
|
||||
assert row.status == "running" and row.last_seq == paused.last_seq
|
||||
assert await _events(paused.id, "approval.inquiry") == []
|
||||
|
||||
|
||||
async def test_events_written_elsewhere_in_between_win(monkeypatch):
|
||||
"""读完运行、写入之前,别处(另一个进程,或者没走这把锁的追加)已经接着写了一条:状态没变,last_seq 变了。
|
||||
条件更新落空、不写——否则两边各拿着同一个序号,事件流里撞号。"""
|
||||
from app.engine.run_append import append_events
|
||||
|
||||
paused = await _paused()
|
||||
taken = paused.last_seq + 1
|
||||
set_seq = bus.set_seq
|
||||
|
||||
def written_in_between(run_id, value):
|
||||
if run_id == paused.id:
|
||||
db = sqlite3.connect(settings.db_path, timeout=5)
|
||||
try:
|
||||
db.execute("INSERT INTO run_events (id, run_id, seq, type, node_id, ts, data) "
|
||||
"VALUES ('elsewhere-1', ?, ?, 'log', NULL, 0, '{}')", (paused.id, taken))
|
||||
db.execute("UPDATE runs SET last_seq = ? WHERE id = ?", (taken, paused.id))
|
||||
db.commit()
|
||||
finally:
|
||||
db.close()
|
||||
set_seq(run_id, value)
|
||||
|
||||
monkeypatch.setattr(bus, "set_seq", written_in_between)
|
||||
bus._seq.pop(paused.id, None) # 另一个进程的序号本进程不知道
|
||||
assert await append_events(paused.id, [inquiry("同时写进来的")], require="interrupted") is None
|
||||
assert (await _row(paused.id)).last_seq == taken
|
||||
assert await _events(paused.id, "approval.inquiry") == []
|
||||
await _no_repeated_seq(paused.id)
|
||||
|
||||
|
||||
async def test_a_last_seq_left_behind_does_not_cause_a_clash():
|
||||
"""last_seq 比库里实际的最后一条小(别处的追加交错着把它写小了):照实际的最后一条接号,顺手把 last_seq 写对。"""
|
||||
from app.engine.run_append import append_events
|
||||
|
||||
paused = await _paused()
|
||||
async with SessionLocal() as session:
|
||||
run = await session.get(Run, paused.id)
|
||||
run.last_seq = paused.last_seq - 3
|
||||
await session.commit()
|
||||
bus._seq.pop(paused.id, None)
|
||||
written = await append_events(paused.id, [inquiry("接在哪里")], require="interrupted")
|
||||
assert written[0].seq == paused.last_seq + 1
|
||||
assert (await _row(paused.id)).last_seq == written[0].seq
|
||||
await _no_repeated_seq(paused.id)
|
||||
|
||||
|
||||
async def test_a_run_that_already_moved_on_gets_nothing(executed):
|
||||
from app.engine.run_append import append_events
|
||||
|
||||
paused = await _paused()
|
||||
await run_manager.resume(paused.id, {"approved": True})
|
||||
final = await _until(paused.id, ("succeeded",), sealed=True)
|
||||
assert await append_events(paused.id, [inquiry("晚到的问题")], require="interrupted") is None
|
||||
assert (await _row(paused.id)).last_seq == final.last_seq
|
||||
assert await _events(paused.id, "approval.inquiry") == []
|
||||
|
||||
|
||||
async def test_an_inquiry_about_an_approval_needs_that_approval_still_pending(monkeypatch):
|
||||
"""质询问的是某一张审批卡:开始问模型时那张卡还在等,问完时运行可能已经批过它、停到了下一张上。只看
|
||||
「运行停在审批」的话,关于第一张的问答会写进第二次停下里。传 approval_id 时那张卡仍待审批才写,
|
||||
和其余条件在同一次条件更新里核对。"""
|
||||
from app.engine.run_append import append_events
|
||||
|
||||
paused = await _paused(TWO_GATES)
|
||||
[first] = await _pending(paused.id)
|
||||
await run_manager.resume(paused.id, {"approved": True}, approval_id=first.id)
|
||||
await asyncio.sleep(0.05)
|
||||
later = await _until(paused.id, ("interrupted",))
|
||||
[second] = await _pending(paused.id)
|
||||
assert (first.node_id, second.node_id) == ("g1", "g2")
|
||||
|
||||
assert await append_events(paused.id, [inquiry("第一道的口径")], require="interrupted",
|
||||
approval_id=first.id) is None
|
||||
assert await append_events(paused.id, [inquiry("不存在的卡")], require="interrupted",
|
||||
approval_id="no-such-approval") is None
|
||||
assert (await _row(paused.id)).last_seq == later.last_seq
|
||||
assert await _events(paused.id, "approval.inquiry") == []
|
||||
|
||||
written = await append_events(paused.id, [inquiry("出具前再确认一次")], require="interrupted",
|
||||
approval_id=second.id)
|
||||
assert written is not None
|
||||
# 返回的就是落了库的那几条:调用方按 id 或序号回头找,两边对得上
|
||||
stored = await _events(paused.id, "approval.inquiry")
|
||||
assert [(e.id, e.seq) for e in stored] == [(e.id, e.seq) for e in written]
|
||||
|
||||
# 读完运行、写入之前那张卡被人批了(并行的审批还没凑齐,运行仍停着):不写
|
||||
set_seq = bus.set_seq
|
||||
|
||||
def answered_in_between(run_id, value):
|
||||
if run_id == paused.id:
|
||||
_sql("UPDATE approvals SET status = 'answered' WHERE id = ?", second.id)
|
||||
set_seq(run_id, value)
|
||||
|
||||
monkeypatch.setattr(bus, "set_seq", answered_in_between)
|
||||
assert await append_events(paused.id, [inquiry("批完才问到的")], require="interrupted",
|
||||
approval_id=second.id) is None
|
||||
assert (await _row(paused.id)).last_seq == written[-1].seq
|
||||
assert len(await _events(paused.id, "approval.inquiry")) == 1
|
||||
with pytest.raises(ValueError):
|
||||
await append_events(paused.id, [inquiry("封存后的卡")], require="sealed", approval_id=second.id)
|
||||
|
||||
|
||||
async def test_published_only_once_stored_and_never_when_not_written(monkeypatch):
|
||||
"""先落库再广播:在线的客户端收到的每一条库里都已经有了(订阅者靠「先读历史、再接实时」拼完整,见
|
||||
RunManager._emit)。没写进运行记录的一条都不广播,否则界面上会多出一条运行记录里没有的问答。"""
|
||||
from app.engine.run_append import append_events
|
||||
|
||||
published: list[tuple[int, int]] = []
|
||||
publish = bus.publish
|
||||
|
||||
async def stored_when_published(event):
|
||||
if event.type == EventType.APPROVAL_INQUIRY:
|
||||
db = sqlite3.connect(settings.db_path, timeout=5)
|
||||
try:
|
||||
(count,) = db.execute("SELECT count(*) FROM run_events WHERE run_id = ? AND seq = ?",
|
||||
(event.run_id, event.seq)).fetchone()
|
||||
finally:
|
||||
db.close()
|
||||
published.append((event.seq, count))
|
||||
await publish(event)
|
||||
|
||||
monkeypatch.setattr(bus, "publish", stored_when_published)
|
||||
paused = await _paused()
|
||||
written = await append_events(paused.id, [inquiry("客单价怎么算的"), inquiry("含不含退款")],
|
||||
require="interrupted")
|
||||
assert published == [(e.seq, 1) for e in written]
|
||||
|
||||
published.clear()
|
||||
set_seq = bus.set_seq
|
||||
|
||||
def resumed_in_between(run_id, value):
|
||||
if run_id == paused.id:
|
||||
_sql("UPDATE runs SET status = 'running' WHERE id = ?", paused.id)
|
||||
set_seq(run_id, value)
|
||||
|
||||
monkeypatch.setattr(bus, "set_seq", resumed_in_between)
|
||||
assert await append_events(paused.id, [inquiry("恢复之后才到的")], require="interrupted") is None
|
||||
assert published == []
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 封存之后追加
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_post_seal_appends_stay_outside_the_seal():
|
||||
from app.engine.run_append import append_events
|
||||
|
||||
final = await _finished()
|
||||
assert final.status == "succeeded", final.error
|
||||
before = await verify_manifest(final.id)
|
||||
written = await append_events(final.id, [inquiry("这个数是含税的吗"), inquiry("和上周口径一样吗")],
|
||||
require="sealed")
|
||||
assert written is not None and all(e.data["post_seal"] is True for e in written)
|
||||
assert written[0].seq > final.manifest_seq and _strictly_increasing([e.seq for e in written])
|
||||
row = await _row(final.id)
|
||||
assert (row.status, row.manifest_seq, row.manifest_hash) == ("succeeded", final.manifest_seq, final.manifest_hash)
|
||||
assert row.last_seq == written[-1].seq
|
||||
after = await verify_manifest(final.id)
|
||||
assert after["ok"] is True and (after["sealed_at"], after["events"]) == (before["sealed_at"], before["events"])
|
||||
assert [e.data["post_seal"] for e in await _events(final.id, "approval.inquiry")] == [True, True]
|
||||
await _no_repeated_seq(final.id)
|
||||
|
||||
|
||||
async def test_a_seal_pinned_by_the_caller_must_still_be_the_seal():
|
||||
from app.engine.run_append import append_events
|
||||
|
||||
final = await _finished()
|
||||
seal = (final.manifest_seq, final.manifest_hash)
|
||||
assert await append_events(final.id, [inquiry("甲")], require="sealed", seal=(seal[0] - 1, seal[1])) is None
|
||||
assert await append_events(final.id, [inquiry("乙")], require="sealed", seal=(seal[0], "0" * 64)) is None
|
||||
assert await _events(final.id, "approval.inquiry") == []
|
||||
written = await append_events(final.id, [inquiry("丙")], require="sealed", seal=seal)
|
||||
assert written is not None and written[0].data["question"] == "丙"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("change", [
|
||||
"UPDATE runs SET status = 'running' WHERE id = ?", # 继续运行占住了它
|
||||
"UPDATE runs SET manifest_hash = 'resealed' WHERE id = ?", # 继续运行之后重新封存了
|
||||
"UPDATE runs SET manifest_seq = manifest_seq + 1 WHERE id = ?",
|
||||
], ids=["continued", "resealed-hash", "resealed-seq"])
|
||||
async def test_a_sealed_run_that_changes_in_between_gets_nothing(monkeypatch, change):
|
||||
"""封存之后追加也是核对和写入同一行上的条件更新:读完运行、写入之前它被继续运行占住、或者重新封存了,就不写。"""
|
||||
from app.engine.run_append import append_events
|
||||
|
||||
final = await _finished()
|
||||
set_seq = bus.set_seq
|
||||
|
||||
def changed_in_between(run_id, value):
|
||||
if run_id == final.id:
|
||||
db = sqlite3.connect(settings.db_path, timeout=5)
|
||||
try:
|
||||
db.execute(change, (final.id,))
|
||||
db.commit()
|
||||
finally:
|
||||
db.close()
|
||||
set_seq(run_id, value)
|
||||
|
||||
monkeypatch.setattr(bus, "set_seq", changed_in_between)
|
||||
assert await append_events(final.id, [inquiry("这期为什么少了")], require="sealed") is None
|
||||
assert (await _row(final.id)).last_seq == final.last_seq
|
||||
assert await _events(final.id, "approval.inquiry") == []
|
||||
|
||||
|
||||
async def test_a_seal_left_from_an_earlier_ending_is_not_the_seal(monkeypatch):
|
||||
"""继续运行后又失败:终态和 run.failed 先提交、封存后算。这中间运行记录上还是上一次结束时的封存,
|
||||
这时追加的事件会被接下来的封存收进去,post_seal 就成了假话——不追加。"""
|
||||
from app.engine.run_append import append_events
|
||||
|
||||
original = compiler.RUNNERS[NodeType.TRANSFORM]
|
||||
|
||||
async def boom(state, ctx):
|
||||
if ctx.node.id == "boom":
|
||||
raise NodeError(ctx.node.id, "数据源暂时不可用")
|
||||
return await original(state, ctx)
|
||||
|
||||
monkeypatch.setitem(compiler.RUNNERS, NodeType.TRANSFORM, boom)
|
||||
first = await _finished(chain(node("start", "input"), node("boom", "transform", mode="template", template="x"),
|
||||
node("out", "output", fields=[{"name": "r", "value": "x"}])))
|
||||
assert first.status == "failed"
|
||||
|
||||
entered, release = asyncio.Event(), asyncio.Event()
|
||||
real_seal = RunManager._seal
|
||||
|
||||
async def held_seal(self, run_id):
|
||||
if run_id == first.id and not entered.is_set():
|
||||
entered.set()
|
||||
await release.wait()
|
||||
return await real_seal(self, run_id)
|
||||
|
||||
monkeypatch.setattr(RunManager, "_seal", held_seal)
|
||||
await run_manager.continue_failed(first.id)
|
||||
await asyncio.wait_for(entered.wait(), 10)
|
||||
try:
|
||||
row = await _row(first.id)
|
||||
failures = [e.seq for e in await _events(first.id, "run.failed")]
|
||||
assert row.status == "failed" and row.manifest_seq == first.manifest_seq < failures[-1]
|
||||
assert await append_events(first.id, [inquiry("为什么又失败了")], require="sealed") is None
|
||||
finally:
|
||||
release.set()
|
||||
await run_manager.wait_idle(first.id)
|
||||
resealed = await _row(first.id)
|
||||
assert resealed.manifest_seq >= failures[-1]
|
||||
written = await append_events(first.id, [inquiry("为什么又失败了")], require="sealed")
|
||||
assert written is not None and written[0].seq > resealed.manifest_seq
|
||||
assert (await verify_manifest(first.id))["ok"] is True
|
||||
|
||||
|
||||
async def _legacy_sealed(ending: str | None) -> Run:
|
||||
"""加 manifest_seq 这一列之前封存的老运行:只记了清单哈希,封到哪一条要由 verify_manifest 按事件类型推。"""
|
||||
from app.core.artifact_store import manifest_hash
|
||||
|
||||
rows = [(1, "run.started", None, {"nodes": 2}), (2, "node.started", "out", {}),
|
||||
(3, "node.finished", "out", {"output": {"r": "完成"}})]
|
||||
if ending:
|
||||
rows.append((4, ending, None, {}))
|
||||
# 老口径:run.finished 在算哈希之后才落库,不在清单里;run.failed / run.cancelled 在
|
||||
covered = rows[:-1] if ending == "run.finished" else rows
|
||||
async with SessionLocal() as session:
|
||||
run = Run(workflow_name="老运行·月报", status="failed" if ending == "run.failed" else "succeeded",
|
||||
graph={}, input={}, last_seq=rows[-1][0], manifest_hash=manifest_hash(covered), manifest_seq=None)
|
||||
session.add(run)
|
||||
await session.flush()
|
||||
for seq, kind, node_id, data in rows:
|
||||
session.add(RunEvent(run_id=run.id, seq=seq, type=kind, node_id=node_id, ts=float(seq), data=data))
|
||||
await session.commit()
|
||||
return run
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ending", [None, "run.finished", "run.failed"], ids=["no-ending", "finished", "failed"])
|
||||
async def test_a_legacy_seal_takes_appends_only_when_its_range_does_not_move(ending):
|
||||
"""老封存的范围是推出来的:有终态事件的,封到最后一条终态事件为止,追加在后面不进范围;没有终态事件的,
|
||||
封到最后一行——追加一条,范围就跟着挪到它身上,我们自己写的批注被当成篡改。这种不追加。"""
|
||||
from app.engine.run_append import append_events
|
||||
|
||||
legacy = await _legacy_sealed(ending)
|
||||
before = await verify_manifest(legacy.id)
|
||||
assert before["ok"] is True and before["legacy"] is True
|
||||
written = await append_events(legacy.id, [inquiry("这期为什么少了一家店")], require="sealed")
|
||||
if ending is None:
|
||||
assert written is None
|
||||
assert (await _row(legacy.id)).last_seq == legacy.last_seq
|
||||
else:
|
||||
assert written is not None and written[0].data["post_seal"] is True
|
||||
after = await verify_manifest(legacy.id)
|
||||
assert after["ok"] is True and (after["sealed_at"], after["events"]) == (before["sealed_at"], before["events"])
|
||||
|
||||
|
||||
async def test_the_wrong_kind_of_run_gets_nothing():
|
||||
from app.engine.run_append import append_events
|
||||
|
||||
paused, final = await _paused(), await _finished()
|
||||
assert await append_events(paused.id, [inquiry("甲")], require="sealed") is None, "停在审批的运行还没封存"
|
||||
assert await append_events(final.id, [inquiry("乙")], require="interrupted") is None, "已结束的运行不在等审批"
|
||||
assert await append_events("no-such-run", [inquiry("丙")], require="interrupted") is None
|
||||
assert await append_events(paused.id, [], require="interrupted") == []
|
||||
with pytest.raises(ValueError):
|
||||
await append_events(paused.id, [inquiry("丁")], require="running")
|
||||
with pytest.raises(ValueError):
|
||||
await append_events(paused.id, [("approval.unknown", None, {})], require="interrupted")
|
||||
for run in (paused, final):
|
||||
assert (await _row(run.id)).last_seq == run.last_seq
|
||||
assert await _events(run.id, "approval.inquiry") == []
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 序号
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_seqs_strictly_increase_and_never_repeat():
|
||||
from app.engine.run_append import append_events
|
||||
|
||||
paused = await _paused()
|
||||
seqs: list[int] = []
|
||||
for batch in ([inquiry("一")], [inquiry("二"), inquiry("三")], [inquiry("四")]):
|
||||
written = await append_events(paused.id, batch, require="interrupted")
|
||||
seqs += [e.seq for e in written]
|
||||
assert seqs[0] > paused.last_seq and _strictly_increasing(seqs)
|
||||
|
||||
# 同时来几个:同一运行在本进程里排队,读 last_seq 到写入之间不会被另一次插进来,所以一个都不落空
|
||||
results = await asyncio.gather(*(append_events(paused.id, [inquiry(f"并发{i}")], require="interrupted")
|
||||
for i in range(6)))
|
||||
assert all(r is not None for r in results), "并发的追加有落空的"
|
||||
together = sorted(r[0].seq for r in results)
|
||||
assert together[0] > seqs[-1] and _strictly_increasing(together)
|
||||
assert (await _row(paused.id)).last_seq == together[-1]
|
||||
await _no_repeated_seq(paused.id)
|
||||
|
||||
await run_manager.resume(paused.id, {"approved": True})
|
||||
final = await _until(paused.id, ("succeeded",), sealed=True)
|
||||
assert (await verify_manifest(final.id))["ok"] is True
|
||||
await _no_repeated_seq(final.id)
|
||||
|
||||
|
||||
async def test_a_restarted_process_continues_after_the_stored_events():
|
||||
"""进程重启过,内存里的序号从 0 起:追加接在库里最后一条后面。"""
|
||||
from app.engine.run_append import append_events
|
||||
|
||||
paused = await _paused()
|
||||
bus._seq.pop(paused.id, None)
|
||||
written = await append_events(paused.id, [inquiry("重启之后的问题")], require="interrupted")
|
||||
assert written[0].seq == paused.last_seq + 1
|
||||
await _no_repeated_seq(paused.id)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("action", ["continue", "resume", "abandon"])
|
||||
async def test_the_runner_numbers_after_the_stored_events_when_last_seq_lags(flaky, action):
|
||||
"""last_seq 比库里实际的最后一条小(怎么来的见下一条),本进程又没有这次运行的序号计数(重启过):继续运行、
|
||||
恢复、放弃都接在实际的最后一条后面。只按 last_seq 接,写出的 run.resumed / run.cancelled 撞上已有的序号,
|
||||
重新封存时两条同号的事件一起进了清单。"""
|
||||
if action == "continue":
|
||||
run = await _finished(FLAKY)
|
||||
assert run.status == "failed"
|
||||
else:
|
||||
run = await _paused()
|
||||
stored = (await _events(run.id))[-1].seq
|
||||
_sql("UPDATE runs SET last_seq = last_seq - 3 WHERE id = ?", run.id)
|
||||
bus._seq.pop(run.id, None)
|
||||
|
||||
if action == "continue":
|
||||
flaky["fail"] = False
|
||||
await run_manager.continue_failed(run.id)
|
||||
first_new = "run.resumed"
|
||||
elif action == "resume":
|
||||
await run_manager.resume(run.id, {"approved": True})
|
||||
first_new = "run.resumed"
|
||||
else:
|
||||
assert await run_manager.cancel(run.id) == "cancelled"
|
||||
first_new = "run.cancelled"
|
||||
final = await _until(run.id, ("succeeded", "cancelled"), sealed=True)
|
||||
assert final.status == ("cancelled" if action == "abandon" else "succeeded"), final.error
|
||||
assert [e.seq for e in await _events(run.id, first_new)][-1] == stored + 1
|
||||
await _no_repeated_seq(run.id)
|
||||
assert (await verify_manifest(run.id))["ok"] is True
|
||||
|
||||
|
||||
async def test_runner_event_writes_never_lower_last_seq():
|
||||
"""runner 记事件时 last_seq 只往大里改。取号和提交的先后可以不一样:别处另开会话记的事件(run_manager.note)
|
||||
和执行任务同时在写,号大的先提交、号小的后提交,以前后提交的那条把 last_seq 写回小的。"""
|
||||
from app.engine.runner import _stage
|
||||
|
||||
paused = await _paused()
|
||||
early = run_manager._event(paused.id, EventType.LOG, data={"level": "info", "message": "先取号"})
|
||||
late = run_manager._event(paused.id, EventType.LOG, data={"level": "info", "message": "后取号"})
|
||||
for event in (late, early):
|
||||
async with SessionLocal() as session:
|
||||
await _stage(session, event)
|
||||
await session.commit()
|
||||
assert (await _row(paused.id)).last_seq == late.seq
|
||||
|
||||
|
||||
def _judged_args(run: Run) -> tuple:
|
||||
"""直接调按需裁判记判定的那一步(api/evidence.py 的 _append_judged)要的参数:问过一次模型、什么都没判成。"""
|
||||
from app.api import evidence
|
||||
|
||||
sealed = evidence._Sealed(run_id=run.id, graph={}, seal={"sealed": True, "ok": True}, events=[],
|
||||
manifest_hash=run.manifest_hash, run_manifest_seq=run.manifest_seq)
|
||||
report = evidence._Report(node_id="out", doc_artifact="doc-1", ok=None, repairs=None, stats=None, doc=None,
|
||||
hash_ok=None, fields=[])
|
||||
outcome = {key: {} for key in ("verdicts", "keys", "unjudged", "skipped", "budget")}
|
||||
outcome.update(model="judge-model", priced=True, same_model=False, writer_model=None, cost_usd=0.0021,
|
||||
calls=1, duration_ms=1200, limits_hit=[], gaps=[], notes=[])
|
||||
return sealed, report, ["u1"], outcome
|
||||
|
||||
|
||||
async def test_a_judge_interleaved_with_a_post_seal_append_survives_a_restart(flaky, monkeypatch):
|
||||
"""按需裁判记判定(_append_judged)的条件更新不带 last_seq 条件:它取了号、还没写,封存后追加抢先提交了,
|
||||
它再提交时把 last_seq 写回自己那个更小的号。同一进程里序号计数只增不减,不会撞号;进程重启后计数从 0 起,
|
||||
各处又按 last_seq 接号——裁判自己也是。所以启动时按库里实际的最后一条改正 last_seq:之后再裁判、继续运行,
|
||||
都接在实际的最后一条后面。"""
|
||||
from app.api import evidence
|
||||
from app.engine.run_append import append_events
|
||||
|
||||
first = await _finished(FLAKY)
|
||||
assert first.status == "failed"
|
||||
held, opened = asyncio.Event(), evidence.SessionLocal
|
||||
|
||||
class HeldBeforeUpdate:
|
||||
"""裁判的会话:执行条件更新之前停住,等封存后追加先提交。"""
|
||||
|
||||
def __init__(self):
|
||||
self._cm = opened()
|
||||
|
||||
async def __aenter__(self):
|
||||
session = await self._cm.__aenter__()
|
||||
execute = session.execute
|
||||
|
||||
async def held_execute(statement, *args, **kwargs):
|
||||
if getattr(statement, "is_update", False):
|
||||
await held.wait()
|
||||
return await execute(statement, *args, **kwargs)
|
||||
|
||||
session.execute = held_execute
|
||||
return session
|
||||
|
||||
async def __aexit__(self, *exc):
|
||||
return await self._cm.__aexit__(*exc)
|
||||
|
||||
monkeypatch.setattr(evidence, "SessionLocal", HeldBeforeUpdate)
|
||||
judging = asyncio.create_task(evidence._append_judged(*_judged_args(first)))
|
||||
for _ in range(200): # 裁判取到号了
|
||||
if bus._seq.get(first.id, 0) > first.last_seq:
|
||||
break
|
||||
await asyncio.sleep(0.01)
|
||||
written = await append_events(first.id, [inquiry("这次为什么失败")], require="sealed")
|
||||
held.set()
|
||||
assert await judging == {"seq": first.last_seq + 1}
|
||||
assert written is not None and written[0].seq == first.last_seq + 2
|
||||
assert (await _row(first.id)).last_seq == first.last_seq + 1, "裁判后提交,last_seq 被写回了小的号"
|
||||
monkeypatch.setattr(evidence, "SessionLocal", opened)
|
||||
|
||||
# 进程重启:内存里的序号计数没了,启动时 setup 照常执行一遍。只改落后的那条,别的运行原样不动
|
||||
lagging = await _row(first.id)
|
||||
others = [await _finished(), await _paused()]
|
||||
await run_manager.shutdown()
|
||||
bus._seq.pop(first.id, None)
|
||||
await run_manager.setup()
|
||||
repaired = await _row(first.id)
|
||||
assert (repaired.last_seq, repaired.updated_at) == (written[-1].seq, lagging.updated_at)
|
||||
for other in others:
|
||||
now = await _row(other.id)
|
||||
assert (now.last_seq, now.updated_at) == (other.last_seq, other.updated_at)
|
||||
assert await evidence._append_judged(*_judged_args(first)) == {"seq": written[-1].seq + 1}
|
||||
await _no_repeated_seq(first.id)
|
||||
|
||||
flaky["fail"] = False
|
||||
await run_manager.continue_failed(first.id)
|
||||
final = await _until(first.id, ("succeeded",), sealed=True)
|
||||
assert final.manifest_seq > written[-1].seq + 1
|
||||
await _no_repeated_seq(first.id)
|
||||
assert (await verify_manifest(first.id))["ok"] is True
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 恢复后的重放:Agent 里批过的工具不重复执行
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _script_two_calls(monkeypatch) -> None:
|
||||
from app.providers import mock_model
|
||||
|
||||
def _decide(self, messages):
|
||||
done = sum(isinstance(m, ToolMessage) for m in messages)
|
||||
if self.tools and done < 2:
|
||||
return AIMessage(content="", tool_calls=[
|
||||
{"name": "inquiry_tally", "args": {"label": f"第{done + 1}次"}, "id": f"call_{done + 1}"}])
|
||||
return AIMessage(content="两笔都记好了。")
|
||||
|
||||
monkeypatch.setattr(mock_model.MockChatModel, "_decide", _decide)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("older_pause", [False, True], ids=["current", "older-protocol"])
|
||||
async def test_inquiries_do_not_disturb_the_replay(monkeypatch, older_pause):
|
||||
"""Agent 停在工具审批上时追加问答,批准后工具各执行一次。升级前写下的断点要按事件流判断能不能
|
||||
安全恢复(runner._stale_replay 只看 tool.start / human.resolved),追加的问答不影响判断。"""
|
||||
from app.engine import replay, runner
|
||||
from app.engine.run_append import append_events
|
||||
|
||||
_script_two_calls(monkeypatch)
|
||||
if older_pause:
|
||||
monkeypatch.setattr(runner, "PROTOCOL", runner.PROTOCOL - 1)
|
||||
graph = chain(node("start", "input"),
|
||||
node("bot", "agent", prompt="记两笔", tools=["inquiry_tally"], approval="always", max_steps=5),
|
||||
node("out", "output", fields=[{"name": "r", "value": "{{ nodes.bot.text }}"}]))
|
||||
run = await run_manager.start(graph=graph, input_payload={})
|
||||
for round_ in range(2):
|
||||
paused = await _until(run.id, ("interrupted", "failed", "succeeded"))
|
||||
assert paused.status == "interrupted", (paused.status, paused.error)
|
||||
assert await append_events(run.id, [inquiry(f"第{round_ + 1}笔记的是什么")], require="interrupted")
|
||||
if older_pause:
|
||||
monkeypatch.setattr(runner, "PROTOCOL", replay.PROTOCOL)
|
||||
await run_manager.resume(run.id, {"approved": True})
|
||||
await asyncio.sleep(0.05)
|
||||
final = await _until(run.id, ("succeeded", "failed"), sealed=True)
|
||||
assert final.status == "succeeded", final.error
|
||||
assert CALLS == ["第1次", "第2次"], "批过的工具又执行了一次"
|
||||
assert len(await _events(run.id, "approval.inquiry")) == 2
|
||||
assert (await verify_manifest(run.id))["ok"] is True
|
||||
await _no_repeated_seq(run.id)
|
||||
@@ -0,0 +1,369 @@
|
||||
"""发起运行的规则搬到 engine/launch.py:HTTP 发起和定时任务到点发起共用同一个 launch_run。
|
||||
|
||||
- 每一种拒绝:POST /api/runs 的状态码、detail、code 先按搬家之前的原话写死(这一组先在旧代码上通过);
|
||||
直接调 launch_run 抛 LaunchRefused,和 HTTP 的返回逐条一致
|
||||
- 发起时定下的治理事件(受管级别那条日志、口径升版处置)在服务里写:不经 HTTP 发起的正式运行一样有;
|
||||
它们和运行记录同一个事务提交,排在 run.started 前面、在封存范围里,写不进去就不建运行
|
||||
- 旧的导入路径(app.api.runs 的 missing_tools、node_setting、tool_bindings、TOOL_MISSING)照旧可用;
|
||||
runner 不再从接口层导入
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import inspect
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import Run, RunEvent, Workflow, WorkflowVersion
|
||||
from app.engine.runner import run_manager
|
||||
from app.engine.schema import GraphSpec
|
||||
from app.main import app
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
async def engine_up():
|
||||
await run_manager.setup()
|
||||
yield
|
||||
await run_manager.shutdown()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def client():
|
||||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
|
||||
yield c
|
||||
|
||||
|
||||
def node(nid, ntype, label=None, **config):
|
||||
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": label or nid, "config": config}}
|
||||
|
||||
|
||||
def chain(*nodes):
|
||||
return {"nodes": list(nodes), "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]}
|
||||
|
||||
|
||||
SIMPLE = chain(node("start", "input"), node("out", "output", fields=[{"name": "r", "value": "完成"}]))
|
||||
|
||||
|
||||
def _tag() -> str:
|
||||
return uuid.uuid4().hex[:8]
|
||||
|
||||
|
||||
async def workflow(graph=SIMPLE, *, published=None, versions=(), snapshot=None, name=None) -> tuple[str, str]:
|
||||
"""建一个工作流。published 是当前发布版本号,versions 是要建快照的版本号,snapshot 是快照里的图(默认同 graph)。"""
|
||||
name = name or f"发起测试·周报-{_tag()}"
|
||||
async with SessionLocal() as session:
|
||||
wf = Workflow(name=name, graph=graph, status="published" if published else "draft",
|
||||
published_version=published, version=max(versions, default=1))
|
||||
session.add(wf)
|
||||
await session.flush()
|
||||
for v in versions:
|
||||
session.add(WorkflowVersion(workflow_id=wf.id, version=v,
|
||||
graph=snapshot if snapshot is not None else graph))
|
||||
await session.commit()
|
||||
return wf.id, name
|
||||
|
||||
|
||||
def tool_graph(*tools: tuple[str, str]) -> dict:
|
||||
"""每个 (节点名, 工具名) 一个调用工具节点。"""
|
||||
middle = [node(f"t{i}", "tool", label, tool=tool, args={}) for i, (label, tool) in enumerate(tools)]
|
||||
return chain(node("start", "input"), *middle, node("out", "output", fields=[{"name": "r", "value": "完成"}]))
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 每一种拒绝:(请求体, (状态码, detail, code))。detail 是 None 时只比两边一致,另由 check 查要点
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def case_formal_without_workflow():
|
||||
return {"run_class": "formal"}, (400, "正式运行需要从已保存的工作流发起:请求中没有指定工作流", None)
|
||||
|
||||
|
||||
async def case_formal_with_canvas_graph():
|
||||
wf, _ = await workflow(published=1, versions=(1,))
|
||||
return ({"run_class": "formal", "workflow_id": wf, "graph": SIMPLE},
|
||||
(400, "正式运行只能使用已发布的版本,画布上未发布的修改不会生效。请先保存并发布", None))
|
||||
|
||||
|
||||
async def case_formal_workflow_gone():
|
||||
return {"run_class": "formal", "workflow_id": f"gone{_tag()}"}, (404, "工作流不存在,可能已被删除", None)
|
||||
|
||||
|
||||
async def case_formal_never_published():
|
||||
wf, name = await workflow()
|
||||
return {"run_class": "formal", "workflow_id": wf}, (409, f"「{name}」还没有发布版本,请先在编排页发布", None)
|
||||
|
||||
|
||||
async def case_formal_not_the_published_version():
|
||||
wf, _ = await workflow(published=2, versions=(1, 2))
|
||||
return ({"run_class": "formal", "workflow_id": wf, "version": 1},
|
||||
(409, "v1 不是当前发布版本(v2)。正式运行只能使用当前发布版本;如需运行这一版,请先发布它。", None))
|
||||
|
||||
|
||||
async def case_formal_snapshot_missing():
|
||||
wf, _ = await workflow(published=3, versions=(1, 2))
|
||||
return {"run_class": "formal", "workflow_id": wf}, (404, "版本 v3 不存在", None)
|
||||
|
||||
|
||||
async def case_formal_snapshot_malformed():
|
||||
"""快照里的图不成形:以前由 main.py 的 ValueError 兜底回 400,detail 是校验的原话。"""
|
||||
broken = {"nodes": "坏掉的图"}
|
||||
wf, _ = await workflow(published=1, versions=(1,), snapshot=broken)
|
||||
with pytest.raises(ValueError) as caught:
|
||||
GraphSpec.model_validate(broken)
|
||||
return {"run_class": "formal", "workflow_id": wf}, (400, str(caught.value), None)
|
||||
|
||||
|
||||
def library(version="v3"):
|
||||
return chain(node("start", "input"),
|
||||
node("caliber", "metrics", "周报口径卡", caliber="周报口径", caliber_version=version,
|
||||
metrics=[{"id": "gmv", "name": "销售额", "unit": "元", "expression": "vars.kpi.gmv"}]),
|
||||
node("out", "output", fields=[{"name": "r", "value": "{{ nodes.caliber.text }}"}]))
|
||||
|
||||
|
||||
def weekly(lib_id: str, **card) -> dict:
|
||||
return chain(node("start", "input"),
|
||||
node("fetch", "transform", mode="expression", expression="{'gmv': 4321.5}", assign_to="kpi"),
|
||||
node("card", "metrics", "本地口径卡",
|
||||
caliber_from={"workflow_id": lib_id, "workflow_version": 1, "node_id": "caliber"}, **card),
|
||||
node("out", "output", fields=[{"name": "r", "value": "{{ nodes.card.text }}"}]))
|
||||
|
||||
|
||||
async def upstream_with_two_versions() -> str:
|
||||
async with SessionLocal() as session:
|
||||
lib = Workflow(name=f"口径库-{_tag()}", graph=library(), status="published", published_version=2, version=2)
|
||||
session.add(lib)
|
||||
await session.flush()
|
||||
session.add(WorkflowVersion(workflow_id=lib.id, version=1, graph=library()))
|
||||
session.add(WorkflowVersion(workflow_id=lib.id, version=2, graph=library(version="v4")))
|
||||
await session.commit()
|
||||
return lib.id
|
||||
|
||||
|
||||
async def case_formal_caliber_upgrade_undeclared():
|
||||
lib = await upstream_with_two_versions()
|
||||
graph = weekly(lib)
|
||||
wf, _ = await workflow(graph, published=1, versions=(1,))
|
||||
return {"run_class": "formal", "workflow_id": wf}, (409, None, None)
|
||||
|
||||
|
||||
def check_formal_caliber_upgrade_undeclared(detail: str) -> None:
|
||||
assert "「本地口径卡」" in detail and "v1" in detail and "v2" in detail and "「上游发布新版本时」" in detail
|
||||
|
||||
|
||||
async def case_saved_workflow_gone():
|
||||
return {"workflow_id": f"gone{_tag()}"}, (404, "工作流不存在,可能已被删除", None)
|
||||
|
||||
|
||||
async def case_nothing_to_run():
|
||||
return {}, (400, "没有要运行的内容:请求中既没有工作流 ID,也没有工作流内容", None)
|
||||
|
||||
|
||||
async def case_saved_workflow_is_empty():
|
||||
wf, _ = await workflow({})
|
||||
return {"workflow_id": wf}, (400, "没有要运行的内容:请求中既没有工作流 ID,也没有工作流内容", None)
|
||||
|
||||
|
||||
async def case_missing_datasource_tool():
|
||||
source = f"gone{_tag()}"
|
||||
return ({"graph": tool_graph(("取数", f"db_query__{source}"))},
|
||||
(422, f"绑定的工具不存在:「取数」(调用工具)绑定的 db_query__{source}。请到「数据」页接入,或在节点中重新选择",
|
||||
"run_tool_missing"))
|
||||
|
||||
|
||||
async def case_missing_custom_tool():
|
||||
tool = f"ghost_{_tag()}"
|
||||
return ({"graph": tool_graph(("查编码", tool))},
|
||||
(422, f"绑定的工具不存在:「查编码」(调用工具)绑定的 {tool}。请到「工具」页接入,或在节点中重新选择",
|
||||
"run_tool_missing"))
|
||||
|
||||
|
||||
async def case_missing_tools_of_both_kinds():
|
||||
source, tool = f"gone{_tag()}", f"ghost_{_tag()}"
|
||||
return ({"graph": tool_graph(("取数", f"db_schema__{source}"), ("查编码", tool))},
|
||||
(422, f"绑定的工具不存在:「取数」(调用工具)绑定的 db_schema__{source};「查编码」(调用工具)绑定的 {tool}。"
|
||||
"数据源工具请到「数据」页接入,其他工具请到「工具」页接入,或在节点中重新选择", "run_tool_missing"))
|
||||
|
||||
|
||||
async def case_more_than_eight_missing_tools():
|
||||
tag = _tag()
|
||||
tools = [(f"第{i}步", f"ghost_{tag}_{i}") for i in range(10)]
|
||||
listed = ";".join(f"「{label}」(调用工具)绑定的 {tool}" for label, tool in tools[:8])
|
||||
return ({"graph": tool_graph(*tools)},
|
||||
(422, f"绑定的工具不存在:{listed}等 10 处。请到「工具」页接入,或在节点中重新选择", "run_tool_missing"))
|
||||
|
||||
|
||||
async def case_formal_missing_tool():
|
||||
source = f"gone{_tag()}"
|
||||
wf, _ = await workflow(tool_graph(("取数", f"db_query__{source}")), published=1, versions=(1,))
|
||||
return ({"run_class": "formal", "workflow_id": wf},
|
||||
(422, f"绑定的工具不存在:「取数」(调用工具)绑定的 db_query__{source}。请到「数据」页接入,或在节点中重新选择",
|
||||
"run_tool_missing"))
|
||||
|
||||
|
||||
async def case_graph_fails_validation():
|
||||
return ({"graph": {"nodes": [], "edges": []}},
|
||||
(400, "工作流未通过校验,请先在画布上修正标红的节点再运行:画布上还没有节点,请先从节点库拖入一个节点", None))
|
||||
|
||||
|
||||
#: 「不存在」的工作流、数据源、自定义工具都带随机后缀,库里不会恰好有同名的
|
||||
CASES = {name[len("case_"):]: fn for name, fn in dict(globals()).items() if name.startswith("case_")}
|
||||
|
||||
|
||||
def _body(status: int, detail: str, code: str | None) -> dict:
|
||||
return {"detail": detail, **({"code": code} if code else {})}
|
||||
|
||||
|
||||
async def _run_count() -> int:
|
||||
async with SessionLocal() as session:
|
||||
return len((await session.execute(select(Run.id))).all())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", list(CASES))
|
||||
async def test_http_refusals_are_unchanged(client, case):
|
||||
"""HTTP 这一侧逐字写死:搬家前后都得是这些状态码和原话。"""
|
||||
body, (status, detail, code) = await CASES[case]()
|
||||
before = await _run_count()
|
||||
res = await client.post("/api/runs", json=body)
|
||||
assert res.status_code == status, res.text
|
||||
got = res.json()
|
||||
if detail is None:
|
||||
globals()[f"check_{case}"](got["detail"])
|
||||
assert set(got) == ({"detail", "code"} if code else {"detail"}), got
|
||||
else:
|
||||
assert got == _body(status, detail, code)
|
||||
assert await _run_count() == before, "被拒的请求不该留下运行记录"
|
||||
|
||||
|
||||
def _launch_kwargs(body: dict) -> dict:
|
||||
"""请求体换成 launch_run 的参数:input 对应 input_payload,其余同名。"""
|
||||
out = {k: v for k, v in body.items() if k != "input"}
|
||||
if "input" in body:
|
||||
out["input_payload"] = body["input"]
|
||||
return out
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", list(CASES))
|
||||
async def test_launch_run_refuses_exactly_like_http(client, case):
|
||||
from app.engine.launch import LaunchRefused, launch_run
|
||||
|
||||
body, (status, detail, code) = await CASES[case]()
|
||||
before = await _run_count()
|
||||
async with SessionLocal() as session:
|
||||
with pytest.raises(LaunchRefused) as caught:
|
||||
await launch_run(session, actor="定时任务", **_launch_kwargs(body))
|
||||
refused = caught.value
|
||||
assert (refused.status_code, refused.code) == (status, code)
|
||||
if detail is not None:
|
||||
assert refused.message == detail
|
||||
res = await client.post("/api/runs", json=body)
|
||||
assert res.status_code == refused.status_code
|
||||
assert res.json() == _body(refused.status_code, refused.message, refused.code)
|
||||
assert await _run_count() == before
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 发起成功:参数照转、治理事件在服务里写
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def _settle(run_id: str) -> Run:
|
||||
for _ in range(400):
|
||||
async with SessionLocal() as session:
|
||||
row = await session.get(Run, run_id)
|
||||
if row.status in ("succeeded", "failed", "cancelled") and row.manifest_seq is not None:
|
||||
return row
|
||||
await asyncio.sleep(0.03)
|
||||
raise AssertionError(f"运行 {run_id} 没有结束:{row.status}")
|
||||
|
||||
|
||||
async def _events(run_id: str, etype: str) -> list[RunEvent]:
|
||||
async with SessionLocal() as session:
|
||||
return list((await session.execute(
|
||||
select(RunEvent).where(RunEvent.run_id == run_id, RunEvent.type == etype).order_by(RunEvent.seq)
|
||||
)).scalars())
|
||||
|
||||
|
||||
async def test_a_formal_launch_records_the_governance_notes():
|
||||
"""不经 HTTP 发起的正式运行(定时任务)也记下按什么级别出具、口径升版怎么处置的。"""
|
||||
from app.engine.launch import launch_run
|
||||
|
||||
lib = await upstream_with_two_versions()
|
||||
wf, name = await workflow(weekly(lib, upgrade_policy="dual"), published=1, versions=(1,))
|
||||
async with SessionLocal() as session:
|
||||
run = await launch_run(session, workflow_id=wf, run_class="formal", input_payload={"week": "第 40 周"},
|
||||
actor="定时任务")
|
||||
assert (run.run_class, run.version, run.workflow_name, run.started_by) == ("formal", 1, name, "定时任务")
|
||||
assert run.version_hash and run.input == {"week": "第 40 周"}
|
||||
row = await _settle(run.id)
|
||||
assert row.status == "succeeded", row.error
|
||||
notes = [e for e in await _events(run.id, "log") if e.data.get("code") == "run_governance"]
|
||||
assert len(notes) == 1 and notes[0].data["governed"] is False and "v1" in notes[0].data["message"]
|
||||
[upgrade] = await _events(run.id, "caliber.upgrade")
|
||||
assert upgrade.node_id == "card" and upgrade.data["policy"] == "dual" and upgrade.data["latest"] == 2
|
||||
# 发起时就有的事件排在 run.started 前面,都在封存范围里
|
||||
[started] = await _events(run.id, "run.started")
|
||||
assert notes[0].seq < upgrade.seq < started.seq <= row.manifest_seq
|
||||
|
||||
|
||||
async def test_launch_events_are_written_with_the_run_or_not_at_all():
|
||||
"""发起时定下的事件和运行记录同一个事务提交:写不进去就不建运行,launch_run 抛出异常就是没有发起。以前运行
|
||||
建好、开始执行之后才另外写,写失败时运行已经在执行,调用方(定时任务)只拿到一个异常,会把一次正在执行的
|
||||
正式运行记成未能发起,封存范围里也少了受管级别那条日志。"""
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
from app.db.base import engine
|
||||
from app.engine.launch import launch_run
|
||||
|
||||
lib = await upstream_with_two_versions()
|
||||
wf, _ = await workflow(weekly(lib, upgrade_policy="dual"), published=1, versions=(1,))
|
||||
before = await _run_count()
|
||||
# 口径升版处置那条写不进去。走异步引擎建删触发器:同步连接等写锁时会卡住事件循环,而持锁的正是这个循环里的写入
|
||||
async with engine.begin() as conn:
|
||||
await conn.execute(text(
|
||||
"CREATE TRIGGER refuse_upgrade_note BEFORE INSERT ON run_events WHEN NEW.type = 'caliber.upgrade' "
|
||||
"BEGIN SELECT RAISE(ABORT, 'refused by test'); END"))
|
||||
try:
|
||||
async with SessionLocal() as session:
|
||||
with pytest.raises(IntegrityError):
|
||||
await launch_run(session, workflow_id=wf, run_class="formal", actor="定时任务")
|
||||
assert await _run_count() == before, "没写进治理事件,却留下了一条运行"
|
||||
finally:
|
||||
async with engine.begin() as conn:
|
||||
await conn.execute(text("DROP TRIGGER IF EXISTS refuse_upgrade_note"))
|
||||
|
||||
|
||||
async def test_an_exploratory_launch_takes_the_saved_graph_and_the_run_defaults():
|
||||
from app.engine.launch import launch_run
|
||||
|
||||
wf, name = await workflow()
|
||||
async with SessionLocal() as session:
|
||||
run = await launch_run(session, workflow_id=wf, actor=None)
|
||||
assert (run.run_class, run.workflow_id, run.workflow_name, run.version) == ("exploratory", wf, name, None)
|
||||
assert (run.memory_scope, run.collection) == ("default", "default") and run.input == {}
|
||||
row = await _settle(run.id)
|
||||
assert row.status == "succeeded", row.error
|
||||
assert [e for e in await _events(run.id, "log") if e.data.get("code") == "run_governance"] == []
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 旧的导入路径
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_the_old_import_paths_still_work():
|
||||
from app.api import runs as runs_api
|
||||
from app.engine import launch
|
||||
|
||||
for name in ("missing_tools", "node_setting", "tool_bindings", "TOOL_MISSING"):
|
||||
assert getattr(runs_api, name) is getattr(launch, name), name
|
||||
|
||||
|
||||
def test_the_runner_no_longer_imports_from_the_api_layer():
|
||||
from app.engine import runner
|
||||
|
||||
assert "app.api.runs" not in inspect.getsource(runner)
|
||||
@@ -0,0 +1,268 @@
|
||||
"""运行选项和来源:Run.options 是发起时定下、整次运行沿用的选项快照,Run.schedule_id 是发起它的定时任务。
|
||||
|
||||
- 旧库升级时补这两列和 schedule_id 的索引;存量运行的选项读出来是 {},来源为空
|
||||
- RunManager.start 收下两者并落库;选项非空时写进 run.started(进封存范围),为空时 run.started 和以前
|
||||
逐字相同;恢复、继续运行沿用发起时的选项,节点从 RunContext.extra 读得到同一份
|
||||
- 接口带出 schedule_id,运行列表能按定时任务筛;launch_run 原样往下交
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
from sqlalchemy import create_engine, select, text
|
||||
|
||||
from app.db.base import SessionLocal, engine as app_engine
|
||||
from app.db.models import Run, RunEvent
|
||||
from app.engine import compiler
|
||||
from app.engine.runner import run_manager
|
||||
from app.engine.schema import NodeType
|
||||
from app.main import app
|
||||
|
||||
#: 加 options 之前 run.started 就有的键:选项为空时一个不多、一个不少
|
||||
LEGACY_STARTED_KEYS = {"nodes", "resumed", "memory_scope", "collection", "approval_default", "tool_trust",
|
||||
"agent_limits", "data_versions", "replay_protocol"}
|
||||
OPTIONS = {"verify": {"dual_check": True, "max_queries": 6}, "note": "第 40 周例行核查"}
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
async def engine_up():
|
||||
await run_manager.setup()
|
||||
yield
|
||||
await run_manager.shutdown()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def client():
|
||||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
|
||||
yield c
|
||||
|
||||
|
||||
def node(nid, ntype, **config):
|
||||
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}}
|
||||
|
||||
|
||||
def chain(*nodes):
|
||||
return {"nodes": list(nodes), "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]}
|
||||
|
||||
|
||||
SIMPLE = chain(node("start", "input"), node("out", "output", fields=[{"name": "r", "value": "完成"}]))
|
||||
GATED = chain(node("start", "input"),
|
||||
node("before", "transform", mode="template", template="前"),
|
||||
node("gate", "human", mode="approve", title="放行吗"),
|
||||
node("after", "transform", mode="template", template="后"),
|
||||
node("out", "output", fields=[{"name": "r", "value": "{{ nodes.after }}"}]))
|
||||
|
||||
|
||||
async def _settle(run_id: str, statuses=("succeeded", "failed", "cancelled")) -> Run:
|
||||
for _ in range(400):
|
||||
async with SessionLocal() as session:
|
||||
row = await session.get(Run, run_id)
|
||||
if row.status in statuses and (row.status == "interrupted" or row.manifest_seq is not None):
|
||||
await run_manager.wait_idle(run_id)
|
||||
return row
|
||||
await asyncio.sleep(0.03)
|
||||
raise AssertionError(f"运行 {run_id} 停在 {row.status}")
|
||||
|
||||
|
||||
async def _started(run_id: str) -> list[dict]:
|
||||
async with SessionLocal() as session:
|
||||
return [e.data for e in (await session.execute(
|
||||
select(RunEvent).where(RunEvent.run_id == run_id, RunEvent.type == "run.started").order_by(RunEvent.seq)
|
||||
)).scalars()]
|
||||
|
||||
|
||||
def _schedule() -> str:
|
||||
return uuid.uuid4().hex
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 旧库升级
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_old_databases_get_both_columns_and_the_index():
|
||||
from app.db.base import _migrate
|
||||
|
||||
db = create_engine("sqlite://")
|
||||
with db.begin() as conn:
|
||||
conn.execute(text("CREATE TABLE runs (id VARCHAR(32) PRIMARY KEY, workflow_name VARCHAR(200), "
|
||||
"status VARCHAR(20), last_seq INTEGER)"))
|
||||
conn.execute(text("INSERT INTO runs (id, workflow_name, status, last_seq) VALUES ('r1', '周报', 'succeeded', 9)"))
|
||||
_migrate(conn)
|
||||
_migrate(conn) # 再执行一次不出错:列和索引都已经在了
|
||||
cols = {row[1] for row in conn.execute(text("PRAGMA table_info(runs)"))}
|
||||
assert {"options", "schedule_id"} <= cols
|
||||
assert conn.execute(text("SELECT options, schedule_id FROM runs WHERE id = 'r1'")).one() == ("{}", None)
|
||||
indexes = {row[1]: row for row in conn.execute(text("PRAGMA index_list(runs)"))}
|
||||
assert "ix_runs_schedule_id" in indexes
|
||||
assert [r[2] for r in conn.execute(text("PRAGMA index_info(ix_runs_schedule_id)"))] == ["schedule_id"]
|
||||
|
||||
|
||||
def test_a_missing_table_is_left_alone():
|
||||
"""还没有 runs 表的库(create_all 之前不会这样,这里只确认迁移不替它建索引、不报错)。"""
|
||||
from app.db.base import _migrate
|
||||
|
||||
db = create_engine("sqlite://")
|
||||
with db.begin() as conn:
|
||||
_migrate(conn)
|
||||
assert conn.execute(text("SELECT name FROM sqlite_master WHERE name = 'ix_runs_schedule_id'")).first() is None
|
||||
|
||||
|
||||
async def test_new_databases_index_schedule_id():
|
||||
async with app_engine.connect() as conn:
|
||||
names = {row[1] for row in (await conn.execute(text("PRAGMA index_list(runs)"))).all()}
|
||||
assert "ix_runs_schedule_id" in names
|
||||
|
||||
|
||||
async def test_a_migrated_row_reads_back_empty_options():
|
||||
"""迁移回填的是字符串 '{}':经 ORM 读出来是空 dict,不是字符串,也不是 None。"""
|
||||
async with SessionLocal() as session:
|
||||
run = Run(workflow_name="迁移回填", status="succeeded", graph={}, input={})
|
||||
session.add(run)
|
||||
await session.commit()
|
||||
run_id = run.id
|
||||
async with app_engine.begin() as conn:
|
||||
await conn.execute(text("UPDATE runs SET options = '{}' WHERE id = :id"), {"id": run_id})
|
||||
async with SessionLocal() as session:
|
||||
row = await session.get(Run, run_id)
|
||||
assert row.options == {} and row.schedule_id is None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 发起:落库、run.started、沿用
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_options_and_schedule_are_stored_and_sealed():
|
||||
sid = _schedule()
|
||||
run = await run_manager.start(graph=SIMPLE, input_payload={}, options=OPTIONS, schedule_id=sid)
|
||||
assert (run.options, run.schedule_id) == (OPTIONS, sid)
|
||||
row = await _settle(run.id)
|
||||
assert row.status == "succeeded", row.error
|
||||
assert (row.options, row.schedule_id) == (OPTIONS, sid)
|
||||
[started] = await _started(run.id)
|
||||
assert started["options"] == OPTIONS
|
||||
assert set(started) == LEGACY_STARTED_KEYS | {"options"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("options", [None, {}])
|
||||
async def test_without_options_run_started_is_unchanged(options):
|
||||
run = await run_manager.start(graph=SIMPLE, input_payload={}, options=options)
|
||||
row = await _settle(run.id)
|
||||
assert row.options == {} and row.schedule_id is None
|
||||
[started] = await _started(run.id)
|
||||
assert set(started) == LEGACY_STARTED_KEYS, "选项为空时 run.started 多出了键"
|
||||
|
||||
|
||||
async def test_the_options_belong_to_the_run_not_to_the_callers_dict():
|
||||
mine = {"verify": {"dual_check": True}}
|
||||
run = await run_manager.start(graph=SIMPLE, input_payload={}, options=mine)
|
||||
mine["verify"]["dual_check"] = False
|
||||
mine["late"] = 1
|
||||
await _settle(run.id)
|
||||
async with SessionLocal() as session:
|
||||
assert (await session.get(Run, run.id)).options == {"verify": {"dual_check": True}}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def seen_options(monkeypatch):
|
||||
"""记下每次执行「数据整形」节点时它从运行上下文里读到的选项。"""
|
||||
from app.engine.runner import RUN_OPTIONS_KEY
|
||||
|
||||
seen: list = []
|
||||
original = compiler.RUNNERS[NodeType.TRANSFORM]
|
||||
|
||||
async def spy(state, ctx):
|
||||
seen.append(ctx.run.extra.get(RUN_OPTIONS_KEY))
|
||||
return await original(state, ctx)
|
||||
|
||||
monkeypatch.setitem(compiler.RUNNERS, NodeType.TRANSFORM, spy)
|
||||
return seen
|
||||
|
||||
|
||||
async def test_resume_keeps_the_options_the_run_started_with(seen_options):
|
||||
run = await run_manager.start(graph=GATED, input_payload={}, options=OPTIONS)
|
||||
assert (await _settle(run.id, ("interrupted",))).status == "interrupted"
|
||||
await run_manager.resume(run.id, {"approved": True})
|
||||
row = await _settle(run.id)
|
||||
assert row.status == "succeeded", row.error
|
||||
assert seen_options == [OPTIONS, OPTIONS], "恢复之后节点读到的选项变了"
|
||||
first, second = await _started(run.id)
|
||||
assert first["options"] == second["options"] == OPTIONS and second["resumed"] is True
|
||||
|
||||
|
||||
async def test_continue_keeps_the_options_too(seen_options, monkeypatch):
|
||||
original = compiler.RUNNERS[NodeType.TRANSFORM]
|
||||
calls = {"n": 0}
|
||||
|
||||
async def flaky(state, ctx):
|
||||
calls["n"] += 1
|
||||
if ctx.node.id == "after" and calls["n"] <= 2:
|
||||
raise RuntimeError("临时故障")
|
||||
return await original(state, ctx)
|
||||
|
||||
monkeypatch.setitem(compiler.RUNNERS, NodeType.TRANSFORM, flaky)
|
||||
graph = chain(node("start", "input"),
|
||||
node("before", "transform", mode="template", template="前"),
|
||||
node("after", "transform", mode="template", template="后"),
|
||||
node("out", "output", fields=[{"name": "r", "value": "{{ nodes.after }}"}]))
|
||||
run = await run_manager.start(graph=graph, input_payload={}, options=OPTIONS)
|
||||
assert (await _settle(run.id)).status == "failed"
|
||||
await run_manager.continue_failed(run.id)
|
||||
row = await _settle(run.id)
|
||||
assert row.status == "succeeded", row.error
|
||||
assert [s.get("options") for s in await _started(run.id)] == [OPTIONS, OPTIONS]
|
||||
|
||||
|
||||
async def test_nodes_read_an_empty_dict_when_there_are_no_options(seen_options):
|
||||
run = await run_manager.start(graph=GATED, input_payload={})
|
||||
await _settle(run.id, ("interrupted",))
|
||||
assert seen_options == [{}]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 接口:带出来源、按定时任务筛、launch_run 照转
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_the_api_shows_and_filters_by_schedule(client):
|
||||
sid, other = _schedule(), _schedule()
|
||||
mine = [await run_manager.start(graph=SIMPLE, input_payload={}, schedule_id=sid) for _ in range(2)]
|
||||
theirs = await run_manager.start(graph=SIMPLE, input_payload={}, schedule_id=other)
|
||||
manual = await run_manager.start(graph=SIMPLE, input_payload={})
|
||||
for run in (*mine, theirs, manual):
|
||||
await _settle(run.id)
|
||||
|
||||
got = (await client.get(f"/api/runs/{mine[0].id}")).json()
|
||||
assert got["schedule_id"] == sid
|
||||
assert (await client.get(f"/api/runs/{manual.id}")).json()["schedule_id"] is None
|
||||
|
||||
listed = (await client.get("/api/runs", params={"schedule_id": sid})).json()
|
||||
assert sorted(r["id"] for r in listed) == sorted(r.id for r in mine)
|
||||
assert all(r["schedule_id"] == sid for r in listed)
|
||||
# 不带这个参数时照旧全列
|
||||
everything = {r["id"] for r in (await client.get("/api/runs", params={"limit": 200})).json()}
|
||||
assert {theirs.id, manual.id} <= everything
|
||||
|
||||
|
||||
async def test_launch_run_passes_both_through():
|
||||
from app.engine.launch import launch_run
|
||||
|
||||
sid = _schedule()
|
||||
async with SessionLocal() as session:
|
||||
run = await launch_run(session, graph=SIMPLE, actor="定时任务", schedule_id=sid, options=OPTIONS)
|
||||
row = await _settle(run.id)
|
||||
assert (row.schedule_id, row.options, row.started_by) == (sid, OPTIONS, "定时任务")
|
||||
[started] = await _started(run.id)
|
||||
assert started["options"] == OPTIONS
|
||||
|
||||
|
||||
async def test_an_http_launch_has_no_schedule_and_no_options(client):
|
||||
res = await client.post("/api/runs", json={"graph": SIMPLE})
|
||||
assert res.status_code == 201, res.text
|
||||
assert res.json()["schedule_id"] is None
|
||||
row = await _settle(res.json()["id"])
|
||||
assert row.options == {}
|
||||
Reference in New Issue
Block a user