审批质询的用量计进运行用量:质询不在图里执行、不发 llm.end,花的 token 和金额写在 approval.inquiry 事件数据的 usage 里(input_tokens、output_tokens、cost_usd、calls,可带 unpriced_calls),以前运行用量只认图的状态和 llm.end,质询花的钱在运行页上看不见。_usage_of_events 把质询的用量单独加总放在 inquiry 下:llm.end 这本账要和状态那本比大小,质询花的两本里都没有,混进去比会被状态那本盖掉。_finalize 照旧算出图的用量(成功取状态,其余两本取大)后再加上质询的,明细另记在运行用量的 inquiry 下;停在审批时被放弃的运行,放弃时补上这次停下期间的质询。停过两次、两次都问过的,按上次记下的 inquiry 先减后加,每次质询只计一遍。没有质询的运行用量和以前逐字相同

This commit is contained in:
ThearchyHelios
2026-10-05 00:18:22 +08:00
parent 3dbd69b561
commit 6b272de6f9
2 changed files with 303 additions and 4 deletions
+55 -4
View File
@@ -551,7 +551,12 @@ class RunManager:
closed = await _close_approvals(session, run_id, actor, now)
run = await session.get(Run, run_id)
timing = await _timing(session, run, 0, now)
run.usage = {**(run.usage or {}), "duration_ms": timing["active_ms"], **timing}
usage = dict(run.usage or {})
# 停下时记的用量里还没有这次停在审批期间问的质询,这里补上(上次停下时计过的不再计)
inquiry = (await _usage_of_events(session, run_id)).get("inquiry")
if inquiry or usage.get("inquiry"):
usage = _with_inquiry(usage, inquiry)
run.usage = {**usage, "duration_ms": timing["active_ms"], **timing}
run.error = "用户取消:等待审批时放弃了这次运行" if closed else "用户取消:挂起期间放弃了这次运行"
bus.set_seq(run_id, run.last_seq or 0)
# 和状态同一个事务提交,理由同 _finalize
@@ -857,6 +862,7 @@ class RunManager:
if run:
timing = await _timing(session, run, elapsed_ms, now)
events_usage = await _usage_of_events(session, run_id)
inquiry = events_usage.pop("inquiry", None)
if status != "succeeded":
# 状态里的用量只算跑完了的节点。失败、取消、停在审批上的那个节点已经
# 花出去的调用(校验修复、agent 的前几步)只在 llm.end 里——以前这部分
@@ -865,6 +871,9 @@ class RunManager:
elif events_usage.get("unpriced_calls"):
# 金额估不出来的调用只有 llm.end 数得出(状态里的用量只有金额,0 分不出是免费还是没价)
usage = {**usage, "unpriced_calls": events_usage["unpriced_calls"]}
if inquiry:
# 审批质询不在图里执行,状态和 llm.end 两本账里都没有它:比完再加上
usage = _with_inquiry(usage, inquiry)
# duration_ms 是各段执行时长之和(= active_ms)。以前用这一段的耗时
# 覆盖它:87 秒、中间等过一次审批的运行,列表上写着 22ms
usage.update(duration_ms=timing["active_ms"], **timing)
@@ -1270,18 +1279,35 @@ async def _close_approvals(session: Any, run_id: str, actor: str | None, now: da
_USAGE_KEYS = ("input_tokens", "output_tokens", "cost_usd", "calls")
#: 审批质询的用量里计进运行用量的键:unpriced_calls 是其中金额估不出来的调用次数(模型不在价格目录里)
_INQUIRY_KEYS = (*_USAGE_KEYS, "unpriced_calls")
async def _usage_of_events(session: Any, run_id: str) -> dict[str, Any]:
"""按 llm.end 事件逐次加总的用量:每次模型调用都有一条,不管节点最后成没成。"""
"""按事件加总的用量。
llm.end 逐次加总:每次模型调用都有一条,不管节点最后成没成。审批质询不在图里执行、不发 llm.end,
它的用量写在 approval.inquiry 事件数据的 usage 里,单独加总放在 inquiry 下(没有质询就没有这个键):
llm.end 这本账要和状态里的用量比大小(_spent),质询花的两本里都没有,混进去比会被状态那本盖掉。
"""
rows = await session.execute(
select(RunEvent.data).where(RunEvent.run_id == run_id, RunEvent.type == EventType.LLM_END)
select(RunEvent.type, RunEvent.data).where(
RunEvent.run_id == run_id, RunEvent.type.in_([EventType.LLM_END, EventType.APPROVAL_INQUIRY]))
)
from app.providers.catalog import unpriced_call
total: dict[str, Any] = dict.fromkeys(_USAGE_KEYS, 0)
unpriced = 0
for (data,) in rows:
inquiry: dict[str, Any] = {}
for kind, data in rows:
if kind == EventType.APPROVAL_INQUIRY:
spent = (data or {}).get("usage")
if isinstance(spent, dict):
for key in _INQUIRY_KEYS:
value = spent.get(key)
if isinstance(value, (int, float)) and not isinstance(value, bool):
inquiry[key] = inquiry.get(key, 0) + value
continue
for key in _USAGE_KEYS:
value = (data or {}).get(key, 1 if key == "calls" else 0)
if isinstance(value, (int, float)) and not isinstance(value, bool):
@@ -1291,9 +1317,34 @@ async def _usage_of_events(session: Any, run_id: str) -> dict[str, Any]:
if unpriced:
# 有几次调用的金额估不出来(模型不在价格目录里):cost_usd 只是估得出的那部分,界面据此不写 $0
total["unpriced_calls"] = unpriced
if inquiry:
if "cost_usd" in inquiry:
inquiry["cost_usd"] = round(inquiry["cost_usd"], 6)
total["inquiry"] = inquiry
return total
def _with_inquiry(usage: dict[str, Any], inquiry: dict[str, Any] | None) -> dict[str, Any]:
"""把审批质询的用量计进运行用量,明细另放在 inquiry 下(界面要说清其中多少是质询花的)。
usage 里已经计过一回的(上次停下时记下的 inquiry)先减掉再加:停过两次、两次都问过的运行,
每次质询只计一遍。
"""
out = dict(usage)
counted = out.pop("inquiry", None) or {}
for key in _INQUIRY_KEYS:
delta = (inquiry or {}).get(key, 0) - counted.get(key, 0)
if delta:
out[key] = (out.get(key) or 0) + delta
if isinstance(out.get("cost_usd"), float):
out["cost_usd"] = round(out["cost_usd"], 6)
if "input_tokens" in out or "output_tokens" in out:
out["total_tokens"] = int(out.get("input_tokens") or 0) + int(out.get("output_tokens") or 0)
if inquiry:
out["inquiry"] = dict(inquiry)
return out
def _spent(state_usage: dict[str, Any], events_usage: dict[str, Any]) -> dict[str, Any]:
"""两本账取大的那本。新引擎每次调用都发 llm.end,事件那本不会更少;
加这条事件之前的老运行(agent 节点不发 llm.end)状态那本更全。"""
+248
View File
@@ -0,0 +1,248 @@
"""审批质询的用量计进运行用量。
质询不在图里执行、不发 llm.end,它花的 token 和金额写在 approval.inquiry 事件数据的 usage 里
(input_tokens、output_tokens、cost_usd、calls,可带 unpriced_calls)。以前运行用量只认图的状态和 llm.end,
质询花的钱在运行页上看不见。
- 运行结束(成功、失败)时:图的用量照旧(成功取状态、其余两本账取大),再加上质询的;明细另放在 inquiry 下
- 停在审批时被放弃:停下时记的用量里还没有这次停下期间的质询,放弃时补上
- 停过两次、两次都问过:每次质询只计一遍
- 没有质询的运行,用量和以前逐字相同
"""
from __future__ import annotations
import asyncio
import pytest
from sqlalchemy import select
from app.core.events import EventType
from app.db.base import SessionLocal
from app.db.models import Run, RunEvent
from app.engine import compiler
from app.engine.context import NodeError
from app.engine.runner import run_manager
from app.engine.schema import NodeType
A = {"input_tokens": 1200, "output_tokens": 300, "cost_usd": 0.0123, "calls": 2}
B = {"input_tokens": 800, "output_tokens": 100, "cost_usd": 0.004, "calls": 1, "unpriced_calls": 1}
@pytest.fixture(autouse=True)
async def engine_up():
await run_manager.setup()
yield
await run_manager.shutdown()
def node(nid, ntype, **config):
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}}
def chain(*nodes):
return {"nodes": list(nodes), "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]}
def gated(*middle, gates: int = 1) -> dict:
nodes = [node("start", "input"), *middle]
nodes += [node(f"gate{i}", "human", mode="approve", title=f"第 {i + 1} 道审批") for i in range(gates)]
nodes.append(node("out", "output", fields=[{"name": "r", "value": "完成"}]))
return chain(*nodes)
def ask(question: str, usage: dict | None) -> tuple:
data = {"question": question, "answer": "按订单表核对一致"}
if usage is not None:
data["usage"] = usage
return (EventType.APPROVAL_INQUIRY, None, data)
async def _row(run_id: str) -> Run:
async with SessionLocal() as session:
return await session.get(Run, run_id)
async def _until(run_id: str, statuses: tuple[str, ...]) -> Run:
for _ in range(500):
row = await _row(run_id)
if row.status in statuses and (row.status == "interrupted" or row.manifest_seq is not None):
await run_manager.wait_idle(run_id)
return await _row(run_id)
await asyncio.sleep(0.03)
raise AssertionError(f"运行 {run_id} 停在 {row.status}")
async def _ask_while_paused(run_id: str, *items: tuple) -> None:
from app.engine.run_append import append_events
assert await append_events(run_id, list(items), require="interrupted"), "质询没写进运行"
def _tokens(usage: dict) -> tuple:
return (usage.get("input_tokens"), usage.get("output_tokens"), usage.get("total_tokens"),
usage.get("calls"), usage.get("unpriced_calls"))
# --------------------------------------------------------------------------
# 按事件加总
# --------------------------------------------------------------------------
async def test_usage_of_events_keeps_inquiries_apart():
"""llm.end 照旧逐次加总;质询的另记在 inquiry 下(_finalize 比完两本账再加,见下面几条)。"""
from app.engine.runner import _usage_of_events
async with SessionLocal() as session:
run = Run(workflow_name="用量加总", status="succeeded", graph={}, input={})
session.add(run)
await session.flush()
rows = [
("llm.end", {"model": "claude-sonnet-4-5", "input_tokens": 100, "output_tokens": 20, "cost_usd": 0.001,
"calls": 1}),
("llm.end", {"model": "claude-sonnet-4-5", "input_tokens": 50, "output_tokens": 5, "cost_usd": 0.0005}),
("approval.inquiry", {"question": "甲", "usage": A}),
("approval.inquiry", {"question": "乙", "usage": B}),
("approval.inquiry", {"question": "没问模型就答了"}),
("approval.inquiry", {"question": "写坏的用量", "usage": "很多"}),
# 别的事件里的金额不算:按需裁判的判定另有自己的账
("evidence.judged", {"cost_usd": 9.9, "calls": 9}),
]
for seq, (kind, data) in enumerate(rows, start=1):
session.add(RunEvent(run_id=run.id, seq=seq, type=kind, node_id=None, ts=float(seq), data=data))
await session.commit()
got = await _usage_of_events(session, run.id)
assert {k: got[k] for k in ("input_tokens", "output_tokens", "calls")} == {
"input_tokens": 150, "output_tokens": 25, "calls": 2}
assert got["cost_usd"] == pytest.approx(0.0015)
assert got["inquiry"] == {"input_tokens": 2000, "output_tokens": 400, "cost_usd": pytest.approx(0.0163),
"calls": 3, "unpriced_calls": 1}
async def test_without_inquiries_there_is_no_inquiry_key():
from app.engine.runner import _usage_of_events
async with SessionLocal() as session:
run = Run(workflow_name="没有质询", status="succeeded", graph={}, input={})
session.add(run)
await session.flush()
session.add(RunEvent(run_id=run.id, seq=1, type="llm.end", node_id="ask", ts=1.0,
data={"model": "claude-sonnet-4-5", "input_tokens": 10, "output_tokens": 2,
"cost_usd": 0.0001, "calls": 1}))
await session.commit()
got = await _usage_of_events(session, run.id)
assert set(got) == {"input_tokens", "output_tokens", "cost_usd", "calls"}
# --------------------------------------------------------------------------
# 运行用量
# --------------------------------------------------------------------------
async def test_a_succeeded_run_counts_the_inquiry():
run = await run_manager.start(graph=gated(), input_payload={})
await _until(run.id, ("interrupted",))
await _ask_while_paused(run.id, ask("销售额为什么涨了", A))
await run_manager.resume(run.id, {"approved": True})
row = await _until(run.id, ("succeeded",))
# 图里没有模型调用,用量全是这次质询的
assert _tokens(row.usage) == (1200, 300, 1500, 2, None)
assert row.usage["cost_usd"] == pytest.approx(0.0123)
assert row.usage["inquiry"] == A
assert row.usage["wall_ms"] >= row.usage["active_ms"]
async def test_the_inquiry_is_added_on_top_of_the_model_calls():
"""图里有模型调用:成功的运行取状态里的用量,再加上质询的。"""
graph = gated(node("ask", "llm", prompt="上周到店多少人"))
run = await run_manager.start(graph=graph, input_payload={})
await _until(run.id, ("interrupted",))
await _ask_while_paused(run.id, ask("到店人数怎么算的", B))
await run_manager.resume(run.id, {"approved": True})
row = await _until(run.id, ("succeeded",))
async with SessionLocal() as session:
ends = [e.data for e in (await session.execute(select(RunEvent).where(
RunEvent.run_id == run.id, RunEvent.type == "llm.end"))).scalars()]
assert ends, "图里的模型调用没有记 llm.end"
model_in = sum(e["input_tokens"] for e in ends)
model_out = sum(e["output_tokens"] for e in ends)
unpriced = sum(1 for e in ends if e.get("unpriced"))
assert row.usage["input_tokens"] == model_in + B["input_tokens"]
assert row.usage["output_tokens"] == model_out + B["output_tokens"]
assert row.usage["total_tokens"] == model_in + model_out + B["input_tokens"] + B["output_tokens"]
assert row.usage["calls"] == len(ends) + B["calls"]
assert row.usage["unpriced_calls"] == unpriced + B["unpriced_calls"]
assert row.usage["cost_usd"] == pytest.approx(sum(e["cost_usd"] for e in ends) + B["cost_usd"])
async def test_a_failed_run_counts_the_inquiry_too(monkeypatch):
original = compiler.RUNNERS[NodeType.TRANSFORM]
async def boom(state, ctx):
if ctx.node.id == "boom":
raise NodeError(ctx.node.id, "数据源暂时不可用")
return await original(state, ctx)
monkeypatch.setitem(compiler.RUNNERS, NodeType.TRANSFORM, boom)
graph = chain(node("start", "input"), node("gate", "human", mode="approve", title="放行吗"),
node("boom", "transform", mode="template", template="x"),
node("out", "output", fields=[{"name": "r", "value": "x"}]))
run = await run_manager.start(graph=graph, input_payload={})
await _until(run.id, ("interrupted",))
await _ask_while_paused(run.id, ask("口径对吗", A))
await run_manager.resume(run.id, {"approved": True})
row = await _until(run.id, ("failed",))
assert _tokens(row.usage) == (1200, 300, 1500, 2, None)
assert row.usage["inquiry"] == A
async def test_each_inquiry_counts_once_across_two_pauses():
run = await run_manager.start(graph=gated(gates=2), input_payload={})
await _until(run.id, ("interrupted",))
await _ask_while_paused(run.id, ask("第一问", A))
await run_manager.resume(run.id, {"approved": True})
await asyncio.sleep(0.05)
paused = await _until(run.id, ("interrupted",))
# 第二次停下时记的用量已经有第一问
assert paused.usage["input_tokens"] == A["input_tokens"] and paused.usage["inquiry"] == A
await _ask_while_paused(run.id, ask("第二问", B))
await run_manager.resume(run.id, {"approved": True})
row = await _until(run.id, ("succeeded",))
assert _tokens(row.usage) == (2000, 400, 2400, 3, 1)
assert row.usage["cost_usd"] == pytest.approx(0.0163)
assert row.usage["inquiry"] == {"input_tokens": 2000, "output_tokens": 400, "cost_usd": pytest.approx(0.0163),
"calls": 3, "unpriced_calls": 1}
@pytest.mark.parametrize("asked_before", [False, True], ids=["first-pause", "second-pause"])
async def test_abandoning_a_paused_run_counts_what_was_asked(asked_before):
"""停在审批时放弃:停下时记的用量里还没有这次停下期间的质询,放弃时补上;上一次停下时已经计过的不再计。"""
run = await run_manager.start(graph=gated(gates=2 if asked_before else 1), input_payload={})
await _until(run.id, ("interrupted",))
if asked_before:
await _ask_while_paused(run.id, ask("第一问", A))
await run_manager.resume(run.id, {"approved": True})
await asyncio.sleep(0.05)
await _until(run.id, ("interrupted",))
await _ask_while_paused(run.id, ask("最后一问", B))
assert await run_manager.cancel(run.id) == "cancelled"
row = await _until(run.id, ("cancelled",))
if asked_before:
assert _tokens(row.usage) == (2000, 400, 2400, 3, 1)
assert row.usage["cost_usd"] == pytest.approx(0.0163)
else:
assert _tokens(row.usage) == (800, 100, 900, 1, 1) and row.usage["inquiry"] == B
assert row.usage["wall_ms"] >= row.usage["active_ms"]
async def test_runs_without_inquiries_keep_their_usage():
run = await run_manager.start(graph=gated(), input_payload={})
await _until(run.id, ("interrupted",))
await run_manager.resume(run.id, {"approved": True})
row = await _until(run.id, ("succeeded",))
assert set(row.usage) == {"duration_ms", "wall_ms", "active_ms", "wait_ms"}
other = await run_manager.start(graph=gated(), input_payload={})
await _until(other.id, ("interrupted",))
assert await run_manager.cancel(other.id) == "cancelled"
row = await _until(other.id, ("cancelled",))
assert set(row.usage) == {"duration_ms", "wall_ms", "active_ms", "wait_ms"}