mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-18 07:22:00 +00:00
审批质询的用量计进运行用量:质询不在图里执行、不发 llm.end,花的 token 和金额写在 approval.inquiry 事件数据的 usage 里(input_tokens、output_tokens、cost_usd、calls,可带 unpriced_calls),以前运行用量只认图的状态和 llm.end,质询花的钱在运行页上看不见。_usage_of_events 把质询的用量单独加总放在 inquiry 下:llm.end 这本账要和状态那本比大小,质询花的两本里都没有,混进去比会被状态那本盖掉。_finalize 照旧算出图的用量(成功取状态,其余两本取大)后再加上质询的,明细另记在运行用量的 inquiry 下;停在审批时被放弃的运行,放弃时补上这次停下期间的质询。停过两次、两次都问过的,按上次记下的 inquiry 先减后加,每次质询只计一遍。没有质询的运行用量和以前逐字相同
This commit is contained in:
@@ -551,7 +551,12 @@ class RunManager:
|
||||
closed = await _close_approvals(session, run_id, actor, now)
|
||||
run = await session.get(Run, run_id)
|
||||
timing = await _timing(session, run, 0, now)
|
||||
run.usage = {**(run.usage or {}), "duration_ms": timing["active_ms"], **timing}
|
||||
usage = dict(run.usage or {})
|
||||
# 停下时记的用量里还没有这次停在审批期间问的质询,这里补上(上次停下时计过的不再计)
|
||||
inquiry = (await _usage_of_events(session, run_id)).get("inquiry")
|
||||
if inquiry or usage.get("inquiry"):
|
||||
usage = _with_inquiry(usage, inquiry)
|
||||
run.usage = {**usage, "duration_ms": timing["active_ms"], **timing}
|
||||
run.error = "用户取消:等待审批时放弃了这次运行" if closed else "用户取消:挂起期间放弃了这次运行"
|
||||
bus.set_seq(run_id, run.last_seq or 0)
|
||||
# 和状态同一个事务提交,理由同 _finalize
|
||||
@@ -857,6 +862,7 @@ class RunManager:
|
||||
if run:
|
||||
timing = await _timing(session, run, elapsed_ms, now)
|
||||
events_usage = await _usage_of_events(session, run_id)
|
||||
inquiry = events_usage.pop("inquiry", None)
|
||||
if status != "succeeded":
|
||||
# 状态里的用量只算跑完了的节点。失败、取消、停在审批上的那个节点已经
|
||||
# 花出去的调用(校验修复、agent 的前几步)只在 llm.end 里——以前这部分
|
||||
@@ -865,6 +871,9 @@ class RunManager:
|
||||
elif events_usage.get("unpriced_calls"):
|
||||
# 金额估不出来的调用只有 llm.end 数得出(状态里的用量只有金额,0 分不出是免费还是没价)
|
||||
usage = {**usage, "unpriced_calls": events_usage["unpriced_calls"]}
|
||||
if inquiry:
|
||||
# 审批质询不在图里执行,状态和 llm.end 两本账里都没有它:比完再加上
|
||||
usage = _with_inquiry(usage, inquiry)
|
||||
# duration_ms 是各段执行时长之和(= active_ms)。以前用这一段的耗时
|
||||
# 覆盖它:87 秒、中间等过一次审批的运行,列表上写着 22ms
|
||||
usage.update(duration_ms=timing["active_ms"], **timing)
|
||||
@@ -1270,18 +1279,35 @@ async def _close_approvals(session: Any, run_id: str, actor: str | None, now: da
|
||||
|
||||
|
||||
_USAGE_KEYS = ("input_tokens", "output_tokens", "cost_usd", "calls")
|
||||
#: 审批质询的用量里计进运行用量的键:unpriced_calls 是其中金额估不出来的调用次数(模型不在价格目录里)
|
||||
_INQUIRY_KEYS = (*_USAGE_KEYS, "unpriced_calls")
|
||||
|
||||
|
||||
async def _usage_of_events(session: Any, run_id: str) -> dict[str, Any]:
|
||||
"""按 llm.end 事件逐次加总的用量:每次模型调用都有一条,不管节点最后成没成。"""
|
||||
"""按事件加总的用量。
|
||||
|
||||
llm.end 逐次加总:每次模型调用都有一条,不管节点最后成没成。审批质询不在图里执行、不发 llm.end,
|
||||
它的用量写在 approval.inquiry 事件数据的 usage 里,单独加总放在 inquiry 下(没有质询就没有这个键):
|
||||
llm.end 这本账要和状态里的用量比大小(_spent),质询花的两本里都没有,混进去比会被状态那本盖掉。
|
||||
"""
|
||||
rows = await session.execute(
|
||||
select(RunEvent.data).where(RunEvent.run_id == run_id, RunEvent.type == EventType.LLM_END)
|
||||
select(RunEvent.type, RunEvent.data).where(
|
||||
RunEvent.run_id == run_id, RunEvent.type.in_([EventType.LLM_END, EventType.APPROVAL_INQUIRY]))
|
||||
)
|
||||
from app.providers.catalog import unpriced_call
|
||||
|
||||
total: dict[str, Any] = dict.fromkeys(_USAGE_KEYS, 0)
|
||||
unpriced = 0
|
||||
for (data,) in rows:
|
||||
inquiry: dict[str, Any] = {}
|
||||
for kind, data in rows:
|
||||
if kind == EventType.APPROVAL_INQUIRY:
|
||||
spent = (data or {}).get("usage")
|
||||
if isinstance(spent, dict):
|
||||
for key in _INQUIRY_KEYS:
|
||||
value = spent.get(key)
|
||||
if isinstance(value, (int, float)) and not isinstance(value, bool):
|
||||
inquiry[key] = inquiry.get(key, 0) + value
|
||||
continue
|
||||
for key in _USAGE_KEYS:
|
||||
value = (data or {}).get(key, 1 if key == "calls" else 0)
|
||||
if isinstance(value, (int, float)) and not isinstance(value, bool):
|
||||
@@ -1291,9 +1317,34 @@ async def _usage_of_events(session: Any, run_id: str) -> dict[str, Any]:
|
||||
if unpriced:
|
||||
# 有几次调用的金额估不出来(模型不在价格目录里):cost_usd 只是估得出的那部分,界面据此不写 $0
|
||||
total["unpriced_calls"] = unpriced
|
||||
if inquiry:
|
||||
if "cost_usd" in inquiry:
|
||||
inquiry["cost_usd"] = round(inquiry["cost_usd"], 6)
|
||||
total["inquiry"] = inquiry
|
||||
return total
|
||||
|
||||
|
||||
def _with_inquiry(usage: dict[str, Any], inquiry: dict[str, Any] | None) -> dict[str, Any]:
|
||||
"""把审批质询的用量计进运行用量,明细另放在 inquiry 下(界面要说清其中多少是质询花的)。
|
||||
|
||||
usage 里已经计过一回的(上次停下时记下的 inquiry)先减掉再加:停过两次、两次都问过的运行,
|
||||
每次质询只计一遍。
|
||||
"""
|
||||
out = dict(usage)
|
||||
counted = out.pop("inquiry", None) or {}
|
||||
for key in _INQUIRY_KEYS:
|
||||
delta = (inquiry or {}).get(key, 0) - counted.get(key, 0)
|
||||
if delta:
|
||||
out[key] = (out.get(key) or 0) + delta
|
||||
if isinstance(out.get("cost_usd"), float):
|
||||
out["cost_usd"] = round(out["cost_usd"], 6)
|
||||
if "input_tokens" in out or "output_tokens" in out:
|
||||
out["total_tokens"] = int(out.get("input_tokens") or 0) + int(out.get("output_tokens") or 0)
|
||||
if inquiry:
|
||||
out["inquiry"] = dict(inquiry)
|
||||
return out
|
||||
|
||||
|
||||
def _spent(state_usage: dict[str, Any], events_usage: dict[str, Any]) -> dict[str, Any]:
|
||||
"""两本账取大的那本。新引擎每次调用都发 llm.end,事件那本不会更少;
|
||||
加这条事件之前的老运行(agent 节点不发 llm.end)状态那本更全。"""
|
||||
|
||||
@@ -0,0 +1,248 @@
|
||||
"""审批质询的用量计进运行用量。
|
||||
|
||||
质询不在图里执行、不发 llm.end,它花的 token 和金额写在 approval.inquiry 事件数据的 usage 里
|
||||
(input_tokens、output_tokens、cost_usd、calls,可带 unpriced_calls)。以前运行用量只认图的状态和 llm.end,
|
||||
质询花的钱在运行页上看不见。
|
||||
|
||||
- 运行结束(成功、失败)时:图的用量照旧(成功取状态、其余两本账取大),再加上质询的;明细另放在 inquiry 下
|
||||
- 停在审批时被放弃:停下时记的用量里还没有这次停下期间的质询,放弃时补上
|
||||
- 停过两次、两次都问过:每次质询只计一遍
|
||||
- 没有质询的运行,用量和以前逐字相同
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.core.events import EventType
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import Run, RunEvent
|
||||
from app.engine import compiler
|
||||
from app.engine.context import NodeError
|
||||
from app.engine.runner import run_manager
|
||||
from app.engine.schema import NodeType
|
||||
|
||||
A = {"input_tokens": 1200, "output_tokens": 300, "cost_usd": 0.0123, "calls": 2}
|
||||
B = {"input_tokens": 800, "output_tokens": 100, "cost_usd": 0.004, "calls": 1, "unpriced_calls": 1}
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
async def engine_up():
|
||||
await run_manager.setup()
|
||||
yield
|
||||
await run_manager.shutdown()
|
||||
|
||||
|
||||
def node(nid, ntype, **config):
|
||||
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}}
|
||||
|
||||
|
||||
def chain(*nodes):
|
||||
return {"nodes": list(nodes), "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]}
|
||||
|
||||
|
||||
def gated(*middle, gates: int = 1) -> dict:
|
||||
nodes = [node("start", "input"), *middle]
|
||||
nodes += [node(f"gate{i}", "human", mode="approve", title=f"第 {i + 1} 道审批") for i in range(gates)]
|
||||
nodes.append(node("out", "output", fields=[{"name": "r", "value": "完成"}]))
|
||||
return chain(*nodes)
|
||||
|
||||
|
||||
def ask(question: str, usage: dict | None) -> tuple:
|
||||
data = {"question": question, "answer": "按订单表核对一致"}
|
||||
if usage is not None:
|
||||
data["usage"] = usage
|
||||
return (EventType.APPROVAL_INQUIRY, None, data)
|
||||
|
||||
|
||||
async def _row(run_id: str) -> Run:
|
||||
async with SessionLocal() as session:
|
||||
return await session.get(Run, run_id)
|
||||
|
||||
|
||||
async def _until(run_id: str, statuses: tuple[str, ...]) -> Run:
|
||||
for _ in range(500):
|
||||
row = await _row(run_id)
|
||||
if row.status in statuses and (row.status == "interrupted" or row.manifest_seq is not None):
|
||||
await run_manager.wait_idle(run_id)
|
||||
return await _row(run_id)
|
||||
await asyncio.sleep(0.03)
|
||||
raise AssertionError(f"运行 {run_id} 停在 {row.status}")
|
||||
|
||||
|
||||
async def _ask_while_paused(run_id: str, *items: tuple) -> None:
|
||||
from app.engine.run_append import append_events
|
||||
|
||||
assert await append_events(run_id, list(items), require="interrupted"), "质询没写进运行"
|
||||
|
||||
|
||||
def _tokens(usage: dict) -> tuple:
|
||||
return (usage.get("input_tokens"), usage.get("output_tokens"), usage.get("total_tokens"),
|
||||
usage.get("calls"), usage.get("unpriced_calls"))
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 按事件加总
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_usage_of_events_keeps_inquiries_apart():
|
||||
"""llm.end 照旧逐次加总;质询的另记在 inquiry 下(_finalize 比完两本账再加,见下面几条)。"""
|
||||
from app.engine.runner import _usage_of_events
|
||||
|
||||
async with SessionLocal() as session:
|
||||
run = Run(workflow_name="用量加总", status="succeeded", graph={}, input={})
|
||||
session.add(run)
|
||||
await session.flush()
|
||||
rows = [
|
||||
("llm.end", {"model": "claude-sonnet-4-5", "input_tokens": 100, "output_tokens": 20, "cost_usd": 0.001,
|
||||
"calls": 1}),
|
||||
("llm.end", {"model": "claude-sonnet-4-5", "input_tokens": 50, "output_tokens": 5, "cost_usd": 0.0005}),
|
||||
("approval.inquiry", {"question": "甲", "usage": A}),
|
||||
("approval.inquiry", {"question": "乙", "usage": B}),
|
||||
("approval.inquiry", {"question": "没问模型就答了"}),
|
||||
("approval.inquiry", {"question": "写坏的用量", "usage": "很多"}),
|
||||
# 别的事件里的金额不算:按需裁判的判定另有自己的账
|
||||
("evidence.judged", {"cost_usd": 9.9, "calls": 9}),
|
||||
]
|
||||
for seq, (kind, data) in enumerate(rows, start=1):
|
||||
session.add(RunEvent(run_id=run.id, seq=seq, type=kind, node_id=None, ts=float(seq), data=data))
|
||||
await session.commit()
|
||||
got = await _usage_of_events(session, run.id)
|
||||
assert {k: got[k] for k in ("input_tokens", "output_tokens", "calls")} == {
|
||||
"input_tokens": 150, "output_tokens": 25, "calls": 2}
|
||||
assert got["cost_usd"] == pytest.approx(0.0015)
|
||||
assert got["inquiry"] == {"input_tokens": 2000, "output_tokens": 400, "cost_usd": pytest.approx(0.0163),
|
||||
"calls": 3, "unpriced_calls": 1}
|
||||
|
||||
|
||||
async def test_without_inquiries_there_is_no_inquiry_key():
|
||||
from app.engine.runner import _usage_of_events
|
||||
|
||||
async with SessionLocal() as session:
|
||||
run = Run(workflow_name="没有质询", status="succeeded", graph={}, input={})
|
||||
session.add(run)
|
||||
await session.flush()
|
||||
session.add(RunEvent(run_id=run.id, seq=1, type="llm.end", node_id="ask", ts=1.0,
|
||||
data={"model": "claude-sonnet-4-5", "input_tokens": 10, "output_tokens": 2,
|
||||
"cost_usd": 0.0001, "calls": 1}))
|
||||
await session.commit()
|
||||
got = await _usage_of_events(session, run.id)
|
||||
assert set(got) == {"input_tokens", "output_tokens", "cost_usd", "calls"}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 运行用量
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_a_succeeded_run_counts_the_inquiry():
|
||||
run = await run_manager.start(graph=gated(), input_payload={})
|
||||
await _until(run.id, ("interrupted",))
|
||||
await _ask_while_paused(run.id, ask("销售额为什么涨了", A))
|
||||
await run_manager.resume(run.id, {"approved": True})
|
||||
row = await _until(run.id, ("succeeded",))
|
||||
# 图里没有模型调用,用量全是这次质询的
|
||||
assert _tokens(row.usage) == (1200, 300, 1500, 2, None)
|
||||
assert row.usage["cost_usd"] == pytest.approx(0.0123)
|
||||
assert row.usage["inquiry"] == A
|
||||
assert row.usage["wall_ms"] >= row.usage["active_ms"]
|
||||
|
||||
|
||||
async def test_the_inquiry_is_added_on_top_of_the_model_calls():
|
||||
"""图里有模型调用:成功的运行取状态里的用量,再加上质询的。"""
|
||||
graph = gated(node("ask", "llm", prompt="上周到店多少人"))
|
||||
run = await run_manager.start(graph=graph, input_payload={})
|
||||
await _until(run.id, ("interrupted",))
|
||||
await _ask_while_paused(run.id, ask("到店人数怎么算的", B))
|
||||
await run_manager.resume(run.id, {"approved": True})
|
||||
row = await _until(run.id, ("succeeded",))
|
||||
async with SessionLocal() as session:
|
||||
ends = [e.data for e in (await session.execute(select(RunEvent).where(
|
||||
RunEvent.run_id == run.id, RunEvent.type == "llm.end"))).scalars()]
|
||||
assert ends, "图里的模型调用没有记 llm.end"
|
||||
model_in = sum(e["input_tokens"] for e in ends)
|
||||
model_out = sum(e["output_tokens"] for e in ends)
|
||||
unpriced = sum(1 for e in ends if e.get("unpriced"))
|
||||
assert row.usage["input_tokens"] == model_in + B["input_tokens"]
|
||||
assert row.usage["output_tokens"] == model_out + B["output_tokens"]
|
||||
assert row.usage["total_tokens"] == model_in + model_out + B["input_tokens"] + B["output_tokens"]
|
||||
assert row.usage["calls"] == len(ends) + B["calls"]
|
||||
assert row.usage["unpriced_calls"] == unpriced + B["unpriced_calls"]
|
||||
assert row.usage["cost_usd"] == pytest.approx(sum(e["cost_usd"] for e in ends) + B["cost_usd"])
|
||||
|
||||
|
||||
async def test_a_failed_run_counts_the_inquiry_too(monkeypatch):
|
||||
original = compiler.RUNNERS[NodeType.TRANSFORM]
|
||||
|
||||
async def boom(state, ctx):
|
||||
if ctx.node.id == "boom":
|
||||
raise NodeError(ctx.node.id, "数据源暂时不可用")
|
||||
return await original(state, ctx)
|
||||
|
||||
monkeypatch.setitem(compiler.RUNNERS, NodeType.TRANSFORM, boom)
|
||||
graph = chain(node("start", "input"), node("gate", "human", mode="approve", title="放行吗"),
|
||||
node("boom", "transform", mode="template", template="x"),
|
||||
node("out", "output", fields=[{"name": "r", "value": "x"}]))
|
||||
run = await run_manager.start(graph=graph, input_payload={})
|
||||
await _until(run.id, ("interrupted",))
|
||||
await _ask_while_paused(run.id, ask("口径对吗", A))
|
||||
await run_manager.resume(run.id, {"approved": True})
|
||||
row = await _until(run.id, ("failed",))
|
||||
assert _tokens(row.usage) == (1200, 300, 1500, 2, None)
|
||||
assert row.usage["inquiry"] == A
|
||||
|
||||
|
||||
async def test_each_inquiry_counts_once_across_two_pauses():
|
||||
run = await run_manager.start(graph=gated(gates=2), input_payload={})
|
||||
await _until(run.id, ("interrupted",))
|
||||
await _ask_while_paused(run.id, ask("第一问", A))
|
||||
await run_manager.resume(run.id, {"approved": True})
|
||||
await asyncio.sleep(0.05)
|
||||
paused = await _until(run.id, ("interrupted",))
|
||||
# 第二次停下时记的用量已经有第一问
|
||||
assert paused.usage["input_tokens"] == A["input_tokens"] and paused.usage["inquiry"] == A
|
||||
await _ask_while_paused(run.id, ask("第二问", B))
|
||||
await run_manager.resume(run.id, {"approved": True})
|
||||
row = await _until(run.id, ("succeeded",))
|
||||
assert _tokens(row.usage) == (2000, 400, 2400, 3, 1)
|
||||
assert row.usage["cost_usd"] == pytest.approx(0.0163)
|
||||
assert row.usage["inquiry"] == {"input_tokens": 2000, "output_tokens": 400, "cost_usd": pytest.approx(0.0163),
|
||||
"calls": 3, "unpriced_calls": 1}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("asked_before", [False, True], ids=["first-pause", "second-pause"])
|
||||
async def test_abandoning_a_paused_run_counts_what_was_asked(asked_before):
|
||||
"""停在审批时放弃:停下时记的用量里还没有这次停下期间的质询,放弃时补上;上一次停下时已经计过的不再计。"""
|
||||
run = await run_manager.start(graph=gated(gates=2 if asked_before else 1), input_payload={})
|
||||
await _until(run.id, ("interrupted",))
|
||||
if asked_before:
|
||||
await _ask_while_paused(run.id, ask("第一问", A))
|
||||
await run_manager.resume(run.id, {"approved": True})
|
||||
await asyncio.sleep(0.05)
|
||||
await _until(run.id, ("interrupted",))
|
||||
await _ask_while_paused(run.id, ask("最后一问", B))
|
||||
assert await run_manager.cancel(run.id) == "cancelled"
|
||||
row = await _until(run.id, ("cancelled",))
|
||||
if asked_before:
|
||||
assert _tokens(row.usage) == (2000, 400, 2400, 3, 1)
|
||||
assert row.usage["cost_usd"] == pytest.approx(0.0163)
|
||||
else:
|
||||
assert _tokens(row.usage) == (800, 100, 900, 1, 1) and row.usage["inquiry"] == B
|
||||
assert row.usage["wall_ms"] >= row.usage["active_ms"]
|
||||
|
||||
|
||||
async def test_runs_without_inquiries_keep_their_usage():
|
||||
run = await run_manager.start(graph=gated(), input_payload={})
|
||||
await _until(run.id, ("interrupted",))
|
||||
await run_manager.resume(run.id, {"approved": True})
|
||||
row = await _until(run.id, ("succeeded",))
|
||||
assert set(row.usage) == {"duration_ms", "wall_ms", "active_ms", "wait_ms"}
|
||||
|
||||
other = await run_manager.start(graph=gated(), input_payload={})
|
||||
await _until(other.id, ("interrupted",))
|
||||
assert await run_manager.cancel(other.id) == "cancelled"
|
||||
row = await _until(other.id, ("cancelled",))
|
||||
assert set(row.usage) == {"duration_ms", "wall_ms", "active_ms", "wait_ms"}
|
||||
Reference in New Issue
Block a user