mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-19 06:41:59 +00:00
模型不在价格目录里时运行金额写「金额无法估算」,不写 $0:真实运行里 token 记到了上百万,金额却是 $0,看着像没花钱。llm.end 发出时,用了 token、模型却不在价格目录里的标 unpriced(目录里标 0 价的本地模型不算);运行用量按 llm.end 数出 unpriced_calls,成功的运行也补上(状态里的用量分不出 0 是免费还是没价)。运行详情、画布运行面板、航迹的金额:一分都没估出来写「金额无法估算」,估出一部分写「≥ $x」,悬停说几次调用不在价格目录中。check-trace 补断言;流式用量测试和注释里的真实调用次数换成概括说法
This commit is contained in:
@@ -56,6 +56,12 @@ class NodeContext:
|
||||
writer = get_stream_writer()
|
||||
except Exception: # noqa: BLE001 - 不在图执行上下文里(比如单元测试)
|
||||
return
|
||||
if str(event_type) == EventType.LLM_END and "unpriced" not in data:
|
||||
from app.providers.catalog import unpriced_call
|
||||
|
||||
if unpriced_call(data):
|
||||
# 金额估不出来(模型不在价格目录里):标出来,界面不把它的 0 当成「没花钱」
|
||||
data = {**data, "unpriced": True}
|
||||
writer(
|
||||
{
|
||||
"__agentlab__": True,
|
||||
|
||||
@@ -837,11 +837,15 @@ class RunManager:
|
||||
run = await session.get(Run, run_id)
|
||||
if run:
|
||||
timing = await _timing(session, run, elapsed_ms, now)
|
||||
events_usage = await _usage_of_events(session, run_id)
|
||||
if status != "succeeded":
|
||||
# 状态里的用量只算跑完了的节点。失败、取消、停在审批上的那个节点已经
|
||||
# 花出去的调用(校验修复、agent 的前几步)只在 llm.end 里——以前这部分
|
||||
# 在运行记录上凭空消失,失败的运行看着像没花钱
|
||||
usage = _spent(usage, await _usage_of_events(session, run_id))
|
||||
usage = _spent(usage, events_usage)
|
||||
elif events_usage.get("unpriced_calls"):
|
||||
# 金额估不出来的调用只有 llm.end 数得出(状态里的用量只有金额,0 分不出是免费还是没价)
|
||||
usage = {**usage, "unpriced_calls": events_usage["unpriced_calls"]}
|
||||
# duration_ms 是各段执行时长之和(= active_ms)。以前用这一段的耗时
|
||||
# 覆盖它:87 秒、中间等过一次审批的运行,列表上写着 22ms
|
||||
usage.update(duration_ms=timing["active_ms"], **timing)
|
||||
@@ -1251,13 +1255,20 @@ async def _usage_of_events(session: Any, run_id: str) -> dict[str, Any]:
|
||||
rows = await session.execute(
|
||||
select(RunEvent.data).where(RunEvent.run_id == run_id, RunEvent.type == EventType.LLM_END)
|
||||
)
|
||||
from app.providers.catalog import unpriced_call
|
||||
|
||||
total: dict[str, Any] = dict.fromkeys(_USAGE_KEYS, 0)
|
||||
unpriced = 0
|
||||
for (data,) in rows:
|
||||
for key in _USAGE_KEYS:
|
||||
value = (data or {}).get(key, 1 if key == "calls" else 0)
|
||||
if isinstance(value, (int, float)) and not isinstance(value, bool):
|
||||
total[key] += value
|
||||
unpriced += unpriced_call(data)
|
||||
total["cost_usd"] = round(total["cost_usd"], 6)
|
||||
if unpriced:
|
||||
# 有几次调用的金额估不出来(模型不在价格目录里):cost_usd 只是估得出的那部分,界面据此不写 $0
|
||||
total["unpriced_calls"] = unpriced
|
||||
return total
|
||||
|
||||
|
||||
@@ -1269,6 +1280,8 @@ def _spent(state_usage: dict[str, Any], events_usage: dict[str, Any]) -> dict[st
|
||||
mine, theirs = out.get(key) or 0, events_usage.get(key) or 0
|
||||
if isinstance(mine, (int, float)) and theirs > mine:
|
||||
out[key] = theirs
|
||||
if events_usage.get("unpriced_calls"):
|
||||
out["unpriced_calls"] = events_usage["unpriced_calls"]
|
||||
if "input_tokens" in out or "output_tokens" in out:
|
||||
out["total_tokens"] = int(out.get("input_tokens") or 0) + int(out.get("output_tokens") or 0)
|
||||
return out
|
||||
|
||||
@@ -229,6 +229,22 @@ def find_context(model: str) -> int | None:
|
||||
return None
|
||||
|
||||
|
||||
def unpriced_call(data: Any) -> bool:
|
||||
"""一次模型调用(llm.end 的数据)的金额估不出来:用了 token,模型却不在价格目录里,estimate_cost 只能给 0。
|
||||
|
||||
这时写 $0 像是没花钱(真实运行里上百次调用、上百万 token,运行页写着 $0)。本地模型在目录里标的是 0 价,
|
||||
不算估不出来。事件自己标了 unpriced 的以它为准。"""
|
||||
if not isinstance(data, dict):
|
||||
return False
|
||||
if isinstance(data.get("unpriced"), bool):
|
||||
return data["unpriced"]
|
||||
tokens = sum(v for v in (data.get("input_tokens"), data.get("output_tokens"))
|
||||
if isinstance(v, (int, float)) and not isinstance(v, bool))
|
||||
model = data.get("model")
|
||||
return tokens > 0 and isinstance(model, str) and bool(model) and not data.get("cost_usd") \
|
||||
and find_pricing(model) is None
|
||||
|
||||
|
||||
def estimate_cost(model: str, input_tokens: int, output_tokens: int) -> float:
|
||||
"""按目录价估算一次调用的花费(美元)。"""
|
||||
pricing = find_pricing(model)
|
||||
|
||||
@@ -171,9 +171,8 @@ async def engine_up():
|
||||
await run_manager.shutdown()
|
||||
|
||||
|
||||
async def test_run_usage_counts_streamed_tokens(server, engine_up):
|
||||
"""整条链:兼容接入 → llm 节点流式调用 → llm.end → 运行的用量。修之前这里是 0 token、0 美元。"""
|
||||
server()
|
||||
async def _run_once(model: str) -> tuple[Run, list[RunEvent]]:
|
||||
"""兼容接入上用 model 跑一个 llm 节点,返回运行行和 llm.end 事件。"""
|
||||
provider = _provider()
|
||||
async with SessionLocal() as session:
|
||||
session.add(provider)
|
||||
@@ -185,7 +184,7 @@ async def test_run_usage_counts_streamed_tokens(server, engine_up):
|
||||
"nodes": [
|
||||
{"id": "in", "type": "input", "data": {"config": {}}},
|
||||
{"id": "ask", "type": "llm", "data": {"label": "问答", "config": {
|
||||
"prompt": "上周到店多少人", "provider": provider.name, "model": PRICED}}},
|
||||
"prompt": "上周到店多少人", "provider": provider.name, "model": model}}},
|
||||
{"id": "out", "type": "output", "data": {"config": {
|
||||
"fields": [{"name": "回答", "value": "{{ nodes.ask.text }}"}]}}},
|
||||
],
|
||||
@@ -202,12 +201,46 @@ async def test_run_usage_counts_streamed_tokens(server, engine_up):
|
||||
async with SessionLocal() as session:
|
||||
ends = list((await session.execute(select(RunEvent).where(
|
||||
RunEvent.run_id == started.id, RunEvent.type == "llm.end"))).scalars())
|
||||
assert [(e.data["input_tokens"], e.data["output_tokens"]) for e in ends] == [(1200, 300)]
|
||||
assert row.usage["input_tokens"] == 1200 and row.usage["output_tokens"] == 300
|
||||
assert row.usage["total_tokens"] == 1500
|
||||
expected = round(catalog.estimate_cost(PRICED, 1200, 300), 6)
|
||||
assert expected > 0 and row.usage["cost_usd"] == pytest.approx(expected)
|
||||
return row, ends
|
||||
finally:
|
||||
async with SessionLocal() as session:
|
||||
await session.execute(delete(Provider).where(Provider.id == provider.id))
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def test_run_usage_counts_streamed_tokens(server, engine_up):
|
||||
"""整条链:兼容接入 → llm 节点流式调用 → llm.end → 运行的用量。修之前这里是 0 token、0 美元。"""
|
||||
server()
|
||||
row, ends = await _run_once(PRICED)
|
||||
assert [(e.data["input_tokens"], e.data["output_tokens"]) for e in ends] == [(1200, 300)]
|
||||
assert row.usage["input_tokens"] == 1200 and row.usage["output_tokens"] == 300
|
||||
assert row.usage["total_tokens"] == 1500
|
||||
expected = round(catalog.estimate_cost(PRICED, 1200, 300), 6)
|
||||
assert expected > 0 and row.usage["cost_usd"] == pytest.approx(expected)
|
||||
assert "unpriced" not in ends[0].data and "unpriced_calls" not in row.usage
|
||||
|
||||
|
||||
async def test_a_model_missing_from_the_price_catalog_is_marked_unpriced(server, engine_up):
|
||||
"""token 记到了,模型却不在价格目录里:金额估不出来,不能让运行页写 $0 像是没花钱。
|
||||
llm.end 标 unpriced(运行中的实时统计据此数),运行用量记 unpriced_calls(界面据此写「无法估算」)。"""
|
||||
server()
|
||||
row, ends = await _run_once("vendor-unlisted-model")
|
||||
assert [(e.data["input_tokens"], e.data["output_tokens"]) for e in ends] == [(1200, 300)]
|
||||
assert ends[0].data["unpriced"] is True and ends[0].data["cost_usd"] == 0
|
||||
assert row.usage["total_tokens"] == 1500 and row.usage["cost_usd"] == 0
|
||||
assert row.usage["unpriced_calls"] == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("data", "unpriced"), [
|
||||
({"model": "vendor-unlisted-model", "input_tokens": 10, "output_tokens": 2, "cost_usd": 0}, True),
|
||||
({"model": PRICED, "input_tokens": 10, "output_tokens": 2, "cost_usd": 0.0001}, False),
|
||||
# 目录里标 0 价的本地模型是真不花钱,不是估不出来
|
||||
({"model": "qwen3:8b", "input_tokens": 10, "output_tokens": 2, "cost_usd": 0}, False),
|
||||
# 没用 token 的调用(失败、被取消)谈不上金额
|
||||
({"model": "vendor-unlisted-model", "input_tokens": 0, "output_tokens": 0}, False),
|
||||
# 老事件没有 unpriced 键:按模型名补判;事件自己标了的以它为准
|
||||
({"model": PRICED, "input_tokens": 10, "unpriced": True}, True),
|
||||
(None, False),
|
||||
])
|
||||
def test_unpriced_call(data, unpriced):
|
||||
assert catalog.unpriced_call(data) is unpriced
|
||||
|
||||
Reference in New Issue
Block a user