diff --git a/backend/app/engine/context.py b/backend/app/engine/context.py index 65fb8a85..ad197844 100644 --- a/backend/app/engine/context.py +++ b/backend/app/engine/context.py @@ -56,6 +56,12 @@ class NodeContext: writer = get_stream_writer() except Exception: # noqa: BLE001 - 不在图执行上下文里（比如单元测试） return + if str(event_type) == EventType.LLM_END and "unpriced" not in data: + from app.providers.catalog import unpriced_call + + if unpriced_call(data): + # 金额估不出来（模型不在价格目录里）：标出来，界面不把它的 0 当成「没花钱」 + data = {**data, "unpriced": True} writer( { "__agentlab__": True, diff --git a/backend/app/engine/runner.py b/backend/app/engine/runner.py index d47639c8..1be770d4 100644 --- a/backend/app/engine/runner.py +++ b/backend/app/engine/runner.py @@ -837,11 +837,15 @@ class RunManager: run = await session.get(Run, run_id) if run: timing = await _timing(session, run, elapsed_ms, now) + events_usage = await _usage_of_events(session, run_id) if status != "succeeded": # 状态里的用量只算跑完了的节点。失败、取消、停在审批上的那个节点已经 # 花出去的调用（校验修复、agent 的前几步）只在 llm.end 里——以前这部分 # 在运行记录上凭空消失，失败的运行看着像没花钱 - usage = _spent(usage, await _usage_of_events(session, run_id)) + usage = _spent(usage, events_usage) + elif events_usage.get("unpriced_calls"): + # 金额估不出来的调用只有 llm.end 数得出（状态里的用量只有金额，0 分不出是免费还是没价） + usage = {**usage, "unpriced_calls": events_usage["unpriced_calls"]} # duration_ms 是各段执行时长之和（= active_ms）。以前用这一段的耗时 # 覆盖它：87 秒、中间等过一次审批的运行，列表上写着 22ms usage.update(duration_ms=timing["active_ms"], **timing) @@ -1251,13 +1255,20 @@ async def _usage_of_events(session: Any, run_id: str) -> dict[str, Any]: rows = await session.execute( select(RunEvent.data).where(RunEvent.run_id == run_id, RunEvent.type == EventType.LLM_END) ) + from app.providers.catalog import unpriced_call + total: dict[str, Any] = dict.fromkeys(_USAGE_KEYS, 0) + unpriced = 0 for (data,) in rows: for key in _USAGE_KEYS: value = (data or {}).get(key, 1 if key == "calls" else 0) if isinstance(value, (int, float)) and not isinstance(value, bool): total[key] += value + unpriced += unpriced_call(data) total["cost_usd"] = round(total["cost_usd"], 6) + if unpriced: + # 有几次调用的金额估不出来（模型不在价格目录里）：cost_usd 只是估得出的那部分，界面据此不写 $0 + total["unpriced_calls"] = unpriced return total @@ -1269,6 +1280,8 @@ def _spent(state_usage: dict[str, Any], events_usage: dict[str, Any]) -> dict[st mine, theirs = out.get(key) or 0, events_usage.get(key) or 0 if isinstance(mine, (int, float)) and theirs > mine: out[key] = theirs + if events_usage.get("unpriced_calls"): + out["unpriced_calls"] = events_usage["unpriced_calls"] if "input_tokens" in out or "output_tokens" in out: out["total_tokens"] = int(out.get("input_tokens") or 0) + int(out.get("output_tokens") or 0) return out diff --git a/backend/app/providers/catalog.py b/backend/app/providers/catalog.py index 116850ab..a610c82d 100644 --- a/backend/app/providers/catalog.py +++ b/backend/app/providers/catalog.py @@ -229,6 +229,22 @@ def find_context(model: str) -> int | None: return None +def unpriced_call(data: Any) -> bool: + """一次模型调用（llm.end 的数据）的金额估不出来：用了 token，模型却不在价格目录里，estimate_cost 只能给 0。 + + 这时写 $0 像是没花钱（真实运行里上百次调用、上百万 token，运行页写着 $0）。本地模型在目录里标的是 0 价， + 不算估不出来。事件自己标了 unpriced 的以它为准。""" + if not isinstance(data, dict): + return False + if isinstance(data.get("unpriced"), bool): + return data["unpriced"] + tokens = sum(v for v in (data.get("input_tokens"), data.get("output_tokens")) + if isinstance(v, (int, float)) and not isinstance(v, bool)) + model = data.get("model") + return tokens > 0 and isinstance(model, str) and bool(model) and not data.get("cost_usd") \ + and find_pricing(model) is None + + def estimate_cost(model: str, input_tokens: int, output_tokens: int) -> float: """按目录价估算一次调用的花费（美元）。""" pricing = find_pricing(model) diff --git a/backend/tests/test_openai_stream_usage.py b/backend/tests/test_openai_stream_usage.py index 99c539f7..14429cb4 100644 --- a/backend/tests/test_openai_stream_usage.py +++ b/backend/tests/test_openai_stream_usage.py @@ -171,9 +171,8 @@ async def engine_up(): await run_manager.shutdown() -async def test_run_usage_counts_streamed_tokens(server, engine_up): - """整条链：兼容接入 → llm 节点流式调用 → llm.end → 运行的用量。修之前这里是 0 token、0 美元。""" - server() +async def _run_once(model: str) -> tuple[Run, list[RunEvent]]: + """兼容接入上用 model 跑一个 llm 节点，返回运行行和 llm.end 事件。""" provider = _provider() async with SessionLocal() as session: session.add(provider) @@ -185,7 +184,7 @@ async def test_run_usage_counts_streamed_tokens(server, engine_up): "nodes": [ {"id": "in", "type": "input", "data": {"config": {}}}, {"id": "ask", "type": "llm", "data": {"label": "问答", "config": { - "prompt": "上周到店多少人", "provider": provider.name, "model": PRICED}}}, + "prompt": "上周到店多少人", "provider": provider.name, "model": model}}}, {"id": "out", "type": "output", "data": {"config": { "fields": [{"name": "回答", "value": "{{ nodes.ask.text }}"}]}}}, ], @@ -202,12 +201,46 @@ async def test_run_usage_counts_streamed_tokens(server, engine_up): async with SessionLocal() as session: ends = list((await session.execute(select(RunEvent).where( RunEvent.run_id == started.id, RunEvent.type == "llm.end"))).scalars()) - assert [(e.data["input_tokens"], e.data["output_tokens"]) for e in ends] == [(1200, 300)] - assert row.usage["input_tokens"] == 1200 and row.usage["output_tokens"] == 300 - assert row.usage["total_tokens"] == 1500 - expected = round(catalog.estimate_cost(PRICED, 1200, 300), 6) - assert expected > 0 and row.usage["cost_usd"] == pytest.approx(expected) + return row, ends finally: async with SessionLocal() as session: await session.execute(delete(Provider).where(Provider.id == provider.id)) await session.commit() + + +async def test_run_usage_counts_streamed_tokens(server, engine_up): + """整条链：兼容接入 → llm 节点流式调用 → llm.end → 运行的用量。修之前这里是 0 token、0 美元。""" + server() + row, ends = await _run_once(PRICED) + assert [(e.data["input_tokens"], e.data["output_tokens"]) for e in ends] == [(1200, 300)] + assert row.usage["input_tokens"] == 1200 and row.usage["output_tokens"] == 300 + assert row.usage["total_tokens"] == 1500 + expected = round(catalog.estimate_cost(PRICED, 1200, 300), 6) + assert expected > 0 and row.usage["cost_usd"] == pytest.approx(expected) + assert "unpriced" not in ends[0].data and "unpriced_calls" not in row.usage + + +async def test_a_model_missing_from_the_price_catalog_is_marked_unpriced(server, engine_up): + """token 记到了，模型却不在价格目录里：金额估不出来，不能让运行页写 $0 像是没花钱。 + llm.end 标 unpriced（运行中的实时统计据此数），运行用量记 unpriced_calls（界面据此写「无法估算」）。""" + server() + row, ends = await _run_once("vendor-unlisted-model") + assert [(e.data["input_tokens"], e.data["output_tokens"]) for e in ends] == [(1200, 300)] + assert ends[0].data["unpriced"] is True and ends[0].data["cost_usd"] == 0 + assert row.usage["total_tokens"] == 1500 and row.usage["cost_usd"] == 0 + assert row.usage["unpriced_calls"] == 1 + + +@pytest.mark.parametrize(("data", "unpriced"), [ + ({"model": "vendor-unlisted-model", "input_tokens": 10, "output_tokens": 2, "cost_usd": 0}, True), + ({"model": PRICED, "input_tokens": 10, "output_tokens": 2, "cost_usd": 0.0001}, False), + # 目录里标 0 价的本地模型是真不花钱，不是估不出来 + ({"model": "qwen3:8b", "input_tokens": 10, "output_tokens": 2, "cost_usd": 0}, False), + # 没用 token 的调用（失败、被取消）谈不上金额 + ({"model": "vendor-unlisted-model", "input_tokens": 0, "output_tokens": 0}, False), + # 老事件没有 unpriced 键：按模型名补判；事件自己标了的以它为准 + ({"model": PRICED, "input_tokens": 10, "unpriced": True}, True), + (None, False), +]) +def test_unpriced_call(data, unpriced): + assert catalog.unpriced_call(data) is unpriced diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index cb37afe9..16ace5fa 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -46,7 +46,7 @@ import type { CatalogCardinality, CatalogMeasure, CatalogProfileNumberKey, CatalogSource, CatalogStatus, CatalogTableKind, NodeType, SnapshotReasonCode, SqlCheckCode, SqlCheckLevel, ToolTrust, } from '../types' -import { formatNumber } from './format' +import { formatCost, formatNumber } from './format' export const TERMS = { workflow: '工作流', @@ -651,6 +651,24 @@ export const JUDGE_ON_UNSUPPORTED_LABEL: Record = { } /** 设置页「证据裁判」一组（后端 engine/judge.py 的 JUDGE_DEFAULTS） */ +/** 运行的金额。有调用的模型不在价格目录中时金额估不出来，不写「$0」（看着像没花钱） */ +export const COST_TEXT = { + unpriced: '金额无法估算', + /** 悬停说明，n 是估不出金额的调用次数 */ + unpricedTitle: (n: number) => `${formatNumber(n)} 次调用的模型不在价格目录中，金额无法估算，未计入`, +} as const + +/** + * 运行级金额的写法。unpriced 是估不出金额的调用次数：一分都没估出来写「金额无法估算」，估出了一部分写「≥ $x」； + * 运行中（approx）只算已上报的调用，也写「≥」。没有金额返回 null，由调用方写占位 + */ +export function runCostText(usd: number | null | undefined, unpriced: number, approx = false): string | null { + const known = typeof usd === 'number' && Number.isFinite(usd) + if (unpriced > 0 && !(known && usd > 0)) return COST_TEXT.unpriced + if (!known) return null + return `${(approx || unpriced > 0) && usd > 0 ? '≥ ' : ''}${formatCost(usd)}` +} + export const JUDGE_SETTING_TEXT = { label: '证据裁判', hint: '报告中的结论句（如「增长主要来自新客首单」）由另一个模型按证据逐句判断：正式运行由报告撰写节点当场裁判，探索运行在点击时逐句裁判。判断由模型给出，不是系统核对，界面上一律标注「非确定」。', diff --git a/frontend/src/pages/runs/Telemetry.tsx b/frontend/src/pages/runs/Telemetry.tsx index 289e933c..01435e46 100644 --- a/frontend/src/pages/runs/Telemetry.tsx +++ b/frontend/src/pages/runs/Telemetry.tsx @@ -10,7 +10,7 @@ import { type StatusCode } from '../../lib/status' import { isExecuting, liveAt, project, type Projection, type RunPhase, type Trace } from '../../run/trace' import { useRunClock } from '../../run/useRunClock' import type { Approval, Run } from '../../types' -import { UNSAVED_HINT, isUnsaved, runName } from '../../lib/terms' +import { COST_TEXT, UNSAVED_HINT, isUnsaved, runCostText, runName } from '../../lib/terms' import { LONG_WAIT_MS, ageMs, cancelReason, runScope, runTiming, type RunTiming, } from './model' @@ -92,6 +92,9 @@ export function Telemetry({ run, trace, phase, code, streaming, pending, labelOf const tokens = settled || !streaming ? tokensSettled ?? (liveTokens || undefined) : liveTokens const cost = settled || !streaming ? (isNum(usage.cost_usd) ? usage.cost_usd : trace.costUsd || undefined) : trace.costUsd const approx = streaming && !settled + // 金额估不出来的调用（模型不在价格目录中）：有的话不写 $0 + const unpriced = (settled || !streaming ? (isNum(usage.unpriced_calls) ? usage.unpriced_calls : undefined) : undefined) + ?? trace.unpricedCalls const running = Object.values(trace.nodes).filter(isExecuting) const waitingId = trace.waitingNodeId ?? pending?.[0]?.node_id @@ -156,7 +159,7 @@ export function Telemetry({ run, trace, phase, code, streaming, pending, labelOf {tokens ? <>{approx && '≥ '}{formatTokens(tokens, { compact: true })} : NONE} - {isNum(cost) && cost > 0 ? `${approx ? '≥ ' : ''}${formatCost(cost)}` : isNum(cost) ? formatCost(cost) : ' '} + {runCostText(cost, unpriced, approx) ?? ' '} ) diff --git a/frontend/src/pages/runs/TracePane.tsx b/frontend/src/pages/runs/TracePane.tsx index 2e34fa0a..8c6523a0 100644 --- a/frontend/src/pages/runs/TracePane.tsx +++ b/frontend/src/pages/runs/TracePane.tsx @@ -11,7 +11,7 @@ import { explainRunError } from '../../lib/explain' import { NONE, formatCost, formatDateTime, formatNumber, formatOffset, formatSpan, formatTokens, shortId, } from '../../lib/format' -import { runClassLabel, nodeTypeLabel } from '../../lib/terms' +import { COST_TEXT, runClassLabel, nodeTypeLabel, runCostText } from '../../lib/terms' import { topology, type GraphLike } from '../../run/derive' import { isActivePhase, isSettled, liveAt, project, type NodeState, type Projection, type RunPhase, type Trace, @@ -245,7 +245,9 @@ function Readout({ sub={tokens ? `输入 ${formatNumber(proj.tokensIn)} · 输出 ${formatNumber(proj.tokensOut)}` : undefined}> {tokens == null ? NONE : formatTokens(tokens, { compact: true })} - {!metered || proj.costUsd == null ? NONE : formatCost(proj.costUsd)} + + {!metered || proj.costUsd == null ? NONE : runCostText(proj.costUsd, trace.unpricedCalls) ?? NONE} +
diff --git a/frontend/src/run/RunHud.tsx b/frontend/src/run/RunHud.tsx index 807846bd..024c808e 100644 --- a/frontend/src/run/RunHud.tsx +++ b/frontend/src/run/RunHud.tsx @@ -8,7 +8,7 @@ import { confirmDialog, Kbd, Spinner, StatusBadge, toast } from '../components/u import { formatCost, formatDuration, formatLapse, formatOffset, formatSpan, formatTokens, NONE } from '../lib/format' import { isTypingTarget, matchShortcut } from '../lib/keys' import { statusMeta, STATUS, type StatusCode } from '../lib/status' -import { issuanceLabel, runClassLabel } from '../lib/terms' +import { COST_TEXT, issuanceLabel, runClassLabel, runCostText } from '../lib/terms' import { lastStampOf, projectSettled, useDock, waitedMs } from '../canvas/RunTimeline' import { structureSig, useStudio } from '../store/studio' import type { RunEvent } from '../types' @@ -91,6 +91,8 @@ export interface RunGlance { series: [number, number][] tokens: number | null cost: number | null + /** 金额估不出来的调用次数（模型不在价格目录中）；有的话金额不写 $0 */ + unpriced: number /** 这一段执行还剩多少时间；null = 不知道上限或眼下不在计时 */ remainingMs: number | null limitMs: number | null @@ -220,6 +222,7 @@ export function useRunGlance(active: boolean): RunGlance { const replayTok = proj.tokensIn != null && proj.tokensOut != null ? proj.tokensIn + proj.tokensOut : null const tokens = replayAt != null ? replayTok : backendTok ?? (seen ? usage.tokensIn + usage.tokensOut : null) const cost = replayAt != null ? proj.costUsd ?? null : backendCost ?? (seen ? usage.costUsd : null) + const unpriced = (settledUsage ? num(runUsage.unpriced_calls) : null) ?? trace.unpricedCalls // 时限按"这一段执行"计：等人工审批时执行已经结束，那段不算（和后端一致） const drive = trace.drives[trace.drives.length - 1] @@ -253,7 +256,7 @@ export function useRunGlance(active: boolean): RunGlance { headline: headlineOf(trace, phase, states, at, label, runError, replayAt == null ? cancelled : ''), executing: phase === 'running' && cells.some((c) => c.state === 'running' && !trace.nodes[c.id]?.looping), cells, done, visited, parallelNow: proj.parallelNow, series: trace.parallelSeries, - tokens, cost, remainingMs, limitMs, attention, + tokens, cost, unpriced, remainingMs, limitMs, attention, failedNodeId: trace.failedNodeId, waitingNodeId: trace.waitingNodeId, } } @@ -591,7 +594,7 @@ export function RunCapsule() { {nodesText} - {tokens}{g.cost ? ` · ${formatCost(g.cost)}` : ''} + {tokens}{g.cost || g.unpriced ? ` · ${runCostText(g.cost, g.unpriced)}` : ''} )} @@ -654,7 +657,9 @@ export function RunCapsule() {
用量
{tokens}
-
{g.cost != null ? `≈ ${formatCost(g.cost)}` : NONE}
+
+ {g.unpriced ? runCostText(g.cost, g.unpriced) : g.cost != null ? `≈ ${formatCost(g.cost)}` : NONE} +
时限
diff --git a/frontend/src/run/trace.ts b/frontend/src/run/trace.ts index da4281bc..d0aedd17 100644 --- a/frontend/src/run/trace.ts +++ b/frontend/src/run/trace.ts @@ -96,6 +96,8 @@ export interface Trace { tokensIn: number tokensOut: number costUsd: number + /** 金额估不出来的调用次数（llm.end 标了 unpriced：模型不在价格目录中）。这时 costUsd 的 0 不是「没花钱」 */ + unpricedCalls: number failedNodeId?: string waitingNodeId?: string issuance?: { @@ -238,7 +240,7 @@ export const lastEventAt = (t: Trace): number => t.book.clock export function emptyTrace(): Trace { return { - phase: 'idle', waitMs: 0, nodes: {}, tokensIn: 0, tokensOut: 0, costUsd: 0, + phase: 'idle', waitMs: 0, nodes: {}, tokensIn: 0, tokensOut: 0, costUsd: 0, unpricedCalls: 0, lastSeq: 0, parallelSeries: [], usageSeries: [], timed: true, drives: [], waits: [], phases: [], lastReplay: false, book: { clock: 0, awaiting: false, dispatchFrom: {}, exactDispatch: {}, asking: {} }, @@ -673,6 +675,7 @@ export function foldEvent(t: Trace, ev: RunEvent): Trace { next.tokensIn += inTok next.tokensOut += outTok next.costUsd += cost + if (d.unpriced === true) next.unpricedCalls += 1 next.usageSeries = [...next.usageSeries, [at, next.tokensIn, next.tokensOut, next.costUsd]] if (nodeId) { const n = node(nodeId) @@ -816,6 +819,7 @@ export function foldEvent(t: Trace, ev: RunEvent): Trace { next.tokensIn = num(u.input_tokens) ?? next.tokensIn next.tokensOut = num(u.output_tokens) ?? next.tokensOut next.costUsd = num(u.cost_usd) ?? next.costUsd + next.unpricedCalls = num(u.unpriced_calls) ?? next.unpricedCalls } const timing = d.timing if (timing && num(timing.wall_ms) != null) { diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 0b1c6c84..1a47d0af 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -110,6 +110,8 @@ export interface RunUsage { input_tokens?: number output_tokens?: number cost_usd?: number + /** 金额估不出来的调用次数（模型不在价格目录中），cost_usd 只是估得出的那部分。没有这种调用时不给 */ + unpriced_calls?: number [key: string]: any } diff --git a/scripts/check-trace.mjs b/scripts/check-trace.mjs index 71a03652..57ee3da2 100644 --- a/scripts/check-trace.mjs +++ b/scripts/check-trace.mjs @@ -540,6 +540,35 @@ await section('右栏收尾（decodeRun）', async () => { fold([start('甲', 1), start('乙', 2), end('甲', 3, 100), end('乙', 4, 300)]).savedMs === 100) }) +await section('金额估不出来的调用', async () => { + // 模型不在价格目录中：token 记到了，金额只能是 0。运行页写 $0 像是没花钱，改写「金额无法估算」 + const ev = (seq, type, node_id, data = {}) => ({ seq, type, node_id, ts: 5500 + seq, data }) + const head = [ + ev(1, 'run.started', null, { nodes: 1 }), + ev(2, 'node.started', 'agent'), + ev(3, 'llm.end', 'agent', { model: 'vendor-unlisted', input_tokens: 900, output_tokens: 90, cost_usd: 0, unpriced: true }), + ev(4, 'llm.end', 'agent', { model: 'vendor-unlisted', input_tokens: 700, output_tokens: 70, cost_usd: 0, unpriced: true }), + ] + const gr = g(['agent'], []) + const live = run(head, gr) + check('运行中：llm.end 标了 unpriced 的逐次数进来', live.unpricedCalls === 2 && live.costUsd === 0, `${live.unpricedCalls}`) + const done = run([...head, ev(5, 'node.finished', 'agent', { duration_ms: 10 }), + ev(6, 'run.finished', null, { usage: { input_tokens: 1600, output_tokens: 160, cost_usd: 0, unpriced_calls: 3 } })], gr) + check('终态以后端总数为准（老后端少发的也算上）', done.unpricedCalls === 3, `${done.unpricedCalls}`) + const plain = run([ev(1, 'run.started', null, { nodes: 1 }), ev(2, 'node.started', 'agent'), + ev(3, 'llm.end', 'agent', { input_tokens: 10, output_tokens: 1, cost_usd: 0.001 })], gr) + check('目录里有价的调用不算', plain.unpricedCalls === 0) + + const { runCostText, COST_TEXT } = await import(await load('/src/lib/terms.ts')) + check('一分都没估出来：写「金额无法估算」，不写 $0', runCostText(0, 3) === COST_TEXT.unpriced, runCostText(0, 3)) + check('……没拿到金额时也一样', runCostText(null, 1) === COST_TEXT.unpriced) + check('估出了一部分：写「≥」', runCostText(0.012, 2) === '≥ $0.012', runCostText(0.012, 2)) + check('全都有价：照常写', runCostText(0.012, 0) === '$0.012' && runCostText(0, 0) === '$0', `${runCostText(0.012, 0)} / ${runCostText(0, 0)}`) + check('运行中只算已上报的：写「≥」', runCostText(0.012, 0, true) === '≥ $0.012') + check('没有金额也没有估不出的：交给调用方写占位', runCostText(undefined, 0) === null) + check('悬停说明写清几次调用、为什么', COST_TEXT.unpricedTitle(3).includes('3 次调用') && COST_TEXT.unpricedTitle(3).includes('价格目录')) +}) + await section('回放时的用量和工具数（逐刻度）', async () => { // 回放到半路，卡片和 HUD 要写「那一刻」的 tokens 和工具数，而不是拿终值冒充 const ev = (seq, type, node_id, data = {}) => ({ seq, type, node_id, ts: 5000 + seq, data })