mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 03:21:58 +00:00
用量计量拿到调用时实际用的模型名:评测汇总里助手用量不再记成 unknown
验收:评测汇总里助手用量的模型是 "unknown",写作用量的模型却是对的。写作用量来自运行的 llm.end(节点配置里 就有模型名);助手用量来自 providers/usage.Meter,它只从回调的 invocation_params 取 model / model_name,而演示 模型 MockChatModel 没有 _identifying_params,调用参数里只有 _type 和 stop,于是落到 UNKNOWN_MODEL。 改法两处: - Meter:调用参数里取不到时,看 LangChain 照模型对象给回调的 metadata 里的 ls_model_name(调用时传的 model, 否则对象上的 model / model_name),再取不到才看回复、才记 unknown。不认得调用参数的模型类也记得对。 - MockChatModel 补上 _identifying_params(model_name),和 OpenAI 兼容、Anthropic 一样在调用参数里报模型名。 测试(test_usage_meter):调用参数里不报模型名的模型、演示模型都按对象上的名字记;三种接入(OpenAI 兼容、 Anthropic、演示模型)由 factory 建出来,照 LangChain 回调时给的参数喂给计量,记的都是接入里配的模型 id, 调用参数里本身也带着;评测作答(eval/answer.Driver)走演示模型时助手用量记在 mock-fast 名下。两处修复都 去掉回跑挂 3 条,只留计量那处时挂「调用参数里本身带着」那一条。
This commit is contained in:
@@ -126,6 +126,12 @@ class MockChatModel(BaseChatModel):
|
||||
def _llm_type(self) -> str:
|
||||
return "mock"
|
||||
|
||||
@property
|
||||
def _identifying_params(self) -> dict[str, Any]:
|
||||
"""调用参数里带上模型名(同 OpenAI 兼容、Anthropic 的 model):用量计量(providers/usage)按它记在哪个模型名下,
|
||||
不报的话评测汇总里助手用量的模型就成了 unknown。"""
|
||||
return {"model_name": self.model_name}
|
||||
|
||||
def bind_tools(self, tools: Sequence[Any], **kwargs: Any) -> "MockChatModel":
|
||||
normalized: list[dict[str, Any]] = []
|
||||
for tool in tools:
|
||||
|
||||
@@ -18,8 +18,10 @@ async generator:with 块跨着 yield 的话,变量在两次 yield 之间留
|
||||
流末尾那一段用量收得到;只有模型在那一行之后还往下写超过 DRAIN_CHARS 字的闲话时才当场关流(不为闲话付钱),连接断了也一样,
|
||||
这一次只拿得到已经报了的部分:照样计次数、计已报的 token,记成估不出,不悄悄少算。
|
||||
|
||||
模型名按发起调用时的参数取(invocation_params 的 model / model_name,就是接入里配的模型 id,价格目录按它查),取不到再
|
||||
看回复里的 model_name。
|
||||
模型名按发起调用时的参数取(invocation_params 的 model / model_name,就是接入里配的模型 id,价格目录按它查);调用参数里
|
||||
没有的(以前的演示模型就没报,评测汇总里助手用量的模型因此记成 unknown),看 LangChain 照模型对象给回调的 metadata 里的
|
||||
ls_model_name;再取不到看回复里的 model_name,都没有才记 UNKNOWN_MODEL。OpenAI 兼容、Anthropic 的模型在调用参数里就带着
|
||||
model,演示模型(providers/mock_model)也把 model_name 报进调用参数。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -54,6 +56,10 @@ class Meter(BaseCallbackHandler):
|
||||
def _started(self, run_id: UUID, serialized: dict[str, Any] | None, kwargs: dict[str, Any]) -> None:
|
||||
params = kwargs.get("invocation_params") or {}
|
||||
model = params.get("model") or params.get("model_name") or params.get("model_id")
|
||||
if not model:
|
||||
# LangChain 照模型对象给回调的 metadata 里带着 ls_model_name(调用时传的 model,否则对象上的 model /
|
||||
# model_name):调用参数里不报模型名的模型类也认得出
|
||||
model = (kwargs.get("metadata") or {}).get("ls_model_name")
|
||||
if not model and isinstance(serialized, dict):
|
||||
model = (serialized.get("kwargs") or {}).get("model") or (serialized.get("kwargs") or {}).get("model_name")
|
||||
with self._lock:
|
||||
|
||||
@@ -105,6 +105,85 @@ async def test_metering_counts_by_model_and_prices_from_the_catalog():
|
||||
assert got["cost_usd"] == pytest.approx(estimate_cost(PRICED, 2000, 400))
|
||||
|
||||
|
||||
class Bare(Scripted):
|
||||
"""不报调用参数的模型(同以前的演示模型):invocation_params 里没有 model,只在对象上有。"""
|
||||
|
||||
@property
|
||||
def _identifying_params(self) -> dict[str, Any]:
|
||||
return {}
|
||||
|
||||
|
||||
async def test_the_model_name_comes_from_the_model_object_when_the_call_parameters_lack_it():
|
||||
"""评测汇总里助手用量的模型以前记成 unknown(下一轮验收):演示模型的调用参数里没有模型名,计量也就认不出。现在
|
||||
调用参数里没有时看 LangChain 照模型对象给回调的 ls_model_name;演示模型自己也把 model_name 报进调用参数。"""
|
||||
from app.providers.mock_model import MockChatModel
|
||||
|
||||
m = usage.meter()
|
||||
with usage.metering(m):
|
||||
await Bare(model=PRICED + "-bare").ainvoke("你好")
|
||||
await MockChatModel(model_name="mock-wide", latency=0).ainvoke("你好")
|
||||
got = m.summary()["by_model"]
|
||||
assert set(got) == {PRICED + "-bare", "mock-wide"}, got
|
||||
assert usage.UNKNOWN_MODEL not in got
|
||||
|
||||
|
||||
@pytest.mark.parametrize("build", ["openai", "anthropic", "mock"])
|
||||
def test_the_call_parameters_of_each_provider_kind_carry_the_configured_model(build):
|
||||
"""三种接入(OpenAI 兼容、Anthropic、演示模型)由 providers/factory 建出来的模型:LangChain 回调时给的调用参数和
|
||||
ls_model_name(这里照它的写法直接取,不发请求)计量都认得出,记的就是接入里配的模型 id。"""
|
||||
import uuid as _uuid
|
||||
from types import SimpleNamespace
|
||||
|
||||
from langchain_core.outputs import LLMResult
|
||||
|
||||
from app.providers.factory import ModelSpec, build_chat_model
|
||||
|
||||
provider = SimpleNamespace(kind=build, name="虚构接入", api_key=None, base_url=None, extra={},
|
||||
default_model=None)
|
||||
model_id = {"openai": "gpt-made-up", "anthropic": "claude-made-up", "mock": "mock-wide"}[build]
|
||||
import os
|
||||
env = {"openai": "OPENAI_API_KEY", "anthropic": "ANTHROPIC_API_KEY"}.get(build)
|
||||
old = os.environ.get(env) if env else None
|
||||
if env:
|
||||
os.environ[env] = "sk-made-up"
|
||||
try:
|
||||
model = build_chat_model(provider, ModelSpec(model=model_id))
|
||||
finally:
|
||||
if env:
|
||||
if old is None:
|
||||
os.environ.pop(env, None)
|
||||
else:
|
||||
os.environ[env] = old
|
||||
m = usage.meter()
|
||||
run_id = _uuid.uuid4()
|
||||
m.on_chat_model_start({}, [], run_id=run_id, invocation_params=model._get_invocation_params(),
|
||||
metadata=dict(model._get_ls_params()))
|
||||
m.on_llm_end(LLMResult(generations=[[ChatGeneration(message=AIMessage(content="好"))]]), run_id=run_id)
|
||||
assert list(m.summary()["by_model"]) == [model_id]
|
||||
# 调用参数里本身就带着(不靠 ls_model_name 兜着)
|
||||
params = model._get_invocation_params()
|
||||
assert model_id in (params.get("model"), params.get("model_name")), params
|
||||
|
||||
|
||||
async def test_the_eval_driver_on_the_demo_model_records_its_name(monkeypatch):
|
||||
"""评测作答走演示模型时,助手用量记在演示模型的名下,不是 unknown(下一轮验收:写作用量的模型对、助手的记成 unknown)。"""
|
||||
import app.api.copilot as copilot
|
||||
from app.eval import answer
|
||||
from app.providers.mock_model import MockChatModel
|
||||
|
||||
async def _model(*_a, **_k):
|
||||
return MockChatModel(model_name="mock-fast", latency=0), "mock-fast"
|
||||
|
||||
monkeypatch.setattr(copilot, "get_chat_model", _model)
|
||||
driver = answer.Driver("按天看销售额", datasource_ids=[], ctx=answer.TurnContext(
|
||||
suite_id="s", suite_name="虚构题集", eval_run_id="e", case_id=None, case_seq=1, actor=None))
|
||||
ops, refused = await driver._generate("按天看销售额", None)
|
||||
assert refused is None
|
||||
driver._take(ops)
|
||||
got = driver.turn.usage["assistant"]["by_model"]
|
||||
assert set(got) == {"mock-fast"} and got["mock-fast"]["calls"] >= 1, got
|
||||
|
||||
|
||||
async def test_outside_the_meter_nothing_is_counted():
|
||||
m = usage.meter()
|
||||
with usage.metering(m):
|
||||
|
||||
Reference in New Issue
Block a user