各条用到模型的路径照实报出接入不存在或已停用,查助手模型的接口不再 500

逐条核对 get_chat_model / resolve_provider 的调用方:模型调用、Agent、报告撰写节点按 NodeError 失败,
双盲复算记无法复算、反方审稿记未能反查、审批质询回答失败、结论句裁判记未裁判,原因都是「核查模型
不可用(…)」「裁判模型不可用(…)」这类整句;助手生成报 400,门控模型转人工审批,定时任务发起的
正式运行失败、通知照常推送。这些调用方原本就接了 ProviderNotConfigured,测试逐条钉住:产出、事件、
llm.end 里都不出现 mock-fast。

GET /api/copilot/model 原来直接调 resolve_provider 不接异常:存的接入不存在(现在会抛)、或只存了
模型名而它的接入已停用(以前就会抛)时整个接口 500。现在照常返回,effective_* 为空,error 写明原因,
能用时 error 为 None。
This commit is contained in:
ThearchyHelios
2026-10-06 12:52:54 +08:00
parent f7d945f94b
commit 6a53a76694
2 changed files with 474 additions and 2 deletions
+11 -2
View File
@@ -518,20 +518,29 @@ async def copilot_model_spec(
@router.get("/model")
async def get_copilot_model(session: AsyncSession = Depends(get_session)) -> dict[str, Any]:
"""当前 Copilot 会用哪个模型,以及是显式配的还是兜底来的。"""
"""当前 Copilot 会用哪个模型,以及是显式配的还是兜底来的。
存的接入不存在、已停用(或者只存了模型名、提供它的接入已停用)时实际什么都用不了:effective_* 为空,
error 写明原因(和生成时报的同一句),能用时 error 为 None。"""
from app.db.models import Setting
from app.providers.factory import resolve_provider
row = await session.get(Setting, COPILOT_SETTING_KEY)
saved = row.value if row else {}
configured = bool(saved.get("provider") or saved.get("model"))
resolved = await resolve_provider(session, saved.get("provider"), saved.get("model"))
try:
resolved = await resolve_provider(session, saved.get("provider"), saved.get("model"))
except ProviderNotConfigured as e:
return {"configured": configured, "provider": saved.get("provider") or None,
"model": saved.get("model") or None, "effective_provider": None, "effective_model": None,
"error": _unconfigured(e)}
return {
"configured": configured,
"provider": saved.get("provider") or None,
"model": saved.get("model") or None,
"effective_provider": resolved.name if resolved else None,
"effective_model": saved.get("model") or (resolved.default_model if resolved else None),
"error": None,
}
+463
View File
@@ -0,0 +1,463 @@
"""节点上、设置里点名的模型接入不存在或已停用时,每条用到模型的路径都照实报给人,不落成演示模型。
以前点名的接入不存在时 get_chat_model 悄悄换成演示模型(mock-fast):报告由它写成假文本,双盲复算、反方审稿、审批
质询由它来「核查」,正式运行照样出具。现在(providers/factory.named_provider):
- 模型调用、Agent、报告撰写节点:节点失败,报错带原因和去哪里改,一次模型调用都不发;
- 双盲复算:该指标无法复算,原因「核查模型不可用(…)」,记缺口;反方审稿:这几句未能反查,原因同上;
- 审批质询:回答 ok 为假,error 说明原因;结论句裁判:按「裁判模型不可用」照实记未裁判和缺口;
- 助手:报错给用户,查当前用哪个模型的接口照常返回、写明原因;门控模型:转人工审批、写明原因;
- 定时任务发起的正式运行:运行失败,通知照常推送。
每条路径都看一遍:产出、事件、llm.end 里不出现 mock-fast。没点名接入的节点(写作者)用一个显式建的演示接入,模型名是
「写作演示模型」,所以事件里一出现 mock-fast 就是回退到了内置的演示模型。接入名、数据源、数字都是虚构的。
"""
from __future__ import annotations
import asyncio
import json
import sqlite3
import uuid
from datetime import datetime, timedelta, timezone
import httpx
import pytest
from httpx import ASGITransport, AsyncClient
from langchain_core.messages import AIMessage
from sqlalchemy import delete, select
from app.core import artifact_store
from app.core.crypto import encrypt
from app.data import introspect
from app.db.base import SessionLocal
from app.db.models import (
Approval, DataSource, Provider, Run, RunEvent, Schedule, ScheduleFire, Setting, Workflow, WorkflowVersion,
)
from app.engine import spend
from app.engine.evidence import iter_units
from app.engine.judge import SPEND_KEY as JUDGE_SPEND_KEY
from app.engine.runner import run_manager
from app.main import app
from app.providers.mock_model import MockChatModel
from tests.fixtures.dual_check_shop import DEFINITION, RIGHT, add_source, build
from tests.fixtures.sources import drop_source
#: 没点名接入的节点用的演示接入:正常的调用在事件里记「写作演示模型」
WRITER_PROVIDER, WRITER_MODEL = "测试演示接入", "写作演示模型"
#: 写错一个字的接入名;一个停用了的演示接入(停用前照样能调用:以前点名它会照常返回、照常调用)
MISSING, DISABLED = "测试演示接人", "停用演示接入"
REASONS = {MISSING: f"模型接入「{MISSING}」不存在", DISABLED: f"模型接入「{DISABLED}」已停用"}
HINTS = {MISSING: "请在「设置 → 模型接入」中核对名称,或换一个接入",
DISABLED: "请在「设置 → 模型接入」中启用它,或换一个接入"}
SETTING_KEYS = ["verify", "judge", "copilot", "run", spend.SPEND_KEY, JUDGE_SPEND_KEY]
UTC = timezone.utc
async def _clean() -> None:
async with SessionLocal() as session:
await session.execute(delete(Setting).where(Setting.key.in_(SETTING_KEYS)))
await session.execute(delete(Provider).where(Provider.name.in_([WRITER_PROVIDER, DISABLED])))
await session.commit()
@pytest.fixture(autouse=True)
async def engine_up():
await _clean()
async with SessionLocal() as session:
session.add_all([
Provider(name=WRITER_PROVIDER, kind="mock", models=[{"id": WRITER_MODEL}], default_model=WRITER_MODEL,
enabled=True, extra={}),
Provider(name=DISABLED, kind="mock", models=[{"id": "停用演示模型"}], default_model="停用演示模型",
enabled=False, extra={}),
])
await session.commit()
await run_manager.setup()
yield
await run_manager.shutdown()
# 建过的接入、写过的设置不留给同一进程里后面的测试:留着的启用接入会改变别处「没点名」时的回退
await _clean()
@pytest.fixture
async def client():
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
yield c
def script(monkeypatch, writer: str = "本周情况平稳。") -> list[str]:
"""演示模型的剧本:写作者回 writer;按结构交结果的调用交空对象、绑了工具的直接收尾(核查、裁判、质询的模型照理
一次都不该被调用到)。返回每次调用的模型名。"""
called: list[str] = []
def decide(model, messages):
called.append(model.model_name)
if model.response_format:
return AIMessage(content="{}")
if model.tools:
return AIMessage(content="查完了。")
return AIMessage(content=writer)
monkeypatch.setattr(MockChatModel, "_decide", decide)
return called
async def put_setting(key: str, value: dict) -> None:
"""直接写库:PUT /api/settings 现在会拦下不存在、已停用的接入,这里模拟的是写好之后接入才被删掉、停用。"""
async with SessionLocal() as session:
row = await session.get(Setting, key)
if row:
row.value = value
else:
session.add(Setting(key=key, value=value))
await session.commit()
def node(nid, ntype, label=None, **config):
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": label or nid, "config": config}}
def chain(*nodes):
return {"nodes": list(nodes), "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]}
async def wait(run_id: str, statuses=("succeeded", "failed")) -> Run:
for _ in range(600):
async with SessionLocal() as session:
row = await session.get(Run, run_id)
if row.status in statuses and (row.status == "interrupted" or row.manifest_seq is not None):
await run_manager.wait_idle(run_id)
return row
await asyncio.sleep(0.03)
raise AssertionError(f"运行 {run_id} 停在 {row.status}:{row.error}")
async def start(spec: dict, *, run_class: str = "exploratory", statuses=("succeeded", "failed")) -> Run:
run = await run_manager.start(graph=spec, input_payload={}, run_class=run_class)
return await wait(run.id, statuses)
async def events(run_id: str) -> list[RunEvent]:
async with SessionLocal() as session:
return list((await session.execute(
select(RunEvent).where(RunEvent.run_id == run_id).order_by(RunEvent.seq))).scalars())
def output_of(evs: list[RunEvent], node_id: str) -> dict:
finished = [e for e in evs if e.type == "node.finished" and e.node_id == node_id]
assert finished, f"节点 {node_id} 没有执行完"
return artifact_store.load(finished[-1].data["artifact"])
def logs(evs: list[RunEvent], code: str) -> list[dict]:
return [e.data for e in evs if e.type == "log" and e.data.get("code") == code]
def no_mock(evs: list[RunEvent], *more) -> None:
"""事件和给出的产出里都不出现内置演示模型的名字。"""
blob = json.dumps([e.data for e in evs] + list(more), ensure_ascii=False, default=str)
assert "mock-fast" not in blob
# --------------------------------------------------------------------------
# 模型调用、Agent、报告撰写节点:节点失败,说清原因和去哪里改
# --------------------------------------------------------------------------
NODES = {
"llm": lambda ref: node("ask", "llm", "提问", provider=ref, prompt="总结本周情况"),
"agent": lambda ref: node("ask", "agent", "查数作答", provider=ref, prompt="回答问题"),
"report": lambda ref: node("ask", "report", "写周报", provider=ref, instructions="写周报"),
}
@pytest.mark.parametrize("ref", [MISSING, DISABLED])
@pytest.mark.parametrize("kind", list(NODES))
async def test_a_node_naming_a_broken_provider_fails_and_says_why(monkeypatch, kind, ref):
called = script(monkeypatch)
spec = chain(node("start", "input"), NODES[kind](ref),
node("out", "output", fields=[{"name": "结果", "value": "{{ last_message }}"}]))
run = await start(spec)
assert run.status == "failed"
assert REASONS[ref] in run.error and HINTS[ref] in run.error, run.error
evs = await events(run.id)
[failed] = [e for e in evs if e.type == "node.failed"]
assert failed.node_id == "ask" and REASONS[ref] in failed.data["error"] and HINTS[ref] in failed.data["error"]
assert not [e for e in evs if e.type in ("llm.start", "llm.end")] and called == []
no_mock(evs, run.output, run.error)
# --------------------------------------------------------------------------
# 双盲复算:无法复算,记缺口
# --------------------------------------------------------------------------
GMV = {"id": "gmv", "name": "销售额", "unit": "元", "decimals": 1, "expression": "cell(nodes.fetch, 0, 'gmv')"}
@pytest.fixture
async def shop(tmp_path):
name, source_id = await add_source(build(tmp_path / "shop.db"))
yield name
await drop_source(source_id)
def dual_graph(source: str, **dual) -> dict:
return chain(
node("start", "input"),
node("fetch", "tool", tool=f"db_query__{source}", args={"sql": RIGHT}),
node("card", "metrics", caliber="销售口径", caliber_version="v1", metrics=[GMV]),
node("dual", "dual_check", "复核销售额", definitions={"gmv": DEFINITION}, **dual),
node("out", "output", fields=[{"name": "结论", "value": "本周销售额见口径卡"}],
contract={"metrics_from": "card", "narrative": "本周销售额见口径卡"}),
)
@pytest.mark.parametrize("where", ["settings", "node"])
@pytest.mark.parametrize("ref", [MISSING, DISABLED])
async def test_dual_check_with_a_broken_verify_provider_records_the_metric_as_unchecked(monkeypatch, shop, ref, where):
called = script(monkeypatch)
if where == "settings":
await put_setting("verify", {"provider": ref})
run = await start(dual_graph(shop, **({"provider": ref} if where == "node" else {})), run_class="formal")
assert run.status == "succeeded", run.error
evs = await events(run.id)
out = output_of(evs, "dual")
assert out["status"] == "unchecked"
[metric] = out["metrics"]
assert metric["status"] == "unchecked"
assert metric["reason"].startswith("核查模型不可用") and REASONS[ref] in metric["reason"], metric["reason"]
assert out["issuance_gaps"] == [f"指标「销售额」无法复算:{metric['reason']}"]
assert any(REASONS[ref] in gap for gap in run.output["_issuance"]["gaps"])
[said] = logs(evs, "dual_check_unchecked")
assert REASONS[ref] in said["metrics"][0]["reason"]
assert called == [] and not [e for e in evs if e.type == "llm.end"]
no_mock(evs, run.output, out)
# --------------------------------------------------------------------------
# 反方审稿:未能反查;审批质询:回答失败、说明原因
# --------------------------------------------------------------------------
SALES = ("CREATE TABLE sales (id INTEGER PRIMARY KEY, region TEXT, amount INTEGER, refunded INTEGER);"
"INSERT INTO sales VALUES (1, '华东', 1200, 0), (2, '华东', 300, 1), (3, '华南', 900, 0), (4, '华北', 400, 0);")
QUERY = "SELECT region AS 区域, SUM(amount) AS 销售额 FROM sales GROUP BY region ORDER BY 销售额 DESC"
#: 一句有依据的结论、一句预埋的错误结论(华南其实低于华东)
TEXT = "华东销售额 [[v:Q1.r0.销售额]],居各区域之首。[[see:Q1]]华南销售额超过华东。[[see:Q1]]"
@pytest.fixture
async def sales(tmp_path):
path = tmp_path / f"{uuid.uuid4().hex[:8]}.db"
db = sqlite3.connect(path)
db.executescript(SALES)
db.commit()
db.close()
async with SessionLocal() as session:
row = DataSource(name=f"sales_{uuid.uuid4().hex[:8]}", kind="sqlite", database=str(path), readonly=True,
enabled=True, options={})
session.add(row)
await session.flush()
row.schema_cache = await introspect.introspect(row)
await session.commit()
name, source_id = row.name, row.id
yield name
await drop_source(source_id)
def report_graph(source: str, *between) -> dict:
return chain(
node("start", "input"),
node("fetch", "tool", tool=f"db_query__{source}", args={"sql": QUERY}),
node("write", "report", "写周报", instructions="写周报"),
*between,
node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}],
contract={"report_from": "write"}),
)
async def test_challenge_with_a_broken_verify_provider_reviews_no_claim_and_says_why(monkeypatch, sales):
called = script(monkeypatch, writer=TEXT)
await put_setting("verify", {"provider": MISSING})
run = await start(report_graph(sales, node("chal", "challenge", "审稿", report="write")), run_class="formal")
assert run.status == "succeeded", run.error
evs = await events(run.id)
out = output_of(evs, "chal")
assert out["counts"]["inconclusive"] == out["unreviewed"] == 2
assert out["counts"]["refuted"] == out["counts"]["stands"] == 0
[said] = logs(evs, "challenge_unreviewed")
assert said["reason"].startswith("核查模型不可用") and REASONS[MISSING] in said["reason"], said["reason"]
record = artifact_store.load(out["challenge_artifact"])
assert all(REASONS[MISSING] in c["rationale"] for c in record["claims"])
# 只有写作者调用了模型
assert called == [WRITER_MODEL]
no_mock(evs, run.output, out, record)
async def test_an_inquiry_with_a_broken_verify_provider_answers_with_the_reason(monkeypatch, sales, client):
called = script(monkeypatch, writer=TEXT)
await put_setting("verify", {"provider": DISABLED})
paused = await start(report_graph(sales, node("gate", "human", mode="approve", title="周报可以出具吗")),
statuses=("interrupted",))
async with SessionLocal() as session:
[approval] = list((await session.execute(select(Approval).where(
Approval.run_id == paused.id, Approval.status == "pending"))).scalars())
r = await client.post(f"/api/runs/{paused.id}/inquiries",
json={"report": "write", "unit": None, "question": "华南真的超过华东吗?",
"approval_id": approval.id})
assert r.status_code == 200, r.text
inquiry = r.json()["inquiry"]
answer = inquiry["answer"]
assert answer["ok"] is False and answer["text"] == ""
assert answer["error"].startswith("核查模型不可用") and REASONS[DISABLED] in answer["error"], answer["error"]
assert inquiry["model"] is None and inquiry["usage"]["calls"] == 0
evs = await events(paused.id)
[event] = [e for e in evs if e.type == "approval.inquiry"]
assert event.data["ok"] is False and event.data["error"] == answer["error"]
assert called == [WRITER_MODEL]
no_mock(evs, r.json())
# --------------------------------------------------------------------------
# 结论句裁判:照「裁判模型不可用」记未裁判和缺口
# --------------------------------------------------------------------------
METRICS = [{"id": "gmv", "name": "销售额", "unit": "元", "decimals": 1, "expression": "vars.kpi.gmv"}]
JUDGED = "本周销售额 [[m:gmv]]。[[see:m:gmv]]增长主要来自新客首单。"
def weekly(**report) -> dict:
return chain(
node("start", "input"),
node("fetch", "transform", mode="expression", expression="{'gmv': 45678.5}", assign_to="kpi"),
node("caliber", "metrics", caliber="周报口径", caliber_version="v2", metrics=METRICS),
node("write", "report", instructions="写周报", **report),
node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}]),
)
@pytest.mark.parametrize("where,ref", [("node", MISSING), ("settings", DISABLED)])
async def test_a_broken_judge_provider_leaves_the_claims_unjudged_and_says_why(monkeypatch, where, ref):
called = script(monkeypatch, writer=JUDGED)
if where == "settings":
await put_setting("judge", {"provider": ref})
report = {"claims": "judge", **({"judge": {"provider": ref}} if where == "node" else {})}
run = await start(weekly(**report), run_class="formal")
assert run.status == "succeeded", run.error
evs = await events(run.id)
[checked] = [e for e in evs if e.type == "report.checked"]
summary = checked.data["judge"]
assert summary["complete"] is False and summary["counts"]["unjudged"] == 2
assert summary["gaps"] and all("裁判模型不可用" in g and REASONS[ref] in g for g in summary["gaps"]), summary["gaps"]
doc = artifact_store.load(checked.data["doc_artifact"])
verdicts = [u["verdict"] for _, u in iter_units(doc) if u.get("verdict")]
assert len(verdicts) == 2 and all(v["status"] == "unjudged" and v["reason"] == "error" for v in verdicts)
assert not [e for e in evs if e.type == "llm.end" and e.data.get("purpose") == "judge"]
assert called == [WRITER_MODEL]
no_mock(evs, run.output, doc)
# --------------------------------------------------------------------------
# 助手、门控模型
# --------------------------------------------------------------------------
async def test_the_assistant_with_a_broken_provider_tells_the_user(monkeypatch, client):
called = script(monkeypatch)
await put_setting("copilot", {"provider": MISSING, "model": ""})
r = await client.post("/api/copilot/generate", json={"instruction": "做一张门店周报"})
assert r.status_code == 400
assert r.json()["detail"] == f"助手使用的模型尚未配置完成:{REASONS[MISSING]}。请到「设置 → 模型接入」检查"
assert called == []
# 查当前用哪个模型:照常返回,不 500,写明为什么用不了
got = await client.get("/api/copilot/model")
assert got.status_code == 200, got.text
body = got.json()
assert (body["configured"], body["provider"]) == (True, MISSING)
assert body["effective_provider"] is None and body["effective_model"] is None
assert REASONS[MISSING] in body["error"]
async def test_the_assistant_model_lookup_survives_a_model_whose_provider_is_disabled(client):
"""只存了模型名、提供它的接入已停用:以前这里直接 500。"""
await put_setting("copilot", {"provider": "", "model": "停用演示模型"})
got = await client.get("/api/copilot/model")
assert got.status_code == 200, got.text
body = got.json()
assert body["effective_provider"] is None and body["effective_model"] is None
assert f"所属的模型接入「{DISABLED}」已停用" in body["error"]
async def test_the_assistant_model_lookup_reports_no_error_when_it_works(client):
await put_setting("copilot", {"provider": WRITER_PROVIDER, "model": ""})
body = (await client.get("/api/copilot/model")).json()
assert body["effective_provider"] == WRITER_PROVIDER and body["effective_model"] == WRITER_MODEL
assert body["error"] is None
async def test_a_broken_gate_provider_hands_the_call_to_a_person_and_says_why(monkeypatch):
from langchain_core.tools import StructuredTool
from app.tools.trust import ask_gate
called = script(monkeypatch)
await put_setting("run", {"tool_gate_provider": MISSING})
def lookup(name: str) -> str:
return name
tool = StructuredTool.from_function(lookup, name="crm_lookup", description="按客户名称查询客户档案")
verdict = await ask_gate(tool=tool, args={"name": "甲公司"}, node_title="查客户", task="查甲公司的档案")
assert verdict.allowed is False
assert verdict.reason.startswith("门控模型未给出判断") and REASONS[MISSING] in verdict.reason, verdict.reason
assert verdict.model == "" and called == []
# --------------------------------------------------------------------------
# 定时任务发起的正式运行:运行失败,通知照常
# --------------------------------------------------------------------------
#: 上海 2026-10-05(周一)09:00
NINE = datetime(2026, 10, 5, 1, 0, tzinfo=UTC)
HOOK = "https://hooks.example.com/agentlab/inbox-k1"
async def test_a_scheduled_formal_run_naming_a_broken_provider_fails_and_is_still_notified(monkeypatch):
from app.scheduler import loop, webhook
called = script(monkeypatch)
sent: list[dict] = []
def receive(request: httpx.Request) -> httpx.Response:
sent.append(json.loads(request.content))
return httpx.Response(200, json={})
monkeypatch.setattr(webhook, "_transport", httpx.MockTransport(receive))
monkeypatch.setattr(webhook, "RETRY_DELAYS", (0, 0, 0))
spec = chain(node("start", "input"), node("ask", "llm", "写日报", provider=MISSING, prompt="写今天的门店日报"),
node("out", "output", fields=[{"name": "日报", "value": "{{ last_message }}"}]))
async with SessionLocal() as session:
wf = Workflow(name=f"门店日报-{uuid.uuid4().hex[:6]}", graph=spec, status="published", published_version=1)
session.add(wf)
await session.flush()
session.add(WorkflowVersion(workflow_id=wf.id, version=1, graph=spec))
sched = Schedule(name="门店日报", workflow_id=wf.id, cron="0 9 * * *", timezone="Asia/Shanghai",
run_class="formal", input_template={}, enabled=True, misfire_grace_s=3600, next_run_at=NINE,
webhook_format="generic", webhook_url=encrypt(HOOK), webhook_events=["finished", "failed"])
session.add(sched)
await session.commit()
sid = sched.id
try:
await loop.fire_due(NINE + timedelta(seconds=5))
async with SessionLocal() as session:
[fire] = list((await session.execute(select(ScheduleFire).where(ScheduleFire.schedule_id == sid))).scalars())
assert fire.status == "started" and fire.run_id
run = await wait(fire.run_id, ("succeeded", "failed", "cancelled"))
assert run.status == "failed" and run.run_class == "formal"
assert REASONS[MISSING] in run.error and HINTS[MISSING] in run.error, run.error
assert await webhook.notify_due(datetime.now(UTC) + timedelta(seconds=webhook.SEAL_WAIT_S + 5)) == 1
[body] = sent
assert (body["event"], body["status"], body["run_id"]) == ("failed", "failed", run.id)
assert called == []
no_mock(await events(run.id), run.output, body)
finally:
async with SessionLocal() as session:
await session.execute(delete(Schedule).where(Schedule.id == sid))
await session.commit()