mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 02:31:58 +00:00
保存设置、画布校验、发布时就查出点名了不存在或已停用的模型接入
运行时点名的接入不存在或已停用现在照实报错,这里把问题提前到配置的时候:
- PUT /api/settings:门控模型、证据裁判、核查模型、助手这几项引用接入的字段写了不存在或已停用的接入
回 422 {detail, code: settings_provider_invalid, field: 组.键},说明写设置页上的叫法、和运行时报的
原因提示同一句;一组出错整次不落库,留空照旧表示沿用上一级。只查这次改了的值:写好之后接入才被删、
停用的,不挡这次保存同组别的项(设置页标着不存在,运行时照实报错),同设置页只校验改过的做法。
- 画布校验给警告,发布、发布前检查、自动修复的复核给错误挡住发布,问题落在节点和字段上(provider、
agents[i].provider、judge.provider,全图默认记 defaults.provider),编号 provider.missing /
provider.disabled。只查执行时真会取模型的那几处:分支让模型分类、校验让模型修复、检索让模型重排时
才算,报告交给模型裁判时才查裁判的接入,有节点会取全图默认时才查默认。判断在 engine/provider_refs.py,
接入表由接口这一层读出来交给它(门禁本身不查库,同 SQL 检查器)。自动修复后只剩接入问题、人又要
交给助手时说清助手处理不了。
test_verify_settings 的整组往返原来写了一个不存在的接入 house,改成先建一个演示类的接入。
This commit is contained in:
@@ -10,18 +10,21 @@ from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.api import health
|
||||
from app.api.coded import CodedHTTPException
|
||||
from app.core.errors import explain, not_configured, raw
|
||||
from app.core.config import settings as app_settings
|
||||
from app.core.crypto import encrypt, mask
|
||||
from app.db.base import get_session
|
||||
from app.db.models import Provider, Setting
|
||||
from app.engine.judge import JUDGE_DEFAULTS, MAX_TOKENS as JUDGE_MAX_TOKENS
|
||||
from app.engine.judge import SETTING_GROUP as JUDGE_GROUP, SPEND_KEY as JUDGE_SPEND
|
||||
from app.engine.judge import SETTING_GROUP as JUDGE_GROUP, SETTING_LABEL as JUDGE_LABEL, SPEND_KEY as JUDGE_SPEND
|
||||
from app.engine.judge import daily_spend, judge_model_spec, judge_settings
|
||||
from app.engine.judge import sanitize_settings as judge_values, settings_problem as judge_problem
|
||||
from app.engine.spend import SPEND_KEY as VERIFY_SPEND, daily_spend as verify_daily_spend
|
||||
from app.providers import catalog
|
||||
from app.providers.factory import ModelSpec, ProviderNotConfigured, _resolve_key, build_chat_model
|
||||
from app.providers.factory import (
|
||||
ModelSpec, ProviderNotConfigured, _resolve_key, build_chat_model, list_providers as all_providers, named_provider,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/api", tags=["settings"])
|
||||
|
||||
@@ -719,6 +722,49 @@ class SettingsIn(BaseModel):
|
||||
values: dict[str, Any]
|
||||
|
||||
|
||||
#: 点名的模型接入不存在或已停用时 PUT /api/settings 回 422 的机读码,field 是出错的那一项(verify.provider……)
|
||||
PROVIDER_INVALID = "settings_provider_invalid"
|
||||
|
||||
|
||||
def _provider_fields() -> dict[tuple[str, str], tuple[str, str]]:
|
||||
"""设置里点名模型接入的项:(组, 键) → (设置页上的叫法, 留空时用什么)。门控、裁判、核查的叫法和设置页一致;助手的
|
||||
模型由助手面板单独设定(PUT /api/copilot/model),这里只防经设置接口写进去的那一份。"""
|
||||
from app.api.copilot import COPILOT_SETTING_KEY # 在这里导入:copilot 模块很大,设置模块被引擎各处导入
|
||||
|
||||
return {
|
||||
("run", "tool_gate_provider"): ("门控模型 · 接入", "留空表示使用默认接入的默认模型"),
|
||||
(JUDGE_GROUP, "provider"): (JUDGE_LABEL["provider"], "留空表示不指定"),
|
||||
(VERIFY_GROUP, "provider"): (VERIFY_SETTING_LABEL["provider"], "留空表示沿用证据裁判模型"),
|
||||
(COPILOT_SETTING_KEY, "provider"): ("助手模型 · 接入", "留空表示不指定"),
|
||||
}
|
||||
|
||||
|
||||
async def _provider_problem(session: AsyncSession, values: dict[str, Any]) -> tuple[str, str] | None:
|
||||
"""这次写进来的设置里点名的模型接入有没有问题:(字段, 说明),没问题返回 None。说明和运行时报的原因、提示同一句
|
||||
(factory.named_provider)。
|
||||
|
||||
以前点名的接入不存在时运行悄悄换成演示模型,名字写错一个字就由演示模型写报告、做核查,现在运行时照实报错;保存时
|
||||
先拦一道。留空照旧表示沿用上一级。只查这次改了的:存着的值没改(写好之后接入才被删、停用)不挡这次保存别的项——
|
||||
设置页标着「不存在」,运行时照实报错,同设置页「只校验改过的」的做法。名字不是字符串这类格式问题由各组自己的
|
||||
检查先报。"""
|
||||
providers = None
|
||||
for (group, key), (label, blank) in _provider_fields().items():
|
||||
sent = values.get(group)
|
||||
if not isinstance(sent, dict) or (name := _model_text(sent.get(key))) is None:
|
||||
continue
|
||||
row = await session.get(Setting, group)
|
||||
saved = row.value if row is not None and isinstance(row.value, dict) else {}
|
||||
if _model_text(saved.get(key)) == name:
|
||||
continue
|
||||
if providers is None:
|
||||
providers = await all_providers(session)
|
||||
try:
|
||||
named_provider(providers, name)
|
||||
except ProviderNotConfigured as e:
|
||||
return f"{group}.{key}", f"「{label}」填写的{e.reason}。{e.hint};{blank}"
|
||||
return None
|
||||
|
||||
|
||||
@router.put("/settings")
|
||||
async def put_settings(
|
||||
payload: SettingsIn, session: AsyncSession = Depends(get_session)
|
||||
@@ -735,6 +781,8 @@ async def put_settings(
|
||||
raise HTTPException(422, problem)
|
||||
# 存规范化之后的:末尾带 / 的地址拼链接会多出一个斜杠
|
||||
payload.values[NOTIFY_GROUP] = notify_values(payload.values[NOTIFY_GROUP])
|
||||
if found := await _provider_problem(session, payload.values):
|
||||
raise CodedHTTPException(422, found[1], PROVIDER_INVALID, field=found[0])
|
||||
for key, value in payload.values.items():
|
||||
if key in (TRUST_KEY, "limits", JUDGE_SPEND, VERIFY_SPEND):
|
||||
# 信任三档只走 PUT /api/tools/trust:设置页整组回写时带着一份旧的,
|
||||
|
||||
@@ -21,6 +21,8 @@ router = APIRouter(prefix="/api/workflows", tags=["workflows"])
|
||||
_GONE = "工作流不存在,可能已被删除"
|
||||
_LEVELS = ("published", "governed")
|
||||
_BAD_LEVEL = "发布级别只能是「已发布」或「受管」"
|
||||
#: 自动修复之后剩下的只有点名接入的问题、人又要交给助手时的说明
|
||||
PROVIDER_LEFT = "剩下的问题是节点指定的模型接入不存在或已停用,助手无法代为处理:请在「设置 → 模型接入」中核对,或在节点中换一个接入"
|
||||
|
||||
|
||||
class WorkflowIn(BaseModel):
|
||||
@@ -335,6 +337,7 @@ async def publish_workflow(
|
||||
|
||||
checkers = await _sql_checkers(session, spec)
|
||||
issues = [i.model_dump() for i in publish_issues(spec, level=payload.level, checkers=checkers)]
|
||||
issues += await _provider_issues(session, spec, level="error")
|
||||
if any(i["level"] == "error" for i in issues):
|
||||
return {"ok": False, "level": payload.level, "version": version, "issues": issues}
|
||||
|
||||
@@ -421,6 +424,22 @@ async def _sql_checkers(session: AsyncSession, spec: GraphSpec) -> dict[str, Any
|
||||
return await load_checkers_by_name(session, graph_sources(spec.nodes))
|
||||
|
||||
|
||||
async def _provider_issues(session: AsyncSession, spec: GraphSpec, *, level: str) -> list[dict[str, Any]]:
|
||||
"""节点点名的模型接入不存在或已停用(engine/provider_refs.py):画布校验给 warning,发布、发布前检查、自动修复的
|
||||
复核给 error、挡住发布。要读接入表,和 _sql_checkers 一样由接口这一层查,门禁本身不查库。没有对应的修复。"""
|
||||
from app.engine.provider_refs import load_issues
|
||||
|
||||
return [i.model_dump() for i in await load_issues(session, spec, level=level)]
|
||||
|
||||
|
||||
def _with_provider_issues(out: dict[str, Any], issues: list[dict[str, Any]], key: str) -> dict[str, Any]:
|
||||
"""发布前检查(issues)、自动修复(remaining)的结果补上点名接入的问题:都是 error,有就不能发布。"""
|
||||
if issues:
|
||||
out[key] = [*out[key], *issues]
|
||||
out["ok"] = False
|
||||
return out
|
||||
|
||||
|
||||
def _unreadable(level: str, e: Exception) -> dict[str, Any]:
|
||||
return {"level": level, "ok": False, "fixes": [], "issues": [
|
||||
{"level": "error", "node_id": None, "edge_id": None, "field": None, "code": None, "fix": None,
|
||||
@@ -443,8 +462,8 @@ async def publish_check(
|
||||
except Exception as e: # noqa: BLE001
|
||||
return _unreadable(payload.level, e)
|
||||
versions, cards = await _gate_context(session, spec)
|
||||
return check(spec, level=payload.level, versions=versions, cards=cards,
|
||||
checkers=await _sql_checkers(session, spec))
|
||||
out = check(spec, level=payload.level, versions=versions, cards=cards, checkers=await _sql_checkers(session, spec))
|
||||
return _with_provider_issues(out, await _provider_issues(session, spec, level="error"), "issues")
|
||||
|
||||
|
||||
@router.post("/{workflow_id}/autofix")
|
||||
@@ -475,9 +494,11 @@ async def autofix(
|
||||
# 人在选项里选了「交给 Copilot」(handoff),和点「交给 Copilot」是一回事
|
||||
if payload.assist or out["handoff"]:
|
||||
if out["ok"]:
|
||||
out["assist"] = {"ok": True, "summary": "修复后已没有阻止发布的错误,未再交给助手",
|
||||
# 剩下的只有点名接入的问题时不这么说:助手改不了接入表,也不知道该换成哪个接入
|
||||
lost = await _provider_issues(session, GraphSpec.model_validate(out["graph"]), level="error")
|
||||
out["assist"] = {"ok": not lost, "summary": PROVIDER_LEFT if lost else "修复后已没有阻止发布的错误,未再交给助手",
|
||||
"questions": []}
|
||||
return out
|
||||
return _with_provider_issues(out, lost, "remaining")
|
||||
from app.api.copilot import assist_publish_fix
|
||||
|
||||
helped = await assist_publish_fix(session, out["graph"], level=payload.level,
|
||||
@@ -493,7 +514,8 @@ async def autofix(
|
||||
out.update(graph=fixed, remaining=after["remaining"], fixes=after["fixes"], ok=after["ok"])
|
||||
elif helped["reason"]:
|
||||
out["rejected"].append({"fix_id": "assist", "reason": helped["reason"]})
|
||||
return out
|
||||
return _with_provider_issues(out, await _provider_issues(session, GraphSpec.model_validate(out["graph"]),
|
||||
level="error"), "remaining")
|
||||
|
||||
|
||||
class ValidateIn(BaseModel):
|
||||
@@ -501,14 +523,17 @@ class ValidateIn(BaseModel):
|
||||
|
||||
|
||||
@router.post("/validate")
|
||||
async def validate(payload: ValidateIn) -> dict[str, Any]:
|
||||
"""画布上的实时校验。前端每次改动都会调它来标红。"""
|
||||
async def validate(payload: ValidateIn, session: AsyncSession = Depends(get_session)) -> dict[str, Any]:
|
||||
"""画布上的实时校验。前端每次改动都会调它来标红。
|
||||
|
||||
点名的模型接入不存在或已停用只给警告:探索运行仍可发起,执行到那个节点时照实报错;发布时才挡住。"""
|
||||
try:
|
||||
spec = GraphSpec.model_validate(payload.graph)
|
||||
except Exception as e: # noqa: BLE001
|
||||
return {"ok": False, "issues": [{"level": "error", "message": f"无法解析工作流结构:{graph_error(e)}"}]}
|
||||
result = validate_graph(spec)
|
||||
return result.model_dump()
|
||||
result = validate_graph(spec).model_dump()
|
||||
result["issues"] += await _provider_issues(session, spec, level="warning")
|
||||
return result
|
||||
|
||||
|
||||
@router.post("/variables")
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
"""工作流里点名的模型接入:不存在或已停用的,画布校验、发布前检查、发布时照实提出来。
|
||||
|
||||
运行时点名的接入不存在或已停用直接报错(providers/factory.named_provider),以前是悄悄换成演示模型。这类问题发布前
|
||||
就该看得到:画布校验给警告(探索运行仍可发起,执行到那里报错),发布、发布前检查、自动修复的复核给错误、挡住
|
||||
发布——正式运行到那里一定失败,或者少一块核查。
|
||||
|
||||
查的是执行时真会去取模型的那几处,取法和执行器一致:
|
||||
- 模型调用、Agent、报告撰写、多 Agent 协作的接入;条件分支只在「让模型分类」时、结构校验只在「让模型修复」时、
|
||||
知识检索只在「让模型重排」时才取模型。这几类节点没写自己的接入时取全图默认(NodeContext.cfg);
|
||||
- 多 Agent 协作各成员的接入;报告撰写交给模型裁判结论句时,「结论句裁判」里的接入;
|
||||
- 双盲复算、反方审稿的接入(核查模型):只读节点自己的配置,不取全图默认;
|
||||
- 全图默认的接入:有节点会取到它时才查。
|
||||
|
||||
接入表要读库,门禁本身不查库(governance.lint_for_publish),所以由接口这一层读出接入表交给 issues(),同数据目录
|
||||
SQL 检查器的做法。设置里点名接入的几项由 api/settings.py 在保存时查。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Callable
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.engine.labels import field_label, type_label
|
||||
from app.engine.schema import CLAIMS_JUDGE, GraphNode, GraphSpec, NodeType, ValidationIssue
|
||||
from app.providers.factory import ProviderNotConfigured, list_providers, named_provider
|
||||
|
||||
|
||||
def _cfg(spec: GraphSpec, node: GraphNode, key: str, default: Any = None) -> Any:
|
||||
"""和 NodeContext.cfg 同一个规则:节点上没写(None、空串)就取全图默认。"""
|
||||
value = node.config.get(key)
|
||||
return spec.defaults.get(key, default) if value in (None, "") else value
|
||||
|
||||
|
||||
#: 按 NodeContext.cfg 取接入的节点 → 执行时真会去取模型的条件(None 是总会取)
|
||||
_TAKES_MODEL: dict[NodeType, Callable[[GraphSpec, GraphNode], bool] | None] = {
|
||||
NodeType.LLM: None,
|
||||
NodeType.AGENT: None,
|
||||
NodeType.REPORT: None,
|
||||
NodeType.SUPERVISOR: None,
|
||||
NodeType.BRANCH: lambda spec, node: _cfg(spec, node, "mode", "expression") == "llm",
|
||||
NodeType.VALIDATE: lambda spec, node: bool(_cfg(spec, node, "repair_with_llm", True)),
|
||||
NodeType.RETRIEVE: lambda spec, node: _cfg(spec, node, "rerank", "off") == "model",
|
||||
}
|
||||
#: 核查节点:接入只读节点自己的配置(verify_model_spec),图级默认的接入是写作模型的
|
||||
_VERIFY = {NodeType.DUAL_CHECK: "的盲算模型", NodeType.CHALLENGE: "的反方模型"}
|
||||
|
||||
|
||||
def _name(value: Any) -> str | None:
|
||||
return value.strip() or None if isinstance(value, str) else None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ProviderRef:
|
||||
"""图里点名接入的一处:落在哪个节点(全图默认为 None)、哪个字段,点的名字,句子里怎么称呼这一处。"""
|
||||
|
||||
node_id: str | None
|
||||
field: str
|
||||
name: str
|
||||
who: str
|
||||
|
||||
|
||||
def provider_refs(spec: GraphSpec) -> list[ProviderRef]:
|
||||
"""图里执行时会用到的接入点名,按画布上节点的顺序,全图默认排最后。"""
|
||||
out: list[ProviderRef] = []
|
||||
falls_back = False
|
||||
for node in spec.nodes:
|
||||
cfg = node.config
|
||||
who = f"「{node.title}」({type_label(node.type)})"
|
||||
if node.type in _VERIFY:
|
||||
if name := _name(cfg.get("provider")):
|
||||
out.append(ProviderRef(node.id, "provider", name, who + _VERIFY[node.type]))
|
||||
continue
|
||||
if node.type not in _TAKES_MODEL:
|
||||
continue
|
||||
cond = _TAKES_MODEL[node.type]
|
||||
if cond is None or cond(spec, node):
|
||||
if name := _name(cfg.get("provider")):
|
||||
out.append(ProviderRef(node.id, "provider", name, who))
|
||||
else:
|
||||
falls_back = True
|
||||
if node.type == NodeType.SUPERVISOR and isinstance(cfg.get("agents"), list):
|
||||
for i, member in enumerate(cfg["agents"]):
|
||||
if isinstance(member, dict) and (name := _name(member.get("provider"))):
|
||||
label = _name(member.get("name")) or f"第 {i + 1} 位"
|
||||
out.append(ProviderRef(node.id, f"agents[{i}].provider", name, f"{who}的成员「{label}」"))
|
||||
if node.type == NodeType.REPORT and _cfg(spec, node, "claims", "off") == CLAIMS_JUDGE \
|
||||
and isinstance(cfg.get("judge"), dict) and (name := _name(cfg["judge"].get("provider"))):
|
||||
out.append(ProviderRef(node.id, "judge.provider", name, f"{who}的{field_label('judge')}"))
|
||||
if falls_back and (name := _name(spec.defaults.get("provider"))):
|
||||
out.append(ProviderRef(None, "defaults.provider", name, field_label("defaults")))
|
||||
return out
|
||||
|
||||
|
||||
def issues(refs: list[ProviderRef], providers: list[Any], *, level: str) -> list[ValidationIssue]:
|
||||
"""点名的接入不存在或已停用的每一处一条问题。providers 是接入表(factory.list_providers),level 由调用方定:
|
||||
画布校验 warning,发布一类 error。说法取自 named_provider,和运行时报的原因、提示同一句。"""
|
||||
out: list[ValidationIssue] = []
|
||||
for ref in refs:
|
||||
try:
|
||||
named_provider(providers, ref.name)
|
||||
except ProviderNotConfigured as e:
|
||||
out.append(ValidationIssue(level=level, node_id=ref.node_id, field=ref.field, # type: ignore[arg-type]
|
||||
code=f"provider.{e.code}", message=f"{ref.who}指定的{e.reason}。{e.hint}"))
|
||||
return out
|
||||
|
||||
|
||||
async def load_issues(session: AsyncSession, spec: GraphSpec, *, level: str) -> list[ValidationIssue]:
|
||||
"""读接入表、查一遍。图里没点名任何接入时不读库。"""
|
||||
refs = provider_refs(spec)
|
||||
if not refs:
|
||||
return []
|
||||
return issues(refs, await list_providers(session), level=level)
|
||||
@@ -0,0 +1,281 @@
|
||||
"""保存设置、校验画布、发布时就查出点名了不存在或已停用的模型接入(运行时它们会照实报错,见 test_provider_ref_runtime)。
|
||||
|
||||
- PUT /api/settings:引用接入的几项(核查模型、证据裁判、门控模型、助手)写了不存在或已停用的接入 → 422,带 field、
|
||||
code 和设置页上的叫法;一组出错整次不落库。留空照旧表示沿用上一级;存着的值没改(写好之后接入才被删、停用)不挡
|
||||
这次保存别的项——设置页标着「不存在」,运行时照实报错;
|
||||
- 画布校验:节点点名的接入不存在或已停用给警告,说清哪个节点、哪个接入;发布前检查、发布、自动修复的复核给错误,
|
||||
挡住发布。只查执行时真会去取模型的那几处(分支「让模型分类」、校验「让模型修复」、检索「让模型重排」时才算)。
|
||||
|
||||
接入名、节点名都是虚构的。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
from sqlalchemy import delete
|
||||
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import Provider, Setting, Workflow, WorkflowVersion
|
||||
from app.main import app
|
||||
|
||||
LIVE, OFF, GONE = "核对用接入", "核对用停用接入", "核对用接人"
|
||||
SETTING_KEYS = ["verify", "judge", "copilot", "run"]
|
||||
|
||||
|
||||
async def _clean() -> None:
|
||||
async with SessionLocal() as session:
|
||||
await session.execute(delete(Setting).where(Setting.key.in_(SETTING_KEYS)))
|
||||
await session.execute(delete(Provider).where(Provider.name.in_([LIVE, OFF])))
|
||||
await session.commit()
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
async def providers():
|
||||
"""一个启用、一个停用的接入(都是演示类,不联网);用完删掉,免得改变同一进程里别的测试「没点名」时的回退。"""
|
||||
await _clean()
|
||||
async with SessionLocal() as session:
|
||||
live = Provider(name=LIVE, kind="mock", models=[{"id": "核对模型"}], default_model="核对模型", enabled=True,
|
||||
extra={})
|
||||
off = Provider(name=OFF, kind="mock", models=[{"id": "核对停用模型"}], default_model="核对停用模型",
|
||||
enabled=False, extra={})
|
||||
session.add_all([live, off])
|
||||
await session.commit()
|
||||
ids = {LIVE: live.id, OFF: off.id}
|
||||
yield ids
|
||||
await _clean()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def client():
|
||||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
|
||||
yield c
|
||||
|
||||
|
||||
async def stored(key: str):
|
||||
async with SessionLocal() as session:
|
||||
row = await session.get(Setting, key)
|
||||
return row.value if row else None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 保存设置
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
#: (组, 键, 设置页上的叫法)
|
||||
FIELDS = [
|
||||
("verify", "provider", "核查模型 · 接入"),
|
||||
("judge", "provider", "裁判模型 · 接入"),
|
||||
("run", "tool_gate_provider", "门控模型 · 接入"),
|
||||
("copilot", "provider", "助手模型 · 接入"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("group,key,label", FIELDS)
|
||||
async def test_saving_a_provider_that_does_not_exist_is_refused_with_the_field(client, group, key, label):
|
||||
r = await client.put("/api/settings", json={"values": {group: {key: GONE}}})
|
||||
assert r.status_code == 422, r.text
|
||||
body = r.json()
|
||||
assert body["code"] == "settings_provider_invalid" and body["field"] == f"{group}.{key}"
|
||||
assert body["detail"].startswith(f"「{label}」填写的模型接入「{GONE}」不存在。"), body["detail"]
|
||||
assert "请在「设置 → 模型接入」中核对名称,或换一个接入" in body["detail"] and "留空表示" in body["detail"]
|
||||
assert await stored(group) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("group,key,label", FIELDS)
|
||||
async def test_saving_a_disabled_provider_is_refused_too(client, group, key, label):
|
||||
r = await client.put("/api/settings", json={"values": {group: {key: OFF}}})
|
||||
assert r.status_code == 422, r.text
|
||||
body = r.json()
|
||||
assert body["field"] == f"{group}.{key}"
|
||||
assert body["detail"].startswith(f"「{label}」填写的模型接入「{OFF}」已停用。"), body["detail"]
|
||||
assert "请在「设置 → 模型接入」中启用它,或换一个接入" in body["detail"]
|
||||
assert await stored(group) is None
|
||||
|
||||
|
||||
async def test_one_bad_group_keeps_the_whole_request_out_of_the_database(client):
|
||||
r = await client.put("/api/settings", json={"values": {
|
||||
"run": {"default_memory_scope": "门店", "tool_gate_provider": LIVE},
|
||||
"judge": {"provider": LIVE, "model": None},
|
||||
"verify": {"provider": GONE},
|
||||
}})
|
||||
assert r.status_code == 422 and r.json()["field"] == "verify.provider"
|
||||
assert await stored("run") is None and await stored("judge") is None and await stored("verify") is None
|
||||
|
||||
|
||||
async def test_blank_means_inherit_and_an_enabled_provider_is_taken_by_name_or_id(client, providers):
|
||||
r = await client.put("/api/settings", json={"values": {
|
||||
"verify": {"provider": None, "model": "核对模型"},
|
||||
"judge": {"provider": " "},
|
||||
"run": {"tool_gate_provider": ""},
|
||||
}})
|
||||
assert r.status_code == 200, r.text
|
||||
r = await client.put("/api/settings", json={"values": {
|
||||
"verify": {"provider": LIVE}, "judge": {"provider": providers[LIVE]},
|
||||
"run": {"tool_gate_provider": LIVE}, "copilot": {"provider": LIVE, "model": ""},
|
||||
}})
|
||||
assert r.status_code == 200, r.text
|
||||
got = r.json()
|
||||
assert got["verify"]["provider"] == LIVE and got["judge"]["provider"] == providers[LIVE]
|
||||
assert got["run"]["tool_gate_provider"] == LIVE
|
||||
|
||||
|
||||
async def test_a_stored_provider_removed_later_does_not_block_saving_the_rest_of_its_group(client):
|
||||
"""写好之后接入才被删掉:存着的值没改,这次保存别的项照常(运行时照实报错,设置页标着「不存在」)。
|
||||
改成另一个不存在的名字仍然拦下。"""
|
||||
async with SessionLocal() as session:
|
||||
session.add(Setting(key="verify", value={"provider": GONE, "daily_max_usd": 2.0}))
|
||||
await session.commit()
|
||||
r = await client.put("/api/settings", json={"values": {"verify": {"provider": f" {GONE} ", "daily_max_usd": 3}}})
|
||||
assert r.status_code == 200, r.text
|
||||
assert (await stored("verify"))["daily_max_usd"] == 3
|
||||
r = await client.put("/api/settings", json={"values": {"verify": {"provider": "另一个不存在的接入"}}})
|
||||
assert r.status_code == 422 and r.json()["field"] == "verify.provider"
|
||||
|
||||
|
||||
async def test_format_problems_are_still_reported_first(client):
|
||||
r = await client.put("/api/settings", json={"values": {"verify": {"provider": 3}}})
|
||||
assert r.status_code == 422
|
||||
assert "需要填写模型接入的名称或模型 ID" in r.json()["detail"]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 画布校验、发布前检查、发布
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def node(nid, ntype, label=None, **config):
|
||||
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": label or nid, "config": config}}
|
||||
|
||||
|
||||
def chain(*nodes, defaults=None):
|
||||
graph = {"nodes": list(nodes), "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]}
|
||||
if defaults:
|
||||
graph["defaults"] = defaults
|
||||
return graph
|
||||
|
||||
|
||||
def writer(ref: str | None, **extra) -> dict:
|
||||
return chain(node("start", "input"),
|
||||
node("write", "llm", "写周报", **({"provider": ref} if ref else {}), prompt="写周报", **extra),
|
||||
node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}]))
|
||||
|
||||
|
||||
def provider_issues(result: dict) -> list[dict]:
|
||||
return [i for i in result["issues"] if str(i.get("code") or "").startswith("provider.")]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ref,code,state", [(GONE, "provider.missing", "不存在"), (OFF, "provider.disabled", "已停用")])
|
||||
async def test_validation_warns_about_a_node_naming_a_broken_provider(client, ref, code, state):
|
||||
r = await client.post("/api/workflows/validate", json={"graph": writer(ref)})
|
||||
assert r.status_code == 200, r.text
|
||||
[issue] = provider_issues(r.json())
|
||||
assert issue["level"] == "warning" and issue["code"] == code
|
||||
assert (issue["node_id"], issue["field"]) == ("write", "provider")
|
||||
assert issue["message"].startswith(f"「写周报」(模型调用)指定的模型接入「{ref}」{state}。"), issue["message"]
|
||||
assert r.json()["ok"] is True, "只是警告:探索运行仍可发起,执行到那里照实报错"
|
||||
|
||||
|
||||
async def test_validation_is_quiet_for_an_enabled_provider_or_none(client, providers):
|
||||
for ref in (LIVE, providers[LIVE], None):
|
||||
r = await client.post("/api/workflows/validate", json={"graph": writer(ref)})
|
||||
assert provider_issues(r.json()) == [], ref
|
||||
|
||||
|
||||
async def test_every_place_a_graph_names_a_provider_is_checked(client):
|
||||
graph = chain(
|
||||
node("start", "input"),
|
||||
node("team", "supervisor", "协作", provider=OFF, agents=[{"name": "分析师", "provider": GONE, "system": "分析"},
|
||||
{"provider": LIVE, "system": "复核"}]),
|
||||
node("write", "report", "写周报", instructions="写周报", claims="judge", judge={"provider": GONE}),
|
||||
node("dual", "dual_check", "复核销售额", provider=GONE),
|
||||
node("chal", "challenge", "审稿", report="write", provider=OFF),
|
||||
node("ask", "llm", "提问", prompt="总结"), # 没写接入:取全图默认
|
||||
node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}]),
|
||||
defaults={"provider": GONE},
|
||||
)
|
||||
r = await client.post("/api/workflows/validate", json={"graph": graph})
|
||||
got = {(i["node_id"], i["field"]): i for i in provider_issues(r.json())}
|
||||
assert set(got) == {("team", "provider"), ("team", "agents[0].provider"), ("write", "judge.provider"),
|
||||
("dual", "provider"), ("chal", "provider"), (None, "defaults.provider")}
|
||||
assert got[("team", "agents[0].provider")]["message"].startswith(
|
||||
f"「协作」(多 Agent 协作)的成员「分析师」指定的模型接入「{GONE}」不存在")
|
||||
assert got[("write", "judge.provider")]["message"].startswith(
|
||||
f"「写周报」(报告撰写)的结论句裁判指定的模型接入「{GONE}」不存在")
|
||||
assert got[("dual", "provider")]["message"].startswith(f"「复核销售额」(双盲复算)的盲算模型指定的模型接入「{GONE}」")
|
||||
assert got[("chal", "provider")]["message"].startswith(f"「审稿」(反方审稿)的反方模型指定的模型接入「{OFF}」已停用")
|
||||
assert got[(None, "defaults.provider")]["message"].startswith(f"工作流默认设置指定的模型接入「{GONE}」不存在")
|
||||
|
||||
|
||||
async def test_only_places_that_actually_call_a_model_are_checked(client):
|
||||
"""分支按表达式判断、校验不让模型修复、检索不让模型重排时根本不取模型;报告不交给模型裁判时裁判的接入用不上;
|
||||
每个节点都写了自己的接入时,全图默认的接入没人用。"""
|
||||
graph = chain(
|
||||
node("start", "input"),
|
||||
node("route", "branch", "分流", mode="expression", provider=GONE, cases=[{"key": "a", "condition": "true"}]),
|
||||
node("check", "validate", "校验", repair_with_llm=False, provider=GONE, schema={"type": "object"}),
|
||||
node("look", "retrieve", "检索", rerank="off", provider=GONE),
|
||||
node("write", "report", "写周报", instructions="写周报", provider=LIVE, judge={"provider": GONE}),
|
||||
node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}]),
|
||||
defaults={"provider": GONE},
|
||||
)
|
||||
r = await client.post("/api/workflows/validate", json={"graph": graph})
|
||||
assert provider_issues(r.json()) == []
|
||||
graph["nodes"][1]["data"]["config"]["mode"] = "llm"
|
||||
graph["nodes"][3]["data"]["config"]["rerank"] = "model"
|
||||
r = await client.post("/api/workflows/validate", json={"graph": graph})
|
||||
assert {i["node_id"] for i in provider_issues(r.json())} == {"route", "look"}
|
||||
# 分类的分支不写自己的接入:取全图默认,默认里的接入这才有人用
|
||||
del graph["nodes"][1]["data"]["config"]["provider"]
|
||||
r = await client.post("/api/workflows/validate", json={"graph": graph})
|
||||
assert {(i["node_id"], i["field"]) for i in provider_issues(r.json())} == {("look", "provider"),
|
||||
(None, "defaults.provider")}
|
||||
|
||||
|
||||
async def _workflow(graph: dict) -> str:
|
||||
async with SessionLocal() as session:
|
||||
wf = Workflow(name=f"门店周报-{uuid.uuid4().hex[:6]}", graph=graph, version=1)
|
||||
session.add(wf)
|
||||
await session.flush()
|
||||
session.add(WorkflowVersion(workflow_id=wf.id, version=1, graph=graph))
|
||||
await session.commit()
|
||||
return wf.id
|
||||
|
||||
|
||||
@pytest.mark.parametrize("level", ["published", "governed"])
|
||||
async def test_publish_check_and_publish_stop_at_a_broken_provider(client, level):
|
||||
wf = await _workflow(writer(GONE))
|
||||
r = await client.post(f"/api/workflows/{wf}/publish-check", json={"level": level})
|
||||
assert r.status_code == 200, r.text
|
||||
check = r.json()
|
||||
[issue] = provider_issues(check)
|
||||
assert issue["level"] == "error" and issue["node_id"] == "write" and issue["fix"] is None
|
||||
assert check["ok"] is False
|
||||
r = await client.post(f"/api/workflows/{wf}/publish", json={"level": level})
|
||||
body = r.json()
|
||||
assert body["ok"] is False and [i["code"] for i in provider_issues(body)] == ["provider.missing"]
|
||||
async with SessionLocal() as session:
|
||||
row = await session.get(Workflow, wf)
|
||||
assert row.published_version is None and row.status == "draft"
|
||||
|
||||
|
||||
async def test_publish_check_reads_the_unsaved_canvas_and_autofix_keeps_the_problem(client):
|
||||
wf = await _workflow(writer(None))
|
||||
r = await client.post(f"/api/workflows/{wf}/publish-check", json={"level": "published", "graph": writer(OFF)})
|
||||
assert r.json()["ok"] is False and [i["code"] for i in provider_issues(r.json())] == ["provider.disabled"]
|
||||
r = await client.post(f"/api/workflows/{wf}/autofix", json={"level": "published", "graph": writer(OFF)})
|
||||
assert r.status_code == 200, r.text
|
||||
out = r.json()
|
||||
assert out["ok"] is False
|
||||
assert [(i["code"], i["level"]) for i in provider_issues({"issues": out["remaining"]})] == [
|
||||
("provider.disabled", "error")]
|
||||
# 换成启用的接入:不再拦
|
||||
r = await client.post(f"/api/workflows/{wf}/publish-check", json={"level": "published", "graph": writer(LIVE)})
|
||||
assert provider_issues(r.json()) == [] and r.json()["ok"] is True
|
||||
|
||||
|
||||
async def test_publishing_a_clean_graph_is_unaffected(client):
|
||||
wf = await _workflow(writer(LIVE))
|
||||
r = await client.post(f"/api/workflows/{wf}/publish", json={"level": "published"})
|
||||
assert r.json()["ok"] is True, r.text
|
||||
@@ -21,7 +21,7 @@ from app.api.settings import (
|
||||
verify_settings,
|
||||
)
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import Setting
|
||||
from app.db.models import Provider, Setting
|
||||
from app.engine import spend
|
||||
from app.engine.judge import JUDGE_DEFAULTS, MAX_TOKENS
|
||||
from app.main import app
|
||||
@@ -64,7 +64,20 @@ async def test_settings_expose_the_verify_group_with_defaults():
|
||||
assert spend.SPEND_KEY not in got
|
||||
|
||||
|
||||
async def test_a_full_group_round_trips_and_null_stays_unlimited():
|
||||
@pytest.fixture
|
||||
async def house():
|
||||
"""设置里点名的接入要真的存在、启用着(保存时会查):建一个演示类的,用完删掉。"""
|
||||
async with SessionLocal() as session:
|
||||
session.add(Provider(name="house", kind="mock", models=[{"id": "verify-model"}], default_model="verify-model",
|
||||
enabled=True, extra={}))
|
||||
await session.commit()
|
||||
yield "house"
|
||||
async with SessionLocal() as session:
|
||||
await session.execute(delete(Provider).where(Provider.name == "house"))
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def test_a_full_group_round_trips_and_null_stays_unlimited(house):
|
||||
value = {
|
||||
"provider": "house", "model": "verify-model",
|
||||
"dual_check": {"enabled": False, "tolerance": 0, "max_queries": None, "timeout_s": None},
|
||||
|
||||
Reference in New Issue
Block a user