diff --git a/backend/app/api/settings.py b/backend/app/api/settings.py index 566fae74..d910029d 100644 --- a/backend/app/api/settings.py +++ b/backend/app/api/settings.py @@ -10,18 +10,21 @@ from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from app.api import health +from app.api.coded import CodedHTTPException from app.core.errors import explain, not_configured, raw from app.core.config import settings as app_settings from app.core.crypto import encrypt, mask from app.db.base import get_session from app.db.models import Provider, Setting from app.engine.judge import JUDGE_DEFAULTS, MAX_TOKENS as JUDGE_MAX_TOKENS -from app.engine.judge import SETTING_GROUP as JUDGE_GROUP, SPEND_KEY as JUDGE_SPEND +from app.engine.judge import SETTING_GROUP as JUDGE_GROUP, SETTING_LABEL as JUDGE_LABEL, SPEND_KEY as JUDGE_SPEND from app.engine.judge import daily_spend, judge_model_spec, judge_settings from app.engine.judge import sanitize_settings as judge_values, settings_problem as judge_problem from app.engine.spend import SPEND_KEY as VERIFY_SPEND, daily_spend as verify_daily_spend from app.providers import catalog -from app.providers.factory import ModelSpec, ProviderNotConfigured, _resolve_key, build_chat_model +from app.providers.factory import ( + ModelSpec, ProviderNotConfigured, _resolve_key, build_chat_model, list_providers as all_providers, named_provider, +) router = APIRouter(prefix="/api", tags=["settings"]) @@ -719,6 +722,49 @@ class SettingsIn(BaseModel): values: dict[str, Any] +#: 点名的模型接入不存在或已停用时 PUT /api/settings 回 422 的机读码,field 是出错的那一项(verify.provider……) +PROVIDER_INVALID = "settings_provider_invalid" + + +def _provider_fields() -> dict[tuple[str, str], tuple[str, str]]: + """设置里点名模型接入的项:(组, 键) → (设置页上的叫法, 留空时用什么)。门控、裁判、核查的叫法和设置页一致;助手的 + 模型由助手面板单独设定(PUT /api/copilot/model),这里只防经设置接口写进去的那一份。""" + from app.api.copilot import COPILOT_SETTING_KEY # 在这里导入:copilot 模块很大,设置模块被引擎各处导入 + + return { + ("run", "tool_gate_provider"): ("门控模型 · 接入", "留空表示使用默认接入的默认模型"), + (JUDGE_GROUP, "provider"): (JUDGE_LABEL["provider"], "留空表示不指定"), + (VERIFY_GROUP, "provider"): (VERIFY_SETTING_LABEL["provider"], "留空表示沿用证据裁判模型"), + (COPILOT_SETTING_KEY, "provider"): ("助手模型 · 接入", "留空表示不指定"), + } + + +async def _provider_problem(session: AsyncSession, values: dict[str, Any]) -> tuple[str, str] | None: + """这次写进来的设置里点名的模型接入有没有问题:(字段, 说明),没问题返回 None。说明和运行时报的原因、提示同一句 + (factory.named_provider)。 + + 以前点名的接入不存在时运行悄悄换成演示模型,名字写错一个字就由演示模型写报告、做核查,现在运行时照实报错;保存时 + 先拦一道。留空照旧表示沿用上一级。只查这次改了的:存着的值没改(写好之后接入才被删、停用)不挡这次保存别的项—— + 设置页标着「不存在」,运行时照实报错,同设置页「只校验改过的」的做法。名字不是字符串这类格式问题由各组自己的 + 检查先报。""" + providers = None + for (group, key), (label, blank) in _provider_fields().items(): + sent = values.get(group) + if not isinstance(sent, dict) or (name := _model_text(sent.get(key))) is None: + continue + row = await session.get(Setting, group) + saved = row.value if row is not None and isinstance(row.value, dict) else {} + if _model_text(saved.get(key)) == name: + continue + if providers is None: + providers = await all_providers(session) + try: + named_provider(providers, name) + except ProviderNotConfigured as e: + return f"{group}.{key}", f"「{label}」填写的{e.reason}。{e.hint};{blank}" + return None + + @router.put("/settings") async def put_settings( payload: SettingsIn, session: AsyncSession = Depends(get_session) @@ -735,6 +781,8 @@ async def put_settings( raise HTTPException(422, problem) # 存规范化之后的:末尾带 / 的地址拼链接会多出一个斜杠 payload.values[NOTIFY_GROUP] = notify_values(payload.values[NOTIFY_GROUP]) + if found := await _provider_problem(session, payload.values): + raise CodedHTTPException(422, found[1], PROVIDER_INVALID, field=found[0]) for key, value in payload.values.items(): if key in (TRUST_KEY, "limits", JUDGE_SPEND, VERIFY_SPEND): # 信任三档只走 PUT /api/tools/trust:设置页整组回写时带着一份旧的, diff --git a/backend/app/api/workflows.py b/backend/app/api/workflows.py index e24a6be8..5bec963f 100644 --- a/backend/app/api/workflows.py +++ b/backend/app/api/workflows.py @@ -21,6 +21,8 @@ router = APIRouter(prefix="/api/workflows", tags=["workflows"]) _GONE = "工作流不存在,可能已被删除" _LEVELS = ("published", "governed") _BAD_LEVEL = "发布级别只能是「已发布」或「受管」" +#: 自动修复之后剩下的只有点名接入的问题、人又要交给助手时的说明 +PROVIDER_LEFT = "剩下的问题是节点指定的模型接入不存在或已停用,助手无法代为处理:请在「设置 → 模型接入」中核对,或在节点中换一个接入" class WorkflowIn(BaseModel): @@ -335,6 +337,7 @@ async def publish_workflow( checkers = await _sql_checkers(session, spec) issues = [i.model_dump() for i in publish_issues(spec, level=payload.level, checkers=checkers)] + issues += await _provider_issues(session, spec, level="error") if any(i["level"] == "error" for i in issues): return {"ok": False, "level": payload.level, "version": version, "issues": issues} @@ -421,6 +424,22 @@ async def _sql_checkers(session: AsyncSession, spec: GraphSpec) -> dict[str, Any return await load_checkers_by_name(session, graph_sources(spec.nodes)) +async def _provider_issues(session: AsyncSession, spec: GraphSpec, *, level: str) -> list[dict[str, Any]]: + """节点点名的模型接入不存在或已停用(engine/provider_refs.py):画布校验给 warning,发布、发布前检查、自动修复的 + 复核给 error、挡住发布。要读接入表,和 _sql_checkers 一样由接口这一层查,门禁本身不查库。没有对应的修复。""" + from app.engine.provider_refs import load_issues + + return [i.model_dump() for i in await load_issues(session, spec, level=level)] + + +def _with_provider_issues(out: dict[str, Any], issues: list[dict[str, Any]], key: str) -> dict[str, Any]: + """发布前检查(issues)、自动修复(remaining)的结果补上点名接入的问题:都是 error,有就不能发布。""" + if issues: + out[key] = [*out[key], *issues] + out["ok"] = False + return out + + def _unreadable(level: str, e: Exception) -> dict[str, Any]: return {"level": level, "ok": False, "fixes": [], "issues": [ {"level": "error", "node_id": None, "edge_id": None, "field": None, "code": None, "fix": None, @@ -443,8 +462,8 @@ async def publish_check( except Exception as e: # noqa: BLE001 return _unreadable(payload.level, e) versions, cards = await _gate_context(session, spec) - return check(spec, level=payload.level, versions=versions, cards=cards, - checkers=await _sql_checkers(session, spec)) + out = check(spec, level=payload.level, versions=versions, cards=cards, checkers=await _sql_checkers(session, spec)) + return _with_provider_issues(out, await _provider_issues(session, spec, level="error"), "issues") @router.post("/{workflow_id}/autofix") @@ -475,9 +494,11 @@ async def autofix( # 人在选项里选了「交给 Copilot」(handoff),和点「交给 Copilot」是一回事 if payload.assist or out["handoff"]: if out["ok"]: - out["assist"] = {"ok": True, "summary": "修复后已没有阻止发布的错误,未再交给助手", + # 剩下的只有点名接入的问题时不这么说:助手改不了接入表,也不知道该换成哪个接入 + lost = await _provider_issues(session, GraphSpec.model_validate(out["graph"]), level="error") + out["assist"] = {"ok": not lost, "summary": PROVIDER_LEFT if lost else "修复后已没有阻止发布的错误,未再交给助手", "questions": []} - return out + return _with_provider_issues(out, lost, "remaining") from app.api.copilot import assist_publish_fix helped = await assist_publish_fix(session, out["graph"], level=payload.level, @@ -493,7 +514,8 @@ async def autofix( out.update(graph=fixed, remaining=after["remaining"], fixes=after["fixes"], ok=after["ok"]) elif helped["reason"]: out["rejected"].append({"fix_id": "assist", "reason": helped["reason"]}) - return out + return _with_provider_issues(out, await _provider_issues(session, GraphSpec.model_validate(out["graph"]), + level="error"), "remaining") class ValidateIn(BaseModel): @@ -501,14 +523,17 @@ class ValidateIn(BaseModel): @router.post("/validate") -async def validate(payload: ValidateIn) -> dict[str, Any]: - """画布上的实时校验。前端每次改动都会调它来标红。""" +async def validate(payload: ValidateIn, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """画布上的实时校验。前端每次改动都会调它来标红。 + + 点名的模型接入不存在或已停用只给警告:探索运行仍可发起,执行到那个节点时照实报错;发布时才挡住。""" try: spec = GraphSpec.model_validate(payload.graph) except Exception as e: # noqa: BLE001 return {"ok": False, "issues": [{"level": "error", "message": f"无法解析工作流结构:{graph_error(e)}"}]} - result = validate_graph(spec) - return result.model_dump() + result = validate_graph(spec).model_dump() + result["issues"] += await _provider_issues(session, spec, level="warning") + return result @router.post("/variables") diff --git a/backend/app/engine/provider_refs.py b/backend/app/engine/provider_refs.py new file mode 100644 index 00000000..5968c6c1 --- /dev/null +++ b/backend/app/engine/provider_refs.py @@ -0,0 +1,113 @@ +"""工作流里点名的模型接入:不存在或已停用的,画布校验、发布前检查、发布时照实提出来。 + +运行时点名的接入不存在或已停用直接报错(providers/factory.named_provider),以前是悄悄换成演示模型。这类问题发布前 +就该看得到:画布校验给警告(探索运行仍可发起,执行到那里报错),发布、发布前检查、自动修复的复核给错误、挡住 +发布——正式运行到那里一定失败,或者少一块核查。 + +查的是执行时真会去取模型的那几处,取法和执行器一致: +- 模型调用、Agent、报告撰写、多 Agent 协作的接入;条件分支只在「让模型分类」时、结构校验只在「让模型修复」时、 + 知识检索只在「让模型重排」时才取模型。这几类节点没写自己的接入时取全图默认(NodeContext.cfg); +- 多 Agent 协作各成员的接入;报告撰写交给模型裁判结论句时,「结论句裁判」里的接入; +- 双盲复算、反方审稿的接入(核查模型):只读节点自己的配置,不取全图默认; +- 全图默认的接入:有节点会取到它时才查。 + +接入表要读库,门禁本身不查库(governance.lint_for_publish),所以由接口这一层读出接入表交给 issues(),同数据目录 +SQL 检查器的做法。设置里点名接入的几项由 api/settings.py 在保存时查。 +""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Callable + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.engine.labels import field_label, type_label +from app.engine.schema import CLAIMS_JUDGE, GraphNode, GraphSpec, NodeType, ValidationIssue +from app.providers.factory import ProviderNotConfigured, list_providers, named_provider + + +def _cfg(spec: GraphSpec, node: GraphNode, key: str, default: Any = None) -> Any: + """和 NodeContext.cfg 同一个规则:节点上没写(None、空串)就取全图默认。""" + value = node.config.get(key) + return spec.defaults.get(key, default) if value in (None, "") else value + + +#: 按 NodeContext.cfg 取接入的节点 → 执行时真会去取模型的条件(None 是总会取) +_TAKES_MODEL: dict[NodeType, Callable[[GraphSpec, GraphNode], bool] | None] = { + NodeType.LLM: None, + NodeType.AGENT: None, + NodeType.REPORT: None, + NodeType.SUPERVISOR: None, + NodeType.BRANCH: lambda spec, node: _cfg(spec, node, "mode", "expression") == "llm", + NodeType.VALIDATE: lambda spec, node: bool(_cfg(spec, node, "repair_with_llm", True)), + NodeType.RETRIEVE: lambda spec, node: _cfg(spec, node, "rerank", "off") == "model", +} +#: 核查节点:接入只读节点自己的配置(verify_model_spec),图级默认的接入是写作模型的 +_VERIFY = {NodeType.DUAL_CHECK: "的盲算模型", NodeType.CHALLENGE: "的反方模型"} + + +def _name(value: Any) -> str | None: + return value.strip() or None if isinstance(value, str) else None + + +@dataclass(frozen=True) +class ProviderRef: + """图里点名接入的一处:落在哪个节点(全图默认为 None)、哪个字段,点的名字,句子里怎么称呼这一处。""" + + node_id: str | None + field: str + name: str + who: str + + +def provider_refs(spec: GraphSpec) -> list[ProviderRef]: + """图里执行时会用到的接入点名,按画布上节点的顺序,全图默认排最后。""" + out: list[ProviderRef] = [] + falls_back = False + for node in spec.nodes: + cfg = node.config + who = f"「{node.title}」({type_label(node.type)})" + if node.type in _VERIFY: + if name := _name(cfg.get("provider")): + out.append(ProviderRef(node.id, "provider", name, who + _VERIFY[node.type])) + continue + if node.type not in _TAKES_MODEL: + continue + cond = _TAKES_MODEL[node.type] + if cond is None or cond(spec, node): + if name := _name(cfg.get("provider")): + out.append(ProviderRef(node.id, "provider", name, who)) + else: + falls_back = True + if node.type == NodeType.SUPERVISOR and isinstance(cfg.get("agents"), list): + for i, member in enumerate(cfg["agents"]): + if isinstance(member, dict) and (name := _name(member.get("provider"))): + label = _name(member.get("name")) or f"第 {i + 1} 位" + out.append(ProviderRef(node.id, f"agents[{i}].provider", name, f"{who}的成员「{label}」")) + if node.type == NodeType.REPORT and _cfg(spec, node, "claims", "off") == CLAIMS_JUDGE \ + and isinstance(cfg.get("judge"), dict) and (name := _name(cfg["judge"].get("provider"))): + out.append(ProviderRef(node.id, "judge.provider", name, f"{who}的{field_label('judge')}")) + if falls_back and (name := _name(spec.defaults.get("provider"))): + out.append(ProviderRef(None, "defaults.provider", name, field_label("defaults"))) + return out + + +def issues(refs: list[ProviderRef], providers: list[Any], *, level: str) -> list[ValidationIssue]: + """点名的接入不存在或已停用的每一处一条问题。providers 是接入表(factory.list_providers),level 由调用方定: + 画布校验 warning,发布一类 error。说法取自 named_provider,和运行时报的原因、提示同一句。""" + out: list[ValidationIssue] = [] + for ref in refs: + try: + named_provider(providers, ref.name) + except ProviderNotConfigured as e: + out.append(ValidationIssue(level=level, node_id=ref.node_id, field=ref.field, # type: ignore[arg-type] + code=f"provider.{e.code}", message=f"{ref.who}指定的{e.reason}。{e.hint}")) + return out + + +async def load_issues(session: AsyncSession, spec: GraphSpec, *, level: str) -> list[ValidationIssue]: + """读接入表、查一遍。图里没点名任何接入时不读库。""" + refs = provider_refs(spec) + if not refs: + return [] + return issues(refs, await list_providers(session), level=level) diff --git a/backend/tests/test_provider_ref_checks.py b/backend/tests/test_provider_ref_checks.py new file mode 100644 index 00000000..68158f17 --- /dev/null +++ b/backend/tests/test_provider_ref_checks.py @@ -0,0 +1,281 @@ +"""保存设置、校验画布、发布时就查出点名了不存在或已停用的模型接入(运行时它们会照实报错,见 test_provider_ref_runtime)。 + +- PUT /api/settings:引用接入的几项(核查模型、证据裁判、门控模型、助手)写了不存在或已停用的接入 → 422,带 field、 + code 和设置页上的叫法;一组出错整次不落库。留空照旧表示沿用上一级;存着的值没改(写好之后接入才被删、停用)不挡 + 这次保存别的项——设置页标着「不存在」,运行时照实报错; +- 画布校验:节点点名的接入不存在或已停用给警告,说清哪个节点、哪个接入;发布前检查、发布、自动修复的复核给错误, + 挡住发布。只查执行时真会去取模型的那几处(分支「让模型分类」、校验「让模型修复」、检索「让模型重排」时才算)。 + +接入名、节点名都是虚构的。 +""" +from __future__ import annotations + +import uuid + +import pytest +from httpx import ASGITransport, AsyncClient +from sqlalchemy import delete + +from app.db.base import SessionLocal +from app.db.models import Provider, Setting, Workflow, WorkflowVersion +from app.main import app + +LIVE, OFF, GONE = "核对用接入", "核对用停用接入", "核对用接人" +SETTING_KEYS = ["verify", "judge", "copilot", "run"] + + +async def _clean() -> None: + async with SessionLocal() as session: + await session.execute(delete(Setting).where(Setting.key.in_(SETTING_KEYS))) + await session.execute(delete(Provider).where(Provider.name.in_([LIVE, OFF]))) + await session.commit() + + +@pytest.fixture(autouse=True) +async def providers(): + """一个启用、一个停用的接入(都是演示类,不联网);用完删掉,免得改变同一进程里别的测试「没点名」时的回退。""" + await _clean() + async with SessionLocal() as session: + live = Provider(name=LIVE, kind="mock", models=[{"id": "核对模型"}], default_model="核对模型", enabled=True, + extra={}) + off = Provider(name=OFF, kind="mock", models=[{"id": "核对停用模型"}], default_model="核对停用模型", + enabled=False, extra={}) + session.add_all([live, off]) + await session.commit() + ids = {LIVE: live.id, OFF: off.id} + yield ids + await _clean() + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +async def stored(key: str): + async with SessionLocal() as session: + row = await session.get(Setting, key) + return row.value if row else None + + +# -------------------------------------------------------------------------- +# 保存设置 +# -------------------------------------------------------------------------- + +#: (组, 键, 设置页上的叫法) +FIELDS = [ + ("verify", "provider", "核查模型 · 接入"), + ("judge", "provider", "裁判模型 · 接入"), + ("run", "tool_gate_provider", "门控模型 · 接入"), + ("copilot", "provider", "助手模型 · 接入"), +] + + +@pytest.mark.parametrize("group,key,label", FIELDS) +async def test_saving_a_provider_that_does_not_exist_is_refused_with_the_field(client, group, key, label): + r = await client.put("/api/settings", json={"values": {group: {key: GONE}}}) + assert r.status_code == 422, r.text + body = r.json() + assert body["code"] == "settings_provider_invalid" and body["field"] == f"{group}.{key}" + assert body["detail"].startswith(f"「{label}」填写的模型接入「{GONE}」不存在。"), body["detail"] + assert "请在「设置 → 模型接入」中核对名称,或换一个接入" in body["detail"] and "留空表示" in body["detail"] + assert await stored(group) is None + + +@pytest.mark.parametrize("group,key,label", FIELDS) +async def test_saving_a_disabled_provider_is_refused_too(client, group, key, label): + r = await client.put("/api/settings", json={"values": {group: {key: OFF}}}) + assert r.status_code == 422, r.text + body = r.json() + assert body["field"] == f"{group}.{key}" + assert body["detail"].startswith(f"「{label}」填写的模型接入「{OFF}」已停用。"), body["detail"] + assert "请在「设置 → 模型接入」中启用它,或换一个接入" in body["detail"] + assert await stored(group) is None + + +async def test_one_bad_group_keeps_the_whole_request_out_of_the_database(client): + r = await client.put("/api/settings", json={"values": { + "run": {"default_memory_scope": "门店", "tool_gate_provider": LIVE}, + "judge": {"provider": LIVE, "model": None}, + "verify": {"provider": GONE}, + }}) + assert r.status_code == 422 and r.json()["field"] == "verify.provider" + assert await stored("run") is None and await stored("judge") is None and await stored("verify") is None + + +async def test_blank_means_inherit_and_an_enabled_provider_is_taken_by_name_or_id(client, providers): + r = await client.put("/api/settings", json={"values": { + "verify": {"provider": None, "model": "核对模型"}, + "judge": {"provider": " "}, + "run": {"tool_gate_provider": ""}, + }}) + assert r.status_code == 200, r.text + r = await client.put("/api/settings", json={"values": { + "verify": {"provider": LIVE}, "judge": {"provider": providers[LIVE]}, + "run": {"tool_gate_provider": LIVE}, "copilot": {"provider": LIVE, "model": ""}, + }}) + assert r.status_code == 200, r.text + got = r.json() + assert got["verify"]["provider"] == LIVE and got["judge"]["provider"] == providers[LIVE] + assert got["run"]["tool_gate_provider"] == LIVE + + +async def test_a_stored_provider_removed_later_does_not_block_saving_the_rest_of_its_group(client): + """写好之后接入才被删掉:存着的值没改,这次保存别的项照常(运行时照实报错,设置页标着「不存在」)。 + 改成另一个不存在的名字仍然拦下。""" + async with SessionLocal() as session: + session.add(Setting(key="verify", value={"provider": GONE, "daily_max_usd": 2.0})) + await session.commit() + r = await client.put("/api/settings", json={"values": {"verify": {"provider": f" {GONE} ", "daily_max_usd": 3}}}) + assert r.status_code == 200, r.text + assert (await stored("verify"))["daily_max_usd"] == 3 + r = await client.put("/api/settings", json={"values": {"verify": {"provider": "另一个不存在的接入"}}}) + assert r.status_code == 422 and r.json()["field"] == "verify.provider" + + +async def test_format_problems_are_still_reported_first(client): + r = await client.put("/api/settings", json={"values": {"verify": {"provider": 3}}}) + assert r.status_code == 422 + assert "需要填写模型接入的名称或模型 ID" in r.json()["detail"] + + +# -------------------------------------------------------------------------- +# 画布校验、发布前检查、发布 +# -------------------------------------------------------------------------- + + +def node(nid, ntype, label=None, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": label or nid, "config": config}} + + +def chain(*nodes, defaults=None): + graph = {"nodes": list(nodes), "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]} + if defaults: + graph["defaults"] = defaults + return graph + + +def writer(ref: str | None, **extra) -> dict: + return chain(node("start", "input"), + node("write", "llm", "写周报", **({"provider": ref} if ref else {}), prompt="写周报", **extra), + node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}])) + + +def provider_issues(result: dict) -> list[dict]: + return [i for i in result["issues"] if str(i.get("code") or "").startswith("provider.")] + + +@pytest.mark.parametrize("ref,code,state", [(GONE, "provider.missing", "不存在"), (OFF, "provider.disabled", "已停用")]) +async def test_validation_warns_about_a_node_naming_a_broken_provider(client, ref, code, state): + r = await client.post("/api/workflows/validate", json={"graph": writer(ref)}) + assert r.status_code == 200, r.text + [issue] = provider_issues(r.json()) + assert issue["level"] == "warning" and issue["code"] == code + assert (issue["node_id"], issue["field"]) == ("write", "provider") + assert issue["message"].startswith(f"「写周报」(模型调用)指定的模型接入「{ref}」{state}。"), issue["message"] + assert r.json()["ok"] is True, "只是警告:探索运行仍可发起,执行到那里照实报错" + + +async def test_validation_is_quiet_for_an_enabled_provider_or_none(client, providers): + for ref in (LIVE, providers[LIVE], None): + r = await client.post("/api/workflows/validate", json={"graph": writer(ref)}) + assert provider_issues(r.json()) == [], ref + + +async def test_every_place_a_graph_names_a_provider_is_checked(client): + graph = chain( + node("start", "input"), + node("team", "supervisor", "协作", provider=OFF, agents=[{"name": "分析师", "provider": GONE, "system": "分析"}, + {"provider": LIVE, "system": "复核"}]), + node("write", "report", "写周报", instructions="写周报", claims="judge", judge={"provider": GONE}), + node("dual", "dual_check", "复核销售额", provider=GONE), + node("chal", "challenge", "审稿", report="write", provider=OFF), + node("ask", "llm", "提问", prompt="总结"), # 没写接入:取全图默认 + node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}]), + defaults={"provider": GONE}, + ) + r = await client.post("/api/workflows/validate", json={"graph": graph}) + got = {(i["node_id"], i["field"]): i for i in provider_issues(r.json())} + assert set(got) == {("team", "provider"), ("team", "agents[0].provider"), ("write", "judge.provider"), + ("dual", "provider"), ("chal", "provider"), (None, "defaults.provider")} + assert got[("team", "agents[0].provider")]["message"].startswith( + f"「协作」(多 Agent 协作)的成员「分析师」指定的模型接入「{GONE}」不存在") + assert got[("write", "judge.provider")]["message"].startswith( + f"「写周报」(报告撰写)的结论句裁判指定的模型接入「{GONE}」不存在") + assert got[("dual", "provider")]["message"].startswith(f"「复核销售额」(双盲复算)的盲算模型指定的模型接入「{GONE}」") + assert got[("chal", "provider")]["message"].startswith(f"「审稿」(反方审稿)的反方模型指定的模型接入「{OFF}」已停用") + assert got[(None, "defaults.provider")]["message"].startswith(f"工作流默认设置指定的模型接入「{GONE}」不存在") + + +async def test_only_places_that_actually_call_a_model_are_checked(client): + """分支按表达式判断、校验不让模型修复、检索不让模型重排时根本不取模型;报告不交给模型裁判时裁判的接入用不上; + 每个节点都写了自己的接入时,全图默认的接入没人用。""" + graph = chain( + node("start", "input"), + node("route", "branch", "分流", mode="expression", provider=GONE, cases=[{"key": "a", "condition": "true"}]), + node("check", "validate", "校验", repair_with_llm=False, provider=GONE, schema={"type": "object"}), + node("look", "retrieve", "检索", rerank="off", provider=GONE), + node("write", "report", "写周报", instructions="写周报", provider=LIVE, judge={"provider": GONE}), + node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}]), + defaults={"provider": GONE}, + ) + r = await client.post("/api/workflows/validate", json={"graph": graph}) + assert provider_issues(r.json()) == [] + graph["nodes"][1]["data"]["config"]["mode"] = "llm" + graph["nodes"][3]["data"]["config"]["rerank"] = "model" + r = await client.post("/api/workflows/validate", json={"graph": graph}) + assert {i["node_id"] for i in provider_issues(r.json())} == {"route", "look"} + # 分类的分支不写自己的接入:取全图默认,默认里的接入这才有人用 + del graph["nodes"][1]["data"]["config"]["provider"] + r = await client.post("/api/workflows/validate", json={"graph": graph}) + assert {(i["node_id"], i["field"]) for i in provider_issues(r.json())} == {("look", "provider"), + (None, "defaults.provider")} + + +async def _workflow(graph: dict) -> str: + async with SessionLocal() as session: + wf = Workflow(name=f"门店周报-{uuid.uuid4().hex[:6]}", graph=graph, version=1) + session.add(wf) + await session.flush() + session.add(WorkflowVersion(workflow_id=wf.id, version=1, graph=graph)) + await session.commit() + return wf.id + + +@pytest.mark.parametrize("level", ["published", "governed"]) +async def test_publish_check_and_publish_stop_at_a_broken_provider(client, level): + wf = await _workflow(writer(GONE)) + r = await client.post(f"/api/workflows/{wf}/publish-check", json={"level": level}) + assert r.status_code == 200, r.text + check = r.json() + [issue] = provider_issues(check) + assert issue["level"] == "error" and issue["node_id"] == "write" and issue["fix"] is None + assert check["ok"] is False + r = await client.post(f"/api/workflows/{wf}/publish", json={"level": level}) + body = r.json() + assert body["ok"] is False and [i["code"] for i in provider_issues(body)] == ["provider.missing"] + async with SessionLocal() as session: + row = await session.get(Workflow, wf) + assert row.published_version is None and row.status == "draft" + + +async def test_publish_check_reads_the_unsaved_canvas_and_autofix_keeps_the_problem(client): + wf = await _workflow(writer(None)) + r = await client.post(f"/api/workflows/{wf}/publish-check", json={"level": "published", "graph": writer(OFF)}) + assert r.json()["ok"] is False and [i["code"] for i in provider_issues(r.json())] == ["provider.disabled"] + r = await client.post(f"/api/workflows/{wf}/autofix", json={"level": "published", "graph": writer(OFF)}) + assert r.status_code == 200, r.text + out = r.json() + assert out["ok"] is False + assert [(i["code"], i["level"]) for i in provider_issues({"issues": out["remaining"]})] == [ + ("provider.disabled", "error")] + # 换成启用的接入:不再拦 + r = await client.post(f"/api/workflows/{wf}/publish-check", json={"level": "published", "graph": writer(LIVE)}) + assert provider_issues(r.json()) == [] and r.json()["ok"] is True + + +async def test_publishing_a_clean_graph_is_unaffected(client): + wf = await _workflow(writer(LIVE)) + r = await client.post(f"/api/workflows/{wf}/publish", json={"level": "published"}) + assert r.json()["ok"] is True, r.text diff --git a/backend/tests/test_verify_settings.py b/backend/tests/test_verify_settings.py index b94fd3fb..e6a2c313 100644 --- a/backend/tests/test_verify_settings.py +++ b/backend/tests/test_verify_settings.py @@ -21,7 +21,7 @@ from app.api.settings import ( verify_settings, ) from app.db.base import SessionLocal -from app.db.models import Setting +from app.db.models import Provider, Setting from app.engine import spend from app.engine.judge import JUDGE_DEFAULTS, MAX_TOKENS from app.main import app @@ -64,7 +64,20 @@ async def test_settings_expose_the_verify_group_with_defaults(): assert spend.SPEND_KEY not in got -async def test_a_full_group_round_trips_and_null_stays_unlimited(): +@pytest.fixture +async def house(): + """设置里点名的接入要真的存在、启用着(保存时会查):建一个演示类的,用完删掉。""" + async with SessionLocal() as session: + session.add(Provider(name="house", kind="mock", models=[{"id": "verify-model"}], default_model="verify-model", + enabled=True, extra={})) + await session.commit() + yield "house" + async with SessionLocal() as session: + await session.execute(delete(Provider).where(Provider.name == "house")) + await session.commit() + + +async def test_a_full_group_round_trips_and_null_stays_unlimited(house): value = { "provider": "house", "model": "verify-model", "dual_check": {"enabled": False, "tolerance": 0, "max_queries": None, "timeout_s": None},