"""单独调用双盲复算的执行器(不在图里执行:@task 直接执行,once 直接发),测图里不好凑的几种情形。 - 盲算模型和产出原数据的 Agent 是同一个模型时提醒;调用工具节点写死的 SQL 没有模型参与,不提醒 - 合并查询、整段结果、遮罩列、文本这类输入不能单独复算:记缺口,一分钱不花 - 口径卡没执行到的直接放行;配置里写了卡上没有的指标 ID 报节点错误 查询快照是现造的工件,表名、数字都是虚构的。 """ from __future__ import annotations from dataclasses import dataclass, field from typing import Any import pytest from sqlalchemy import delete from app.core import artifact_store from app.db.base import SessionLocal from app.db.models import Setting from app.engine.context import NodeContext, NodeError, RunContext from app.engine.evidence import cell_eid from app.engine.nodes.dual_check import run_dual_check from app.engine.runner import RUN_OPTIONS_KEY from app.engine.schema import GraphSpec from app.tools.datasource import RUN_VERSIONS_KEY from tests.fixtures.dual_check_shop import RIGHT, Blind, add_source, build from tests.fixtures.sources import drop_source def node(nid, ntype, label=None, **config): return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": label or nid, "config": config}} @dataclass class Recorder(NodeContext): """记下节点发的事件。不在图里执行:@task 直接执行,once 直接发。""" events: list[tuple[str, dict[str, Any]]] = field(default_factory=list) def emit(self, event_type, **data): self.events.append((str(event_type), data)) def logs(self, code: str) -> list[dict[str, Any]]: return [d for t, d in self.events if t == "log" and d.get("code") == code] @pytest.fixture async def shop(tmp_path): async with SessionLocal() as session: await session.execute(delete(Setting).where(Setting.key == "verify")) await session.commit() name, source_id = await add_source(build(tmp_path / "shop.db"), options={"mask_columns": "secret_gmv"}) yield name await drop_source(source_id) async def snapshot(source: str, *, columns=("gmv",), rows=((330.0,),), sql=RIGHT, **extra) -> str: content = {"columns": list(columns), "rows": [list(r) for r in rows], "row_count": len(rows), "truncated": False, "sql": sql, "source": source, **extra} return await artifact_store.put_json(content, kind="query_snapshot", run_id="unit", node_id="agent") def context(graph: dict, node_id: str = "dual") -> Recorder: parsed = GraphSpec.model_validate(graph) run = RunContext(run_id="unit-run", thread_id="unit-run", spec=parsed, extra={RUN_OPTIONS_KEY: {"verify": {}}, RUN_VERSIONS_KEY: None}) return Recorder(node=parsed.node_map()[node_id], run=run) AGENT_GRAPH = {"nodes": [node("start", "input"), node("agent", "agent", prompt="查销售额"), node("card", "metrics", metrics=[{"id": "gmv", "expression": "vars.kpi.gmv"}]), node("dual", "dual_check", "复核"), node("out", "output", fields=[])], "edges": [{"source": a, "target": b} for a, b in (("start", "agent"), ("agent", "card"), ("card", "dual"), ("dual", "out"))]} def card_of(*metrics: dict) -> dict: return {"kind": "metric_set", "caliber": "销售口径", "caliber_version": "v1", "metrics": list(metrics), "artifact": "card-artifact"} def metric(artifact: str, *, path: str = "vars.kpi.gmv", locator: dict | None = None, value: Any = 330.0, via: str = "agent_field", node_id: str = "agent", expression: str | None = None) -> dict: where = locator or {"row": 0, "column": "gmv"} return {"id": "gmv", "name": "销售额", "unit": "元", "decimals": 1, "format": "thousands", "value": 330.0, "rendered": "330.0元", "expression": expression or path, "substituted": "330.0", "status": "ok", "inputs": [{"path": path, "value": value, "node_id": node_id, "via": via, "field": "gmv", "artifact": artifact, "locator": where, "status": "verified", **({"eid": cell_eid(artifact, where["row"], where["column"])} if "row" in where else {})}]} async def test_the_same_model_as_the_agent_that_fetched_the_data_is_flagged(monkeypatch, shop): art = await snapshot(shop) Blind(monkeypatch, shop, {"销售额": (RIGHT, "gmv")}) ctx = context(AGENT_GRAPH) state = {"nodes": {"agent": {"model": "mock-fast", "data": {"gmv": 330.0}}, "card": card_of(metric(art))}, "evidence": [], "trail": []} out = (await run_dual_check(state, ctx))["nodes"]["dual"] assert out["status"] == "agreed" and out["same_model"] is True and out["model"] == "mock-fast" [warned] = ctx.logs("dual_check_same_model") assert warned["model"] == "mock-fast" and "盲算模型与产出原数据的模型相同" in warned["message"] assert "设置 → 偏好设置 → 核查" in warned["message"] async def test_data_fetched_by_fixed_sql_raises_no_same_model_warning(monkeypatch, shop): art = await snapshot(shop) Blind(monkeypatch, shop, {"销售额": (RIGHT, "gmv")}) ctx = context(AGENT_GRAPH) state = {"nodes": {"agent": {"model": "mock-fast"}, "card": card_of(metric(art, path="cell(nodes.fetch, 0, 'gmv')", via="tool_cell", node_id="fetch"))}, "evidence": [], "trail": []} out = (await run_dual_check(state, ctx))["nodes"]["dual"] assert out["status"] == "agreed" and out["same_model"] is False and not ctx.logs("dual_check_same_model") @pytest.mark.parametrize("case, reason", [ ("merge", "来自合并查询"), ("rows", "取的是一整段查询结果"), ("masked", "设置了遮罩的列"), ("text", "不是数值"), ]) async def test_inputs_that_cannot_be_checked_alone_become_gaps(monkeypatch, shop, case, reason): blind = Blind(monkeypatch, shop, {"销售额": (RIGHT, "gmv")}) if case == "merge": art = await snapshot("合并查询", sources=[shop]) item = metric(art) elif case == "rows": art = await snapshot(shop, rows=((100.0,), (230.0,))) item = metric(art, path="vars.kpi.weeks", expression="sum(vars.kpi.weeks)", locator={"rows": [0, 1], "column": "gmv"}, value="[2 项]") elif case == "masked": art = await snapshot(shop, columns=("secret_gmv",)) item = metric(art, locator={"row": 0, "column": "secret_gmv"}) else: art = await snapshot(shop, columns=("gmv",), rows=(("三百三",),)) item = metric(art, value="三百三") # 指标值是数,输入却是文本(比如拿它做了判断) ctx = context(AGENT_GRAPH) state = {"nodes": {"agent": {"model": "other"}, "card": card_of(item)}, "evidence": [], "trail": []} out = (await run_dual_check(state, ctx))["nodes"]["dual"] [row] = out["metrics"] assert row["status"] == "unchecked" and reason in row["reason"] assert out["issuance_gaps"] == [f"指标「销售额」无法复算:{row['reason']}"] assert blind.calls == [], "不能单独复算的输入也花了钱" async def test_a_card_that_did_not_run_passes_through(shop): ctx = context(AGENT_GRAPH) out = await run_dual_check({"nodes": {}, "evidence": [], "trail": []}, ctx) assert out == {"nodes": {"dual": {"status": "skipped", "reason": "no_inputs"}}} assert [d["reason"] for d in ctx.logs("dual_check_skipped")] == ["no_inputs"] async def test_unknown_metric_ids_fail_the_node(shop): art = await snapshot(shop) graph = {**AGENT_GRAPH, "nodes": [*AGENT_GRAPH["nodes"][:3], node("dual", "dual_check", "复核", metrics=["refund"]), AGENT_GRAPH["nodes"][4]]} ctx = context(graph) with pytest.raises(NodeError, match="「复算的指标」中的「refund」不是口径卡上的指标 ID"): await run_dual_check({"nodes": {"card": card_of(metric(art))}, "evidence": [], "trail": []}, ctx) async def test_constant_columns_of_the_original_query_are_not_shown(monkeypatch, shop): """原 SQL 里写死的文字列('下单日' AS basis)是原流程的取舍,不是分组:不交给盲算。""" sql = "SELECT '下单日' AS basis, '华东' AS region, SUM(amount) AS gmv FROM orders WHERE region = '华东'" art = await snapshot(shop, columns=("basis", "region", "gmv"), rows=(("下单日", "华东", 330.0),), sql=sql) blind = Blind(monkeypatch, shop, {"销售额": (RIGHT, "gmv")}) ctx = context(AGENT_GRAPH) item = metric(art, path="cell(nodes.fetch, 0, 'gmv')", via="tool_cell", node_id="fetch", locator={"row": 0, "column": "gmv"}) state = {"nodes": {"agent": {"model": "other"}, "card": card_of(item)}, "evidence": [], "trail": []} await run_dual_check(state, ctx) [task] = blind.tasks assert "下单日" not in task and "basis" not in task and "华东" not in task async def test_blind_checks_run_side_by_side_after_the_first(monkeypatch, shop): """几个盲算同时进行(有上限),先单独试第一个:核查模型用不了时不必个个去试。结果按格子对回各自的指标。""" import asyncio from app.engine import verify_agent from app.engine.nodes import dual_check art = await snapshot(shop, columns=("gmv",), rows=tuple((float(100 + i),) for i in range(5))) running = {"now": 0, "peak": 0, "order": []} async def fake(**kwargs): running["now"] += 1 running["peak"] = max(running["peak"], running["now"]) running["order"].append(kwargs["task"]) await asyncio.sleep(0.05) running["now"] -= 1 return verify_agent.VerifyOutcome(result=None, cells=[], queries=[], rationale=None, usage={"calls": 1, "cost_usd": 0.01}, error="没有交结果", model="mock-fast") monkeypatch.setattr(verify_agent, "run_verify_agent", fake) metrics = [] for i in range(5): item = metric(art, locator={"row": i, "column": "gmv"}, value=float(100 + i)) metrics.append({**item, "id": f"m{i}", "name": f"第 {i} 组销售额", "value": float(100 + i), "rendered": f"{100 + i}.0元", "substituted": f"{100 + i}.0"}) ctx = context(AGENT_GRAPH) state = {"nodes": {"agent": {"model": "other"}, "card": card_of(*metrics)}, "evidence": [], "trail": []} updates = await run_dual_check(state, ctx) out = updates["nodes"]["dual"] assert [m["status"] for m in out["metrics"]] == ["unchecked"] * 5 assert 1 < running["peak"] <= dual_check.BLIND_CONCURRENCY assert "第 0 组销售额" in running["order"][0] assert updates["usage"]["calls"] == 5 and updates["usage"]["cost_usd"] == 0.05