From ff23c6a8ad49bc153238752963b0a1fc16379c1d Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Mon, 5 Oct 2026 14:23:00 +0800 Subject: [PATCH] =?UTF-8?q?=E5=8F=8C=E7=9B=B2=E5=A4=8D=E7=AE=97=E7=9A=84?= =?UTF-8?q?=E9=85=8D=E7=BD=AE=E4=B8=8E=E5=8F=91=E8=B5=B7=E6=97=B6=E7=9A=84?= =?UTF-8?q?=E8=AE=BE=E7=BD=AE=E5=BF=AB=E7=85=A7=EF=BC=9A=E7=94=BB=E5=B8=83?= =?UTF-8?q?=E6=A0=A1=E9=AA=8C=E3=80=81=E6=A0=B8=E6=9F=A5=E8=AE=BE=E7=BD=AE?= =?UTF-8?q?=E5=BF=AB=E7=85=A7=E8=BF=9B=E8=BF=90=E8=A1=8C=E9=80=89=E9=A1=B9?= =?UTF-8?q?=E3=80=81=E5=8A=A9=E6=89=8B=E8=8A=82=E7=82=B9=E8=AF=B4=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 画布校验(engine/schema.py):「口径卡」必须是上游的口径卡,没写时上游只能有一张(dual_check_card,节点执行也用这一个认法); 「复算的指标」「口径说明」的键必须是那张卡上的指标 ID(引用自别的工作流的口径卡画图时查不出,不查);开关、允许的相对误差、 未采用原流程时的取值。报错写检查器上的中文叫法,叫法暂放在 DUAL_CHECK_FIELD_LABEL(labels.py 不归这个工作包)。 - 发起(engine/launch.py):图里有双盲复算或反方审稿节点时,把核查设置快照进 options["verify"]。这两种节点在运行中途读设置, 审批停下期间改了设置,恢复后的那一段就会按另一套规则走;发起时定下来,重放和正式运行才可复现。没有这两种节点时 options 原样不动,run.started 和以前逐字相同;调用方已经给了 options["verify"] 的照它的。 - 助手的节点说明(NODE_REFERENCE)加 dual_check 一条:要正式出具、口径卡上的指标来自查询时接在口径卡之后、报告之前,各配置项 怎么写(口径说明按业务写、盲算模型默认不写)。 --- backend/app/api/copilot.py | 11 ++ backend/app/engine/launch.py | 30 +++- backend/app/engine/schema.py | 93 +++++++++++ backend/tests/test_dual_check_config.py | 202 ++++++++++++++++++++++++ 4 files changed, 335 insertions(+), 1 deletion(-) create mode 100644 backend/tests/test_dual_check_config.py diff --git a/backend/app/api/copilot.py b/backend/app/api/copilot.py index 6b849d96..fdb042d6 100644 --- a/backend/app/api/copilot.py +++ b/backend/app/api/copilot.py @@ -214,6 +214,17 @@ NODE_REFERENCE = """\ 写 2 只剩「2%」,0.004 这样的小比率还会显示不出来、报告引用不了。on_missing 默认 fail(缺输入整张卡失败);写 null 则缺输入的指标记为空值,交给出具契约判档 取 tool 节点查库的结果用 cell(nodes.取数节点id, 行, '列名')(行号从 0 数,列按列名):cell() 直接认查询工具 交回的结果,出处精确到快照里的那一格,不用再接 transform 解析。取 agent 的字段写 vars..字段 +- dual_check:双盲复算。正式出具前,口径卡上输入来自查询的指标另起一个看不到原 SQL 的只读 agent 独立写 SQL 再算一遍: + 一致就放行;不一致时停下来,审批卡并排列出两边的 SQL、差异说明和两个值,由人逐个指标选采用原流程、采用盲算还是 + 两边都不对。选盲算或都不对的计入缺口、出具降档;两边只差一个方面(业务日期按哪一列、有效记录条件、去重)时,选择 + 会写进数据目录。**要正式出具的模板、口径卡上的指标来自查询(tool 节点查库,或 agent 查库并开了 cite_fields)时加**, + 接在口径卡之后、report 之前(取数 → 口径卡 → dual_check → report → output);只回答一个问题的问数据、指标都是 + 常量或运行输入的不必加。config: {card, metrics, definitions, provider, model, tolerance, on_reject, enabled}: + card 是要复算的口径卡节点 id(上游只有一张口径卡时不写);metrics 是要复算的指标 id 列表(不写就是输入来自查询的 + 全部指标);definitions 是 {指标 id: 口径说明},写清业务定义(按哪个日期归属、排除哪些行、是否去重),盲算 agent + 只看得到它和数据目录,所以要按业务写,不要抄原 SQL;provider / model 指定盲算模型(**默认不写**,取设置里的核查 + 模型;建议和取数的 agent 不同);tolerance 是允许的相对误差(不写取设置);on_reject:degrade(默认,计入缺口、 + 出具降档)/ fail(审核人没采用原流程的结果就判为失败);enabled 不写跟随设置里的开关,true / false 强制开关 - report:报告撰写(带引用)。config: {instructions, system, metrics_from:[口径卡id], numbers, on_violation, max_repairs, claims, entities, assign_to} **凡是给人看的、带数字的报告、分析、结论,都用 report,不要用 llm**。它自动收集上游的口径卡指标、查询结果 diff --git a/backend/app/engine/launch.py b/backend/app/engine/launch.py index 087be199..0f05c1c0 100644 --- a/backend/app/engine/launch.py +++ b/backend/app/engine/launch.py @@ -60,7 +60,8 @@ async def launch_run( 运行也没有开始执行。 参数和 POST /api/runs 的请求体一一对应(input 叫 input_payload);actor 是记在运行上的发起人。 - schedule_id 是发起它的定时任务,options 是整次运行沿用的选项快照,都原样交给 run_manager.start。 + schedule_id 是发起它的定时任务,options 是整次运行沿用的选项快照,都原样交给 run_manager.start;图里有双盲复算、 + 反方审稿节点时 options 另加核查设置的快照(_with_verify_snapshot)。 """ # 没挂在工作流上(画布没保存就发起、问数据当场搭的)。老数据里是「临时图」,前端两种都认 name = "未保存的工作流" @@ -135,6 +136,7 @@ async def launch_run( if not graph: raise LaunchRefused(400, "没有要运行的内容:请求中既没有工作流 ID,也没有工作流内容") await _refuse_missing_tools(session, graph) + options = await _with_verify_snapshot(session, graph, options) from app.api.settings import resolve_run_scope @@ -178,6 +180,32 @@ async def launch_run( return run +#: 运行中途要读核查设置的节点:双盲复算、反方审稿 +_VERIFY_NODES = (NodeType.DUAL_CHECK, NodeType.CHALLENGE) + + +async def _with_verify_snapshot(session: AsyncSession, graph: dict[str, Any], + options: dict[str, Any] | None) -> dict[str, Any] | None: + """图里有双盲复算或反方审稿节点时,把核查设置(api/settings.py 的 verify 一组)快照进 options["verify"]。 + + 这两种节点在运行中途读设置(开关、上限、核查模型)。审批停下期间有人改了设置,恢复后的那一段读到的就和前面 + 不一样:本该复算的被关掉了、重放时走了另一条路,正式运行也说不清当时按什么规则核的。所以在发起这一刻定下来, + 节点只读运行选项里的这一份(同 _approval_default 的道理)。没有这两种节点的图 options 原样不动,run.started + 和以前逐字相同;调用方已经给了 options["verify"] 的照它的。 + """ + from app.api.settings import VERIFY_GROUP, verify_settings + + if isinstance(options, dict) and VERIFY_GROUP in options: + return options + try: + spec = GraphSpec.model_validate(graph) + except Exception: # noqa: BLE001 - 图本身不成形交给 run_manager.start 报 + return options + if not any(node.type in _VERIFY_NODES for node in spec.nodes): + return options + return {**(options or {}), VERIFY_GROUP: await verify_settings(session)} + + # -------------------------------------------------------------------------- # 发起前检查绑定的工具 # -------------------------------------------------------------------------- diff --git a/backend/app/engine/schema.py b/backend/app/engine/schema.py index d3ae6325..306f5a2a 100644 --- a/backend/app/engine/schema.py +++ b/backend/app/engine/schema.py @@ -425,6 +425,97 @@ def _check_report_sources(node: GraphNode, spec: GraphSpec, result: ValidationRe node_id=node.id, field="metrics_from", code=code) +#: 双盲复算节点各配置项在检查器上的叫法(前端 nodeDefs 的 dual_check 字段)。按 labels.py 的规矩应放在那里, +#: 那个文件不归双盲复算的工作包,先放在校验旁边,节点执行时报错也从这里取 +DUAL_CHECK_FIELD_LABEL = { + "card": "口径卡", "metrics": "复算的指标", "definitions": "口径说明", "provider": "盲算模型 · 接入", + "model": "盲算模型", "tolerance": "允许的相对误差", "on_reject": "未采用原流程时", "enabled": "启用双盲复算", +} +#: 审核人没有采用原流程的结果(选了盲算,或者判断两边都不对)时:degrade 计入缺口、出具降档(默认)/ fail 判为失败 +DUAL_CHECK_ON_REJECT = ("degrade", "fail") +DUAL_CHECK_ON_REJECT_LABEL = {"degrade": "计入缺口、出具降档", "fail": "判为失败"} + + +def dual_check_card(spec: GraphSpec, node: GraphNode) -> tuple[str | None, str | None]: + """双盲复算要复算的口径卡:(节点 id, None) 或 (None, 为什么认不出)。写了「口径卡」(card)就是它,否则取上游唯一的 + 那张口径卡。校验和节点执行用的是这一个认法;写了的那个节点存不存在、是不是口径卡、在不在上游由调用方查。""" + card = node.config.get("card") + if card not in (None, ""): + return (card, None) if isinstance(card, str) else (None, f"「{DUAL_CHECK_FIELD_LABEL['card']}」需要选择一个口径卡节点") + upstream = _ancestors(spec, node.id) + cards = [n for n in spec.nodes if n.id in upstream and n.type == NodeType.METRICS] + if not cards: + return None, "「双盲复算」的上游没有口径卡:它复算的是口径卡上的指标,请把它接在口径卡之后" + if len(cards) > 1: + names = "、".join(f"「{n.title}」" for n in cards[:5]) + return None, (f"「双盲复算」的上游有 {len(cards)} 张口径卡({names})," + f"请在「{DUAL_CHECK_FIELD_LABEL['card']}」中选择要复算的那一张") + return cards[0].id, None + + +def _check_dual_check(node: GraphNode, spec: GraphSpec, result: ValidationResult) -> None: + """双盲复算的配置:口径卡必须是它上游的口径卡;复算的指标、口径说明的键必须是那张卡上的指标 ID;开关、 + 误差、处置的取值。指标定义从别的工作流引用来的口径卡(caliber_from),画图时不知道有哪些指标,不查指标 ID。""" + cfg = node.config + label = DUAL_CHECK_FIELD_LABEL + + def add(message: str, field: str, code: str) -> None: + result.add(message, node_id=node.id, field=field, code=code) + + if cfg.get("enabled") not in (None, True, False): + add(f"「{label['enabled']}」只能选跟随设置、开启或关闭,当前为「{cfg.get('enabled')}」", "enabled", + "dual_check.option_invalid") + if cfg.get("on_reject") not in (None, "", *DUAL_CHECK_ON_REJECT): + allowed = "或".join(f"「{v}」" for v in DUAL_CHECK_ON_REJECT_LABEL.values()) + add(f"「{label['on_reject']}」只能选{allowed},当前为「{cfg.get('on_reject')}」", "on_reject", + "dual_check.option_invalid") + tolerance = cfg.get("tolerance") + if tolerance not in (None, "") and (isinstance(tolerance, bool) or not isinstance(tolerance, (int, float)) + or not math.isfinite(tolerance) or tolerance < 0): + add(f"「{label['tolerance']}」需要填写不小于 0 的数(例如 0.000001 表示百万分之一),留空表示按设置," + f"当前为「{tolerance}」", "tolerance", "dual_check.option_invalid") + + card_id, problem = dual_check_card(spec, node) + if problem: + add(problem, "card", "dual_check.card_invalid") + return + target = spec.node_map().get(card_id or "") + if target is None: + add(f"「{label['card']}」选择了「{card_id}」,但工作流中不存在这个节点", "card", "dual_check.card_invalid") + return + if target.type != NodeType.METRICS: + add(f"「{label['card']}」选择了「{target.title}」,它是「{type_label(target.type)}」节点,不是口径卡;" + "双盲复算只能复算口径卡上的指标", "card", "dual_check.card_invalid") + return + if target.id not in _ancestors(spec, node.id): + add(f"「{label['card']}」选择的口径卡「{target.title}」不在双盲复算的上游:复算时这张卡还没有计算。" + "请把口径卡连到双盲复算之前", "card", "dual_check.card_invalid") + return + if target.config.get("caliber_from") not in (None, "", {}): + known = None + else: + known = [str(d.get("id")) for d in target.config.get("metrics") or [] if isinstance(d, dict) and d.get("id")] + hint = (",可选的指标 ID:" + "、".join(known[:12])) if known else "" + + metrics = cfg.get("metrics") + if metrics not in (None, "", []): + if not isinstance(metrics, list) or not all(isinstance(m, str) and m.strip() for m in metrics): + add(f"「{label['metrics']}」需要填写口径卡上的指标 ID 列表,留空表示复算输入来自查询的全部指标", "metrics", + "dual_check.metrics_invalid") + elif known is not None and (unknown := [m for m in metrics if m not in known]): + add(f"「{label['metrics']}」中的「{'、'.join(unknown[:5])}」不是口径卡「{target.title}」上的指标 ID{hint}", + "metrics", "dual_check.metrics_invalid") + definitions = cfg.get("definitions") + if definitions not in (None, "", {}): + if not isinstance(definitions, dict) or not all(isinstance(k, str) and isinstance(v, str) + for k, v in definitions.items()): + add(f"「{label['definitions']}」需要写成「指标 ID → 口径说明」的对照,口径说明写成文字", "definitions", + "dual_check.definitions_invalid") + elif known is not None and (unknown := [k for k in definitions if k not in known]): + add(f"「{label['definitions']}」中的「{'、'.join(unknown[:5])}」不是口径卡「{target.title}」上的指标 ID{hint}", + "definitions", "dual_check.definitions_invalid") + + def _produces_query(node: GraphNode) -> bool: """这个节点的产出是一份存成快照的查询结果:调用数据库查询工具的「调用工具」节点,或另一个合并查询。""" if node.type == NodeType.MERGE: @@ -674,6 +765,8 @@ def validate_graph(spec: GraphSpec) -> ValidationResult: if not d.get("id") or not d.get("expression"): result.add(f"指标定义缺少 ID 或表达式:{d.get('id') or '(空)'}", node_id=node.id, field=f"metrics[{i}]") + elif node.type == NodeType.DUAL_CHECK: + _check_dual_check(node, spec, result) elif node.type == NodeType.REPORT: _check_report_sources(node, spec, result) _check_report_claims(node, spec, result) diff --git a/backend/tests/test_dual_check_config.py b/backend/tests/test_dual_check_config.py new file mode 100644 index 00000000..c88696bb --- /dev/null +++ b/backend/tests/test_dual_check_config.py @@ -0,0 +1,202 @@ +"""双盲复算的配置、发起时的设置快照、助手的节点说明。 + +- 画布校验(engine/schema.py):口径卡必须是上游的口径卡(没写时上游只能有一张),复算的指标、口径说明的键必须是 + 那张卡上的指标 ID,开关、误差、处置的取值;报错写检查器上的中文叫法 +- 发起(engine/launch.py):图里有双盲复算或反方审稿节点时把核查设置快照进 options["verify"],没有时 options 不变、 + run.started 和以前逐字相同;调用方给了的照它的 +- 助手的节点说明(api/copilot.py 的 NODE_REFERENCE)有 dual_check 一条:什么时候加、各配置项 + +表名、数字都是虚构的。 +""" +from __future__ import annotations + +import asyncio + +import pytest +from sqlalchemy import delete, select + +from app.db.base import SessionLocal +from app.db.models import Run, RunEvent, Setting +from app.engine.runner import run_manager +from app.engine.schema import GraphSpec, validate_graph + + +def node(nid, ntype, label=None, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": label or nid, "config": config}} + + +CARD = node("card", "metrics", "销售口径卡", caliber="销售口径", metrics=[ + {"id": "gmv", "name": "销售额", "expression": "cell(nodes.fetch, 0, 'gmv')"}, + {"id": "orders", "name": "订单数", "expression": "cell(nodes.fetch, 0, 'n')"}]) + + +def spec(*middle: dict, dual: dict | None = None, edges: list | None = None) -> dict: + nodes = [node("start", "input"), node("fetch", "tool", tool="db_query__shop", args={"sql": "SELECT 1"}), *middle, + node("dual", "dual_check", "复核", **(dual or {})), node("out", "output", fields=[])] + chain = edges or [(a["id"], b["id"]) for a, b in zip(nodes, nodes[1:])] + return {"nodes": nodes, "edges": [{"source": s, "target": t} for s, t in chain]} + + +def problems(graph: dict) -> list[tuple[str | None, str]]: + report = validate_graph(GraphSpec.model_validate(graph)) + return [(i.field, i.message) for i in report.issues if i.level == "error" and i.node_id == "dual"] + + +# -------------------------------------------------------------------------- +# 画布校验 +# -------------------------------------------------------------------------- + + +def test_a_valid_config_has_no_problems(): + assert problems(spec(CARD, dual={"card": "card", "metrics": ["gmv"], "definitions": {"gmv": "按支付日期,不含退款"}, + "tolerance": 0.001, "on_reject": "fail", "enabled": True})) == [] + # 不写口径卡:上游只有一张,就是它 + assert problems(spec(CARD)) == [] + + +def test_the_card_must_be_an_upstream_metrics_card(): + [(where, text)] = problems(spec(CARD, dual={"card": "fetch"})) + assert where == "card" and "「口径卡」选择了「fetch」" in text and "不是口径卡" in text + [(where, text)] = problems(spec(CARD, dual={"card": "ghost"})) + assert where == "card" and "不存在这个节点" in text + # 口径卡接在双盲复算后面:复算时它还没算 + nodes = [node("start", "input"), node("dual", "dual_check", "复核", card="card"), CARD] + graph = {"nodes": nodes, "edges": [{"source": "start", "target": "dual"}, {"source": "dual", "target": "card"}]} + [(where, text)] = problems(graph) + assert where == "card" and "不在双盲复算的上游" in text + + +def test_without_a_card_there_must_be_exactly_one_upstream(): + [(where, text)] = problems(spec()) + assert where == "card" and "上游没有口径卡" in text + second = node("card2", "metrics", "利润口径卡", metrics=[{"id": "profit", "expression": "1"}]) + [(where, text)] = problems(spec(CARD, second)) + assert where == "card" and "2 张口径卡" in text and "「销售口径卡」" in text and "「利润口径卡」" in text + assert problems(spec(CARD, second, dual={"card": "card2"})) == [] + + +def test_metrics_and_definitions_must_be_on_the_card(): + found = dict(problems(spec(CARD, dual={"metrics": ["gmv", "refund"], + "definitions": {"gmv": "按支付日", "cost": "成本"}}))) + assert "「复算的指标」中的「refund」不是口径卡「销售口径卡」上的指标 ID" in found["metrics"] + assert "可选的指标 ID:gmv、orders" in found["metrics"] + assert "「口径说明」中的「cost」不是口径卡「销售口径卡」上的指标 ID" in found["definitions"] + [(where, text)] = problems(spec(CARD, dual={"definitions": ["gmv"]})) + assert where == "definitions" and "指标 ID → 口径说明" in text + + +def test_a_card_pinned_elsewhere_is_not_checked_for_metric_ids(): + pinned = node("card", "metrics", "引用的口径卡", caliber_from={"workflow_id": "w", "workflow_version": 1, "node_id": "c"}) + assert problems(spec(pinned, dual={"definitions": {"anything": "说明"}})) == [] + + +def test_option_values_are_checked(): + found = dict(problems(spec(CARD, dual={"enabled": "yes", "on_reject": "ignore", "tolerance": -1}))) + assert "「启用双盲复算」" in found["enabled"] + assert "「未采用原流程时」只能选「计入缺口、出具降档」或「判为失败」" in found["on_reject"] + assert "「允许的相对误差」需要填写不小于 0 的数" in found["tolerance"] + + +# -------------------------------------------------------------------------- +# 发起:设置快照 +# -------------------------------------------------------------------------- + + +@pytest.fixture +async def engine(): + await run_manager.setup() + async with SessionLocal() as session: + await session.execute(delete(Setting).where(Setting.key == "verify")) + await session.commit() + yield + async with SessionLocal() as session: + await session.execute(delete(Setting).where(Setting.key == "verify")) + await session.commit() + await run_manager.shutdown() + + +async def settle(run_id: str) -> Run: + for _ in range(400): + await asyncio.sleep(0.03) + async with SessionLocal() as session: + row = await session.get(Run, run_id) + if row.status in ("succeeded", "failed") and row.manifest_seq is not None: + await run_manager.wait_idle(run_id) + return row + raise AssertionError(f"运行 {run_id} 停在 {row.status}") + + +async def started_of(run_id: str) -> dict: + async with SessionLocal() as session: + return (await session.execute(select(RunEvent).where( + RunEvent.run_id == run_id, RunEvent.type == "run.started"))).scalars().first().data + + +PLAIN = {"nodes": [node("start", "input"), node("out", "output", fields=[{"name": "r", "value": "完成"}])], + "edges": [{"source": "start", "target": "out"}]} +GATED = {"nodes": [node("start", "input"), node("card", "metrics", metrics=[{"id": "x", "expression": "1"}]), + node("dual", "dual_check"), node("out", "output", fields=[])], + "edges": [{"source": "start", "target": "card"}, {"source": "card", "target": "dual"}, + {"source": "dual", "target": "out"}]} + + +async def test_a_graph_with_a_verify_node_snapshots_the_settings(engine): + from app.api.settings import verify_settings + from app.engine.launch import launch_run + + async with SessionLocal() as session: + session.add(Setting(key="verify", value={"dual_check": {"tolerance": 0.01}, "daily_max_usd": 3})) + await session.commit() + expected = await verify_settings(session) + run = await launch_run(session, graph=GATED, actor="测试") + assert expected["dual_check"]["tolerance"] == 0.01 and run.options == {"verify": expected} + row = await settle(run.id) + assert row.status == "succeeded", row.error + assert (await started_of(run.id))["options"] == {"verify": expected} + + +async def test_a_challenge_node_also_snapshots_the_settings(engine): + from app.engine.launch import _with_verify_snapshot + + graph = {"nodes": [node("start", "input"), node("review", "challenge")], "edges": []} + async with SessionLocal() as session: + out = await _with_verify_snapshot(session, graph, None) + assert set(out) == {"verify"} and out["verify"]["challenge"]["enabled"] is True + + +async def test_without_verify_nodes_options_and_run_started_are_unchanged(engine): + from app.engine.launch import launch_run + + async with SessionLocal() as session: + run = await launch_run(session, graph=PLAIN, actor="测试") + assert run.options == {} + await settle(run.id) + assert "options" not in await started_of(run.id) + + +async def test_options_given_by_the_caller_are_kept(engine): + from app.engine.launch import launch_run + + mine = {"verify": {"dual_check": {"enabled": False}}, "note": "例行"} + async with SessionLocal() as session: + run = await launch_run(session, graph=GATED, actor="测试", options=mine) + assert run.options == mine + await settle(run.id) + + +# -------------------------------------------------------------------------- +# 助手的节点说明 +# -------------------------------------------------------------------------- + + +def test_the_copilot_reference_explains_when_and_how_to_add_it(): + from app.api.copilot import NODE_REFERENCE + + assert "- dual_check:" in NODE_REFERENCE + block = NODE_REFERENCE.split("- dual_check:", 1)[1].split("\n- ", 1)[0] + for key in ("card", "metrics", "definitions", "provider", "model", "tolerance", "on_reject", "enabled"): + assert key in block, key + assert "正式出具" in block and "口径卡之后" in block and "degrade" in block and "fail" in block + # 排在口径卡和报告之间:搭图的先后就是这样 + at = NODE_REFERENCE.index + assert at("- metrics:") < at("- dual_check:") < at("- report:")