mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 06:41:58 +00:00
ThearchyHelios
1491b5c7f3
写数据目录（dual_check_facts.py）
- 有效记录条件只收「排除无效记录」的条件：被排除的取值是退款、作废、测试这类（或码值表里写着这类意思），
标记列只认留下有效记录的一边（is_test = 0、is_valid = 1、deleted_at IS NULL）；取值出现在指标名称、口径说明、
所在分组、运行输入里的，用到分组列的，一律不写，原因写明「可能是指标自己的筛选范围」。以前「华东销售额」的
region = '华东' 会被记成订单表人工确认过的有效记录条件，之后所有查订单表的工作流（含下一次盲算）都只看华东
- 目录里已有的有效记录条件被采纳的一边也按它筛了的，和新条件合起来写（已确认的不改，照实说）；没按它筛的不动。
以前整项替换，两边都用着的 is_test = 0 会被静默丢掉。推断的去重规则说法不同的也不替换
- 业务日期只在两边都按时间筛、按的列不同时才写；一边根本没按时间筛的不写
- 采纳了不去重的一边不写去重规则：那是指标自己的算法，写成「不去重」会把别的指标带偏
- 同一次裁决里几个指标对同一项给出矛盾的说法，整组都不写
节点（nodes/dual_check.py）
- 盲算任务里不再给原流程结果的列名和同一行的维度列名：分组的只给取值（日期说成「2026-09-02 这一天」），
原 SQL 写死的文字列连取值也不给；只有几项输入都来自查询又没写口径说明、列名看不出取舍时才给列名。
以前「order_date 为 2026-09-02」等于告诉盲算按下单日归属，下单日和支付日的陷阱正好漏掉
- 差异说明先把两条 SQL 收窄到这一格（只留算出这一列的那一项，分组的按这一行的分组值改成等值条件）再比，
多列查询、分组取一行时不再混进「选出的列」「分组」这类噪声，只差一个方面的也写得进目录
- 产出状态：全部一致才是 agreed，一个都没核成是 unchecked，核成一部分是 partial；产出和台账加 counts；
无法复算的指标发 dual_check_unchecked {metrics: [{id, name, reason}]}
- 共用一格的几个指标触顶时每个指标各发一条 dual_check_limit
- 查询次数刚好用完、交回的格子是真查出来的照常比对（时长、金额触顶仍按无法复算）
- 审核人没选或选了认不出的照实记「审核人没有对指标作出选择」，不再替人记成「两边都不对」
- 盲算先单独做第一个，其余最多 3 个同时进行，避免串行撞上运行单段的时长上限
配置
- 画布校验：双盲复算配了失败重试报错（重试会把盲算整轮重做、再问审核人一次）；enabled 用 is 判断，0、1 不再放过
- 发起时的核查设置快照：子工作流（按运行时的取法读）里有双盲复算、反方审稿节点时也快照
- 助手节点说明补一句不要给双盲复算配 retries
测试：补上遮罩两道防线、规范化后一致但值不同、两个输入都对不上、只复算配置的指标、原值为 0、摘要最多 3 条、
驳回、多列查询、分组取一行、指标筛选范围不写目录等用例；每条核心改动都做过去掉实现、测试变红的验证
244 lines
12 KiB
Python
244 lines
12 KiB
Python
"""双盲复算的配置、发起时的设置快照、助手的节点说明。
|
||
|
||
- 画布校验（engine/schema.py）：口径卡必须是上游的口径卡（没写时上游只能有一张），复算的指标、口径说明的键必须是
|
||
那张卡上的指标 ID，开关（只认 true / false，0、1 不算）、误差、处置的取值，不能配失败重试；报错写检查器上的中文叫法
|
||
- 发起（engine/launch.py）：图里（含子工作流）有双盲复算或反方审稿节点时把核查设置快照进 options["verify"]，没有时
|
||
options 不变、run.started 和以前逐字相同；调用方给了的照它的
|
||
- 助手的节点说明（api/copilot.py 的 NODE_REFERENCE）有 dual_check 一条：什么时候加、各配置项
|
||
|
||
表名、数字都是虚构的。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
|
||
import pytest
|
||
from sqlalchemy import delete, select
|
||
|
||
from app.db.base import SessionLocal
|
||
from app.db.models import Run, RunEvent, Setting, Workflow, WorkflowVersion
|
||
from app.engine.runner import run_manager
|
||
from app.engine.schema import GraphSpec, validate_graph
|
||
|
||
|
||
def node(nid, ntype, label=None, **config):
|
||
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": label or nid, "config": config}}
|
||
|
||
|
||
CARD = node("card", "metrics", "销售口径卡", caliber="销售口径", metrics=[
|
||
{"id": "gmv", "name": "销售额", "expression": "cell(nodes.fetch, 0, 'gmv')"},
|
||
{"id": "orders", "name": "订单数", "expression": "cell(nodes.fetch, 0, 'n')"}])
|
||
|
||
|
||
def spec(*middle: dict, dual: dict | None = None, edges: list | None = None) -> dict:
|
||
nodes = [node("start", "input"), node("fetch", "tool", tool="db_query__shop", args={"sql": "SELECT 1"}), *middle,
|
||
node("dual", "dual_check", "复核", **(dual or {})), node("out", "output", fields=[])]
|
||
chain = edges or [(a["id"], b["id"]) for a, b in zip(nodes, nodes[1:])]
|
||
return {"nodes": nodes, "edges": [{"source": s, "target": t} for s, t in chain]}
|
||
|
||
|
||
def problems(graph: dict) -> list[tuple[str | None, str]]:
|
||
report = validate_graph(GraphSpec.model_validate(graph))
|
||
return [(i.field, i.message) for i in report.issues if i.level == "error" and i.node_id == "dual"]
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 画布校验
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
def test_a_valid_config_has_no_problems():
|
||
assert problems(spec(CARD, dual={"card": "card", "metrics": ["gmv"], "definitions": {"gmv": "按支付日期，不含退款"},
|
||
"tolerance": 0.001, "on_reject": "fail", "enabled": True})) == []
|
||
# 不写口径卡：上游只有一张，就是它
|
||
assert problems(spec(CARD)) == []
|
||
|
||
|
||
def test_the_card_must_be_an_upstream_metrics_card():
|
||
[(where, text)] = problems(spec(CARD, dual={"card": "fetch"}))
|
||
assert where == "card" and "「口径卡」选择了「fetch」" in text and "不是口径卡" in text
|
||
[(where, text)] = problems(spec(CARD, dual={"card": "ghost"}))
|
||
assert where == "card" and "不存在这个节点" in text
|
||
# 口径卡接在双盲复算后面：复算时它还没算
|
||
nodes = [node("start", "input"), node("dual", "dual_check", "复核", card="card"), CARD]
|
||
graph = {"nodes": nodes, "edges": [{"source": "start", "target": "dual"}, {"source": "dual", "target": "card"}]}
|
||
[(where, text)] = problems(graph)
|
||
assert where == "card" and "不在双盲复算的上游" in text
|
||
|
||
|
||
def test_without_a_card_there_must_be_exactly_one_upstream():
|
||
[(where, text)] = problems(spec())
|
||
assert where == "card" and "上游没有口径卡" in text
|
||
second = node("card2", "metrics", "利润口径卡", metrics=[{"id": "profit", "expression": "1"}])
|
||
[(where, text)] = problems(spec(CARD, second))
|
||
assert where == "card" and "2 张口径卡" in text and "「销售口径卡」" in text and "「利润口径卡」" in text
|
||
assert problems(spec(CARD, second, dual={"card": "card2"})) == []
|
||
|
||
|
||
def test_metrics_and_definitions_must_be_on_the_card():
|
||
found = dict(problems(spec(CARD, dual={"metrics": ["gmv", "refund"],
|
||
"definitions": {"gmv": "按支付日", "cost": "成本"}})))
|
||
assert "「复算的指标」中的「refund」不是口径卡「销售口径卡」上的指标 ID" in found["metrics"]
|
||
assert "可选的指标 ID：gmv、orders" in found["metrics"]
|
||
assert "「口径说明」中的「cost」不是口径卡「销售口径卡」上的指标 ID" in found["definitions"]
|
||
[(where, text)] = problems(spec(CARD, dual={"definitions": ["gmv"]}))
|
||
assert where == "definitions" and "指标 ID → 口径说明" in text
|
||
|
||
|
||
def test_a_card_pinned_elsewhere_is_not_checked_for_metric_ids():
|
||
pinned = node("card", "metrics", "引用的口径卡", caliber_from={"workflow_id": "w", "workflow_version": 1, "node_id": "c"})
|
||
assert problems(spec(pinned, dual={"definitions": {"anything": "说明"}})) == []
|
||
|
||
|
||
def test_option_values_are_checked():
|
||
found = dict(problems(spec(CARD, dual={"enabled": "yes", "on_reject": "ignore", "tolerance": -1})))
|
||
assert "「启用双盲复算」" in found["enabled"]
|
||
assert "「未采用原流程时」只能选「计入缺口、出具降档」或「判为失败」" in found["on_reject"]
|
||
assert "「允许的相对误差」需要填写不小于 0 的数" in found["tolerance"]
|
||
|
||
|
||
@pytest.mark.parametrize("value", [0, 1, 0.0])
|
||
def test_zero_and_one_are_not_a_switch(value):
|
||
"""0 == False、1 == True：用 in 比会放过去，执行时却按「跟随设置」走（执行器认的是 True / False 本身）。"""
|
||
[(where, text)] = problems(spec(CARD, dual={"enabled": value}))
|
||
assert where == "enabled" and "「启用双盲复算」" in text
|
||
|
||
|
||
def test_retries_are_refused():
|
||
"""双盲复算失败都是确定的（审核人没采用原流程、口径卡不对）：重试只会把盲算再做一遍、再问审核人一次。"""
|
||
[(where, text)] = problems(spec(CARD, dual={"retries": 1}))
|
||
assert where == "retries" and "「失败重试次数」" in text and "双盲复算" in text
|
||
assert problems(spec(CARD, dual={"retries": 0})) == []
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 发起：设置快照
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.fixture
|
||
async def engine():
|
||
await run_manager.setup()
|
||
async with SessionLocal() as session:
|
||
await session.execute(delete(Setting).where(Setting.key == "verify"))
|
||
await session.commit()
|
||
yield
|
||
async with SessionLocal() as session:
|
||
await session.execute(delete(Setting).where(Setting.key == "verify"))
|
||
await session.commit()
|
||
await run_manager.shutdown()
|
||
|
||
|
||
async def settle(run_id: str) -> Run:
|
||
for _ in range(400):
|
||
await asyncio.sleep(0.03)
|
||
async with SessionLocal() as session:
|
||
row = await session.get(Run, run_id)
|
||
if row.status in ("succeeded", "failed") and row.manifest_seq is not None:
|
||
await run_manager.wait_idle(run_id)
|
||
return row
|
||
raise AssertionError(f"运行 {run_id} 停在 {row.status}")
|
||
|
||
|
||
async def started_of(run_id: str) -> dict:
|
||
async with SessionLocal() as session:
|
||
return (await session.execute(select(RunEvent).where(
|
||
RunEvent.run_id == run_id, RunEvent.type == "run.started"))).scalars().first().data
|
||
|
||
|
||
PLAIN = {"nodes": [node("start", "input"), node("out", "output", fields=[{"name": "r", "value": "完成"}])],
|
||
"edges": [{"source": "start", "target": "out"}]}
|
||
GATED = {"nodes": [node("start", "input"), node("card", "metrics", metrics=[{"id": "x", "expression": "1"}]),
|
||
node("dual", "dual_check"), node("out", "output", fields=[])],
|
||
"edges": [{"source": "start", "target": "card"}, {"source": "card", "target": "dual"},
|
||
{"source": "dual", "target": "out"}]}
|
||
|
||
|
||
async def test_a_graph_with_a_verify_node_snapshots_the_settings(engine):
|
||
from app.api.settings import verify_settings
|
||
from app.engine.launch import launch_run
|
||
|
||
async with SessionLocal() as session:
|
||
session.add(Setting(key="verify", value={"dual_check": {"tolerance": 0.01}, "daily_max_usd": 3}))
|
||
await session.commit()
|
||
expected = await verify_settings(session)
|
||
run = await launch_run(session, graph=GATED, actor="测试")
|
||
assert expected["dual_check"]["tolerance"] == 0.01 and run.options == {"verify": expected}
|
||
row = await settle(run.id)
|
||
assert row.status == "succeeded", row.error
|
||
assert (await started_of(run.id))["options"] == {"verify": expected}
|
||
|
||
|
||
async def test_a_challenge_node_also_snapshots_the_settings(engine):
|
||
from app.engine.launch import _with_verify_snapshot
|
||
|
||
graph = {"nodes": [node("start", "input"), node("review", "challenge")], "edges": []}
|
||
async with SessionLocal() as session:
|
||
out = await _with_verify_snapshot(session, graph, None)
|
||
assert set(out) == {"verify"} and out["verify"]["challenge"]["enabled"] is True
|
||
|
||
|
||
async def test_a_verify_node_inside_a_sub_workflow_also_snapshots_the_settings(engine):
|
||
"""子工作流里的双盲复算同样在运行中途读设置：按运行时的取法找子工作流（固定了版本的读那一版），任何一层有就快照。"""
|
||
from app.engine.launch import _with_verify_snapshot
|
||
|
||
inner_v1 = {"nodes": [node("start", "input"), node("out", "output", fields=[])], "edges": []}
|
||
inner_v2 = {"nodes": [node("start", "input"), node("review", "dual_check")], "edges": []}
|
||
async with SessionLocal() as session:
|
||
wf = Workflow(name="子流程-复核", graph=inner_v1, version=2)
|
||
session.add(wf)
|
||
await session.flush()
|
||
session.add_all([WorkflowVersion(workflow_id=wf.id, version=1, graph=inner_v1),
|
||
WorkflowVersion(workflow_id=wf.id, version=2, graph=inner_v2)])
|
||
await session.commit()
|
||
wid = wf.id
|
||
|
||
def outer(**config) -> dict:
|
||
return {"nodes": [node("start", "input"), node("sub", "subgraph", workflow_id=wid, **config)], "edges": []}
|
||
|
||
async with SessionLocal() as session:
|
||
pinned = await _with_verify_snapshot(session, outer(workflow_version=2), None)
|
||
old_pin = await _with_verify_snapshot(session, outer(workflow_version=1), None)
|
||
current = await _with_verify_snapshot(session, outer(), None)
|
||
assert set(pinned) == {"verify"} and pinned["verify"]["dual_check"]["enabled"] is True
|
||
# 固定在没有核查节点的那一版、或者读工作流现在的图（也没有）：options 原样不动
|
||
assert old_pin is None and current is None
|
||
|
||
|
||
async def test_without_verify_nodes_options_and_run_started_are_unchanged(engine):
|
||
from app.engine.launch import launch_run
|
||
|
||
async with SessionLocal() as session:
|
||
run = await launch_run(session, graph=PLAIN, actor="测试")
|
||
assert run.options == {}
|
||
await settle(run.id)
|
||
assert "options" not in await started_of(run.id)
|
||
|
||
|
||
async def test_options_given_by_the_caller_are_kept(engine):
|
||
from app.engine.launch import launch_run
|
||
|
||
mine = {"verify": {"dual_check": {"enabled": False}}, "note": "例行"}
|
||
async with SessionLocal() as session:
|
||
run = await launch_run(session, graph=GATED, actor="测试", options=mine)
|
||
assert run.options == mine
|
||
await settle(run.id)
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 助手的节点说明
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
def test_the_copilot_reference_explains_when_and_how_to_add_it():
|
||
from app.api.copilot import NODE_REFERENCE
|
||
|
||
assert "- dual_check：" in NODE_REFERENCE
|
||
block = NODE_REFERENCE.split("- dual_check：", 1)[1].split("\n- ", 1)[0]
|
||
for key in ("card", "metrics", "definitions", "provider", "model", "tolerance", "on_reject", "enabled"):
|
||
assert key in block, key
|
||
assert "正式出具" in block and "口径卡之后" in block and "degrade" in block and "fail" in block
|
||
# 排在口径卡和报告之间：搭图的先后就是这样
|
||
at = NODE_REFERENCE.index
|
||
assert at("- metrics：") < at("- dual_check：") < at("- report：")
|