mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-16 04:42:01 +00:00
be164bd5c1
反方审稿:unit.challenge 带上节点产出里整份报告的计数(counts)和没能反查的句数(unreviewed,产出里有才带)。证据面板借审批卡的 展示画这一句,以前凭这一句拼出「被推翻 1 句、其余 0 句」,读着像整份报告的结果。节点设为「报错并停止运行」、句子被推翻时节点报错、 没有 node.finished,以前证据接口认不出这次审稿;现在按审完那一刻的 challenge_done 日志认(只认审稿记录存上了的,处置写 fail), 运行因此失败了也看得到被推翻的是哪几句。 双盲复算:节点以 no_inputs 直通时,那句说的是整张卡。以前原样套到卡上每个指标,配了「复算的指标」子集时,明明取自查询、只是没被 选上的指标也写成「口径卡上没有来自查询的输入」。现在按这个指标自己说:不在子集里、值不是数的是 not_targeted,自己没有来自查询的 输入的才是 no_inputs,关闭了的照旧 disabled;节点跑了、没复算的指标同样这样分。 遮罩:认哪个输入取自哪份查询结果的哪几列抽成 recompute.input_origin,证据面板、推演(之后还有可验证报告)同一个认法。核对不了的 宁可多遮,和推演、可验证报告的原则一致:cell()、Agent 字段落在的快照读不出来;按位置取的一格(以 .rows[行][列] 结尾)取值链上的 产出读不出来;那份结果没存快照、只带着数据源名。证据面板不给值,mask_unverified 写原因、不进遮罩列;推演按读不出来处理,不给值、 不推演。既没带快照 id 也没带数据源名的结果(数据整形改过形状)认不出出处,照常显示,写进交付说明。 recompute.recompute 给双盲复算的预检和盲值用,整个指标重算不了时原因不再说「推演」(_Plan.stuck,判断同 blocked,只换措辞)。 按位置的遮罩接进 recompute 之后,双盲复算的预检对「cell() 输入 + 按位置取自遮罩列的一格」的指标判无法复算(以前照常盲算,遮着的 号码还会作为字面量进盲算模型的计算式),运行对比里这种输入两边都不给值:各补一个用例钉住。 一个指标里同一件工件只读一次:按位置取了好几格时,写变量的节点那份可能几 MB 的产出以前每格重读一遍。 补测:遮罩列上 Agent 字段、cell() 对不上时 model_value / snapshot_value 不在答复里;代入式重新代不出来时整条不给。 test_recompute 的夹具补上数据整形节点 parse 的产出:以前缺着,按位置取的那一格现在会因此核对不了。 去掉实现后新用例各自变红(计数、跑停的审稿、子集的跳过原因、两处宁可多遮、推演、预检措辞、按位置遮罩、只读一次)。
114 lines
5.6 KiB
Python
114 lines
5.6 KiB
Python
"""按位置取自遮罩列的一格(vars.res.rows[1][2]、nodes.fetch.rows[0][0])在推演之外的两处:期 4 I2a 把它接进了
|
||
recompute 的还原(_positional),用 recompute 的地方都跟着变,这里把变化钉住。
|
||
|
||
- 双盲复算的预检(dual_check._plan_metric 什么都不改先重算一遍):以前这种指标照常盲算,遮着的那一格还会作为字面量
|
||
写进给盲算模型的计算式;现在预检判它用到了遮罩列,按无法复算记,原因不说「推演」(这里不是推演)
|
||
- 运行对比(engine/compare.py):这种输入两边的值都不给,标 masked,和 cell() 取的遮罩格一样
|
||
|
||
库和数字都是编的:号码写成带横线的文本,区域名是「东区」这类。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
|
||
from app.engine import compare
|
||
from app.engine.runner import run_manager
|
||
from tests.fixtures.compare_runs import BASE_ROWS, NEXT_ROWS, metric, scalar_in, week
|
||
from tests.test_evidence_metric_masks import ( # noqa: F401 - 夹具按名字注入(engine_up 自动生效)
|
||
METRICS,
|
||
PHONES,
|
||
SQL,
|
||
client,
|
||
engine_up,
|
||
node,
|
||
set_source,
|
||
shop,
|
||
)
|
||
|
||
|
||
def dual_graph():
|
||
"""取数 → 数据整形(vars.res)→ 口径卡 → 双盲复算(只复算 mix)→ 成果。"""
|
||
nodes = [
|
||
node("start", "input"),
|
||
node("fetch", "tool", tool="db_query__shop", args={"sql": SQL}),
|
||
node("parse", "transform", mode="json", template="{{ nodes.fetch }}", assign_to="res"),
|
||
node("card", "metrics", caliber="遮罩口径", caliber_version="v1", metrics=METRICS),
|
||
node("dual", "dual_check", metrics=["mix"]),
|
||
node("out", "output", fields=[{"name": "结果", "value": "{{ nodes.card.text }}"}]),
|
||
]
|
||
return {"nodes": nodes, "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]}
|
||
|
||
|
||
async def dual_output(run_id: str) -> dict:
|
||
from sqlalchemy import select
|
||
|
||
from app.core import artifact_store
|
||
from app.db.base import SessionLocal
|
||
from app.db.models import RunEvent
|
||
|
||
async with SessionLocal() as session:
|
||
[finished] = (await session.execute(select(RunEvent).where(
|
||
RunEvent.run_id == run_id, RunEvent.type == "node.finished", RunEvent.node_id == "dual"))).scalars()
|
||
return artifact_store.load(finished.data["artifact"])
|
||
|
||
|
||
async def test_the_dual_check_does_not_blind_compute_a_metric_reading_a_masked_cell_by_position(shop):
|
||
"""mix = round(cell(nodes.fetch, 1, 'amount') + len(vars.res.rows[1][2]), 1):cell() 那一格不在遮罩列上,按位置取的
|
||
号码在。预检判它用到了遮罩列,按无法复算记缺口,原因不说「推演」;没遮罩时预检照常通过、指标进入盲算。"""
|
||
import asyncio
|
||
|
||
from app.db.base import SessionLocal
|
||
from app.db.models import Run
|
||
|
||
async def run(options):
|
||
await set_source(shop, options)
|
||
run_id = (await run_manager.start(graph=dual_graph(), input_payload={})).id
|
||
for _ in range(400):
|
||
await asyncio.sleep(0.05)
|
||
async with SessionLocal() as session:
|
||
row = await session.get(Run, run_id)
|
||
if row.status in ("succeeded", "failed") and row.manifest_seq is not None:
|
||
await run_manager.wait_idle(run_id)
|
||
assert row.status == "succeeded", row.error
|
||
return await dual_output(run_id)
|
||
raise AssertionError(f"运行 {run_id} 没跑完:{row.status}")
|
||
|
||
masked = await run({"mask_columns": ["phone"]})
|
||
[mix] = masked["metrics"]
|
||
assert mix["metric_id"] == "mix" and mix["status"] == "unchecked"
|
||
assert mix["reason"] == "无法按原表达式重算这个指标(这个指标用到了设置了遮罩的列)"
|
||
assert "推演" not in json.dumps(masked, ensure_ascii=False)
|
||
assert not [p for p in PHONES if p in json.dumps(masked, ensure_ascii=False)]
|
||
assert any("遮罩" in gap for gap in masked["issuance_gaps"])
|
||
|
||
# 没遮罩:预检通过,原因不再是遮罩(盲算模型没配,按盲算没交回数记,但已经走到了盲算这一步)
|
||
plain = await run({})
|
||
[mix] = plain["metrics"]
|
||
assert "遮罩" not in (mix.get("reason") or "") and "无法按原表达式重算" not in (mix.get("reason") or "")
|
||
|
||
|
||
def region_len(fetch, bot, rows, bot_rows):
|
||
"""东区名称的字数:按位置取自取数结果的第 0 行第 0 列(region)。"""
|
||
return [metric("rlen", "len(nodes.fetch.rows[0][0])", len(rows[0][0]),
|
||
[scalar_in("nodes.fetch.rows[0][0]", rows[0][0], "fetch", "tool")], name="首个区域名字数",
|
||
unit="字", decimals=0)]
|
||
|
||
|
||
def test_the_compare_page_withholds_a_masked_cell_read_by_position():
|
||
"""运行对比:按位置取自遮罩列(region)的一格两边都不给值、标 masked,和 cell() 取的遮罩格一样;没遮罩时照给。"""
|
||
def masked(run):
|
||
run.masked = {a: frozenset({"region"}) for a in run.queries}
|
||
return run
|
||
|
||
result = compare.compare(masked(week("r1", metrics=region_len)).side(),
|
||
masked(week("r2", NEXT_ROWS, metrics=region_len)).side())
|
||
[rlen] = [m for m in result["metrics"] if m["metric_id"] == "rlen"]
|
||
[inp] = rlen["inputs"]
|
||
assert inp["masked"] is True and inp["base"] is None and inp["target"] is None
|
||
assert "东区" not in json.dumps(result, ensure_ascii=False)
|
||
|
||
shown = compare.compare(week("r1", metrics=region_len).side(), week("r2", NEXT_ROWS, metrics=region_len).side())
|
||
[rlen] = [m for m in shown["metrics"] if m["metric_id"] == "rlen"]
|
||
[inp] = rlen["inputs"]
|
||
assert not inp.get("masked") and inp["base"] == BASE_ROWS[0][0] and inp["target"] == NEXT_ROWS[0][0]
|