mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 11:12:00 +00:00
ThearchyHelios
9d8b41430e
- 完善口径编辑、只读试运行、统一质量验证与执行语义指纹 - 增加可执行指标约束、有预算的主动调查和过程评测 - 提取发布检查、证据读取与起草服务，支持任务停止、重试和检查点恢复 - 增加版本传递影响分析，修复并发覆盖、敏感值遮罩、异常评分及页面状态问题 - 修复 SQLite 复合查询追溯和 microVM 可选依赖缺失处理 - 纳入当前设计文档及部署配置调整 验证：相关主回归 3822 项通过，4 项跳过、8 项预期失败；补充回归通过。 前端构建、文案检查及 git diff --check 通过。 全量检查仍有环境权限、PostgreSQL 可选依赖和既有部署配置、SQLite 文件基线差异导致的未通过项。
155 lines
8.3 KiB
Python
155 lines
8.3 KiB
Python
"""验证口径约束、执行身份和调查完成条件，覆盖会造成错误业务结论的输入。"""
|
|
from __future__ import annotations
|
|
|
|
import copy
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
from app.core.artifact_store import execution_hash, graph_hash
|
|
from app.data.caliber_identity import definition_hash, semantic_hash
|
|
from app.engine.investigation import InvestigationPolicy, assess
|
|
from app.engine.metric_spec import MetricContract, contract_findings, metric_expressions
|
|
from app.engine.validation import metric_quality, validation_report
|
|
from tests import ledger_samples as S
|
|
|
|
|
|
def test_execution_and_semantic_hashes_distinguish_layout_from_sql():
|
|
graph = S.caliber_graph()
|
|
layout = copy.deepcopy(graph)
|
|
layout["nodes"][1]["position"] = {"x": 999, "y": 777}
|
|
layout["nodes"].reverse()
|
|
assert graph_hash(layout) != graph_hash(graph) # 历史封存算法不改写
|
|
assert execution_hash(layout) == execution_hash(graph)
|
|
assert semantic_hash(layout, "card") == semantic_hash(graph, "card")
|
|
sql = copy.deepcopy(graph)
|
|
fetch = next(n for n in sql["nodes"] if n["id"] == "fetch")
|
|
fetch["data"]["config"]["args"]["sql"] = "SELECT SUM(amount) AS gmv FROM orders WHERE status='paid'"
|
|
assert semantic_hash(sql, "card") != semantic_hash(graph, "card")
|
|
assert execution_hash(sql) != execution_hash(graph)
|
|
|
|
|
|
@pytest.mark.parametrize("key,value", [("grain", "order"), ("filters", ["status=paid"]),
|
|
("time_field", "paid_at"), ("contract", {"shape": "scalar"})])
|
|
def test_declared_semantics_are_part_of_definition_identity(key, value):
|
|
metric = {"id": "n", "expression": "input.n"}
|
|
assert definition_hash("", "", [metric]) != definition_hash("", "", [{**metric, key: value}])
|
|
|
|
|
|
def test_inactive_formula_is_not_a_rule_dependency():
|
|
assert metric_expressions({"kind": "rule", "expression": "nodes.old.value",
|
|
"rule": {"inputs": {"n": "input.n"}}}) == ["input.n"]
|
|
|
|
|
|
def test_contract_catches_multiline_first_cell_and_duplicate_grain():
|
|
snapshot = {"columns": ["shop", "gmv"], "rows": [["east", 4], ["east", 7]]}
|
|
metric = {"value": 4}
|
|
bad = contract_findings({"contract": {"shape": "scalar"}}, metric, [snapshot])
|
|
assert any("恰好一行" in problem for problem in bad)
|
|
bad = contract_findings({"contract": {"shape": "table", "grain": ["shop"]}}, metric, [snapshot])
|
|
assert any("重复行" in problem for problem in bad)
|
|
|
|
|
|
@pytest.mark.parametrize("rows,value,problem", [([[2, 10]], .2, None), ([[2, 0]], 0, "分母"),
|
|
([[12, 10]], 1.2, "子集"), ([[2, 10]], 20, "倍率"),
|
|
([[None, 10]], .2, "有限")])
|
|
def test_ratio_contract_verifies_value_and_subset(rows, value, problem):
|
|
definition = {"contract": {"ratio": {"numerator": "refunds", "denominator": "orders"}}}
|
|
found = contract_findings(definition, {"value": value}, [{"columns": ["refunds", "orders"], "rows": rows}])
|
|
assert any(problem in p for p in found) if problem else not found
|
|
|
|
|
|
def test_contract_rejects_unknown_fields_and_nonfinite_bounds():
|
|
for raw in ({"minimum": float("nan")}, {"shape": "scalar", "typo": True}, {"minimum": 4, "maximum": 3}):
|
|
with pytest.raises(ValueError):
|
|
MetricContract.model_validate(raw)
|
|
|
|
|
|
def test_value_range_contract_can_validate_non_query_inputs():
|
|
definition = {"expression": "input.score", "contract": {"minimum": 0, "maximum": 100}}
|
|
assert contract_findings(definition, {"value": 85}, []) == []
|
|
assert contract_findings(definition, {"value": float("nan")}, [])
|
|
assert contract_findings({"contract": {"shape": "scalar"}}, {"value": 85}, [])
|
|
|
|
|
|
@pytest.mark.parametrize("flag", ["incomplete", "no_data", "sql_check_failed"])
|
|
def test_successful_execution_does_not_hide_data_quality_failure(flag):
|
|
metric = {"id": "n", "status": "ok", "value": 0, flag: True}
|
|
report = validation_report([metric], execution="succeeded", trusted=True)
|
|
assert report["status"] == "fail" and report["execution"] == "succeeded"
|
|
|
|
|
|
def test_unknown_comparability_and_unsealed_evidence_never_pass():
|
|
metric = {"id": "n", "status": "ok", "value": 0}
|
|
assert validation_report([metric], execution="succeeded", trusted=True)["status"] == "pass"
|
|
assert validation_report([metric], execution="succeeded", trusted=False)["status"] == "unknown"
|
|
assert metric_quality({**metric, "comparability": {"verdict": "conditional"}})["status"] == "unknown"
|
|
assert metric_quality({**metric, "value": float("nan")})["status"] == "fail"
|
|
|
|
|
|
def scope(*results):
|
|
return SimpleNamespace(queries={"Q1": {}}, results={f"A{i}": r for i, r in enumerate(results)})
|
|
|
|
|
|
def decomposition(*, depth=0, code="driver", caveats=None, share="0.9"):
|
|
return {"status": "ok", "op": "decompose", "mode": "additive", "params": {
|
|
"depth": depth, **({"parent": {"of": "A0", "key": ["east"]}} if depth else {})},
|
|
"table": {"columns": ["region", "share"], "rows": [["east", share]]},
|
|
"verdict": {"code": code}, "values": {"top_share": share}, "caveats": caveats or []}
|
|
|
|
|
|
def test_investigation_requires_closure_and_drill_before_completion():
|
|
policy = InvestigationPolicy()
|
|
question = "销售额为什么下降"
|
|
assert assess(scope(), question, policy)["stage"] == "decompose"
|
|
assert assess(scope(decomposition(code="unclosed")), question, policy)["stage"] == "closure"
|
|
assert assess(scope(decomposition()), question, policy)["stage"] == "drill"
|
|
done = assess(scope(decomposition(), decomposition(depth=1)), question, policy)
|
|
assert done["status"] == "complete" and done["coverage"] == pytest.approx(.81)
|
|
unknown = assess(scope(decomposition(), decomposition(depth=1, caveats=[{"code": "comparability"}])), question, policy)
|
|
assert unknown["status"] == "partial" and unknown["stage"] == "comparability"
|
|
|
|
|
|
def test_local_drill_coverage_does_not_count_as_coverage_of_the_whole_change():
|
|
found = assess(scope(decomposition(share="0.6"), decomposition(depth=1)), "销售额为什么下降", InvestigationPolicy())
|
|
assert found["status"] == "partial" and found["stage"] == "coverage"
|
|
assert found["coverage"] == pytest.approx(.54)
|
|
unclosed = assess(scope(decomposition(code="unclosed"), decomposition(depth=1)), "为什么下降", InvestigationPolicy())
|
|
assert unclosed["stage"] == "closure"
|
|
unlinked = assess(scope(decomposition(depth=1)), "为什么下降", InvestigationPolicy())
|
|
assert unlinked["stage"] == "lineage"
|
|
|
|
|
|
def test_anomaly_activates_investigation_but_plain_value_request_does_not():
|
|
policy = InvestigationPolicy()
|
|
assert not assess(scope(), "本周销售额是多少", policy)["active"]
|
|
anomaly = {"op": "baseline", "status": "ok", "verdict": {"code": "outside_band"}}
|
|
assert assess(scope(anomaly), "本周销售额是多少", policy)["active"]
|
|
assert not assess(scope(anomaly), "本周销售额是多少", InvestigationPolicy(enabled=False))["active"]
|
|
|
|
|
|
def test_eval_fingerprint_tracks_investigation_policy_changes(monkeypatch):
|
|
from app.engine import investigation
|
|
from app.eval.runner import prompt_fingerprint
|
|
|
|
before = prompt_fingerprint()
|
|
monkeypatch.setattr(investigation, "INVESTIGATION_VERSION", investigation.INVESTIGATION_VERSION + 1)
|
|
assert prompt_fingerprint() != before
|
|
|
|
|
|
def test_investigation_eval_checks_depth_budget_and_stopping_reason():
|
|
from app.eval.cases import _expectation
|
|
from app.eval.score import Answer, score_case
|
|
|
|
exp = _expectation({"id": "i", "kind": "investigation", "minimum_depth": 1, "max_queries": 4},
|
|
0, gold_kind=None, taken=set())
|
|
answer = Answer("run", investigations=[{"status": "complete", "depth": 1, "query_count": 3}])
|
|
case = {"expectations": [exp]}
|
|
assert score_case(case, None, answer, path="ran").verdict == "pass"
|
|
answer.investigations[0]["query_count"] = 5
|
|
assert score_case(case, None, answer, path="ran").verdict == "fail"
|
|
answer.investigations = [{"status": "partial", "depth": 0, "query_count": 2, "stop_reason": "evidence_insufficient"}]
|
|
partial = _expectation({"id": "i", "kind": "investigation", "status": "partial",
|
|
"stop_reason": "evidence_insufficient"}, 0, gold_kind=None, taken=set())
|
|
assert score_case({"expectations": [partial]}, None, answer, path="ran").verdict == "pass"
|