mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-22 10:02:01 +00:00
完善口径管理、主动调查与后台任务,修复通用问题
- 完善口径编辑、只读试运行、统一质量验证与执行语义指纹 - 增加可执行指标约束、有预算的主动调查和过程评测 - 提取发布检查、证据读取与起草服务,支持任务停止、重试和检查点恢复 - 增加版本传递影响分析,修复并发覆盖、敏感值遮罩、异常评分及页面状态问题 - 修复 SQLite 复合查询追溯和 microVM 可选依赖缺失处理 - 纳入当前设计文档及部署配置调整 验证:相关主回归 3822 项通过,4 项跳过、8 项预期失败;补充回归通过。 前端构建、文案检查及 git diff --check 通过。 全量检查仍有环境权限、PostgreSQL 可选依赖和既有部署配置、SQLite 文件基线差异导致的未通过项。
This commit is contained in:
@@ -105,9 +105,9 @@ def test_the_analysis_and_its_input_travel_with_the_file_and_verify():
|
||||
analysis_item = item(bundle, a)
|
||||
assert analysis_item["kind"] == ARTIFACT_KIND and "b64" in analysis_item and analysis_item["node_id"] == "an"
|
||||
# 导出时服务端真的复算了一遍:按封存的输入快照(这份文件里哈希对得上的原字节)重算,逐字段相同
|
||||
assert analysis_item["recheck"] == {"status": "same", "method": "decompose.additive@1",
|
||||
assert analysis_item["recheck"] == {"status": "same", "method": "decompose.additive@2",
|
||||
"text": "复算一致:按封存的输入重新计算,结论逐项相同"}
|
||||
# 方法的叫法由服务端带上(和导出页同一份),核对页只显示「变化拆解(可加) · decompose.additive@1」
|
||||
# 方法的叫法由服务端带上(和导出页同一份),核对页只显示「变化拆解(可加) · decompose.additive@2」
|
||||
assert analysis_item["method_label"] == "变化拆解(可加)"
|
||||
[query] = [x for x in bundle["artifacts"] if x["kind"] == "query"]
|
||||
assert "b64" in query
|
||||
|
||||
@@ -26,7 +26,7 @@ def checks(result):
|
||||
async def test_two_weeks_that_differ_only_in_dates_are_comparable():
|
||||
result = await compare(period("Q1", "2026-10-05", "2026-10-12"), period("Q2", "2026-10-12", "2026-10-19"),
|
||||
probe=fixed_prober(RANGE), measure="gmv")
|
||||
assert result["method"] == "comparable@1" and result["mode"] is None
|
||||
assert result["method"] == "comparable@2" and result["mode"] is None
|
||||
assert result["verdict"] == {"code": "comparable", "subjects": [], "direction": None}
|
||||
assert result["phrase"] == "两期可比"
|
||||
assert checks(result) == {"同口径": "通过", "窗口长度": "通过", "整期": "通过", "星期构成": "通过", "数据问题": "通过"}
|
||||
@@ -107,8 +107,8 @@ async def test_a_failed_probe_falls_back_to_profile_facts_then_says_it_cannot_ch
|
||||
assert fallback["probes"] == [{"source": "shop", "table": "orders", "column": "order_date", "min": "2025-01-01",
|
||||
"max": "2026-10-19", "basis": "profile", "at": "2026-10-20"}]
|
||||
nothing = await compare(base, target, measure="gmv")
|
||||
assert nothing["verdict"]["code"] == "comparable"
|
||||
assert nothing["phrase"] == "两期可比,整期核对不了" and nothing["probes"] == []
|
||||
assert nothing["verdict"]["code"] == "conditional"
|
||||
assert nothing["phrase"] == "两期有条件可比:整期核对不了" and nothing["probes"] == []
|
||||
|
||||
|
||||
async def test_bounds_computed_from_the_run_day_are_evaluated_in_the_database_and_recorded():
|
||||
@@ -127,7 +127,7 @@ async def test_bounds_computed_from_the_run_day_are_evaluated_in_the_database_an
|
||||
assert all(p["basis"] == "live" for p in result["probes"])
|
||||
# 界算不出来:窗口核对不了,不判不可比
|
||||
blind = await compare(base, target, probe=fixed_prober(RANGE), measure="gmv")
|
||||
assert blind["verdict"]["code"] == "comparable" and "窗口长度" in blind["phrase"]
|
||||
assert blind["verdict"]["code"] == "conditional" and "窗口长度" in blind["phrase"]
|
||||
|
||||
|
||||
async def test_data_problems_on_the_snapshots():
|
||||
@@ -180,7 +180,7 @@ async def test_a_masked_date_column_is_neither_used_nor_probed():
|
||||
result = await compare(period("Q1", "2026-10-05", "2026-10-12", masked=["order_date"]),
|
||||
period("Q2", "2026-10-12", "2026-10-19", masked=["order_date"]), probe=probe, measure="gmv")
|
||||
assert probe.calls == [] and result["probes"] == []
|
||||
assert result["verdict"]["code"] == "comparable" and "窗口长度" in result["phrase"]
|
||||
assert result["verdict"]["code"] == "conditional" and "窗口长度" in result["phrase"]
|
||||
|
||||
|
||||
async def test_one_snapshot_of_sqlite_weeks_uses_the_real_days_of_each_week():
|
||||
@@ -203,7 +203,8 @@ async def test_one_snapshot_of_sqlite_weeks_uses_the_real_days_of_each_week():
|
||||
# 不知道方言(认不出周号算法):窗口、整期、星期构成都核对不了,不判可比以外的结论
|
||||
plain = snap("Q1", sql, ["week", "orders"], rows, types={"week": "text", "orders": "number"})
|
||||
unknown = await analyze("comparable", snaps=[plain], query="Q1", period="week", base="2026-W41", target="2026-W42")
|
||||
assert unknown["phrase"] == "两期可比,窗口长度、整期、星期构成核对不了"
|
||||
assert unknown["verdict"]["code"] == "conditional"
|
||||
assert unknown["phrase"] == "两期有条件可比:窗口长度、整期、星期构成核对不了"
|
||||
|
||||
|
||||
def _same(a, b):
|
||||
|
||||
@@ -40,7 +40,7 @@ def _rows(base: dict, target: dict) -> list[list]:
|
||||
|
||||
async def test_a_driver_is_named_with_its_share_and_the_whole_shape_is_filled():
|
||||
result = await additive(label="销售额", hypothesis="华东拖累")
|
||||
assert result["method"] == "decompose.additive@1" and result["status"] == "ok"
|
||||
assert result["method"] == "decompose.additive@2" and result["status"] == "ok"
|
||||
assert result["verdict"] == {"code": "driver", "subjects": [["华东"]], "direction": "down"}
|
||||
assert result["phrase"] == "销售额下降 194(-8.7%),主要来自华东(-120,占 61.9%)"
|
||||
assert result["values"]["delta_total"] == "-194" and result["values"]["top_key"] == "华东"
|
||||
|
||||
@@ -24,7 +24,7 @@ async def rate(rows, **params):
|
||||
|
||||
async def test_small_samples_use_fisher_and_say_it_may_be_noise():
|
||||
result = await rate([["西南", 2, 40], ["华东", 5, 230]])
|
||||
assert result["method"] == "differ.rate@1"
|
||||
assert result["method"] == "differ.rate@2"
|
||||
assert result["verdict"] == {"code": "no_evidence", "subjects": [["西南"], ["华东"]], "direction": None}
|
||||
assert result["values"]["test"] == "fisher" and result["values"]["n_a"] == "40" and result["values"]["n_b"] == "230"
|
||||
assert Fraction(result["values"]["p"]) == pytest.approx(stats.fisher_exact(2, 38, 5, 225), rel=1e-9)
|
||||
@@ -91,7 +91,7 @@ async def test_means_from_sums_and_squares_use_welch():
|
||||
assert Fraction(result["values"]["p"]) == pytest.approx(expected.p, rel=1e-9)
|
||||
assert result["verdict"]["code"] == "differs" and result["values"]["test"] == "welch"
|
||||
assert result["phrase"] == "线上的客单价(50)高于线下(40),差异站得住"
|
||||
assert result["method"] == "differ.welch@1" and result["settings"] == {"differ_alpha": 0.05, "min_n": 30}
|
||||
assert result["method"] == "differ.welch@2" and result["settings"] == {"differ_alpha": 0.05, "min_n": 30}
|
||||
|
||||
|
||||
async def test_means_from_mean_sd_and_n_and_small_samples():
|
||||
@@ -111,7 +111,7 @@ async def test_rows_mode_flags_a_single_value_that_flips_the_verdict():
|
||||
online[-1] = ["线上", "231.3"]
|
||||
s = snap("Q1", "SELECT channel, amount FROM orders", ["channel", "amount"], online + base)
|
||||
result = await analyze("differ", "rows", snaps=[s], query="Q1", by="channel", value="amount", label="客单价")
|
||||
assert result["method"] == "differ.welch_rows@1"
|
||||
assert result["method"] == "differ.welch_rows@2"
|
||||
assert result["verdict"]["code"] == "differs"
|
||||
note = next(c for c in result["caveats"] if c["code"] == "outlier_sensitive")
|
||||
assert note["text"] == "去掉最大的一笔后,差异不再站得住"
|
||||
|
||||
@@ -184,7 +184,8 @@ async def test_check_rewrites_once_flags_the_wrong_source_and_keeps_the_tier(mon
|
||||
|
||||
# 出口独立复核:核出来的是同一处,只写进出具声明,档位和写对了的那次一样
|
||||
issuance = row.output["_issuance"]
|
||||
assert issuance["tier"] == clean.output["_issuance"]["tier"] == "formal"
|
||||
assert issuance["tier"] == clean.output["_issuance"]["tier"] == "degraded"
|
||||
assert any("异常调查未完成" in gap for gap in issuance["gaps"])
|
||||
assert _analysis_gaps(issuance) == []
|
||||
declared = issuance["analysis_claims"]
|
||||
assert (declared["mode"], declared["counted"], declared["counts"]) == ("check", False, {ac.MISMATCH: 1})
|
||||
|
||||
@@ -70,7 +70,7 @@ async def test_a_card_conclusion_opens_and_rechecks_everywhere(monkeypatch, shop
|
||||
judged = body["chain"][0]
|
||||
assert (judged["origin"], judged["item"]) == ("card", "where")
|
||||
again = next(s for s in body["chain"] if s["step"] == "analysis_recompute")
|
||||
assert again["status"] == "same" and again["method"] == "decompose.additive@1", again
|
||||
assert again["status"] == "same" and again["method"] == "decompose.additive@2", again
|
||||
inputs = [s for s in body["chain"] if s["step"] == "analysis_input"]
|
||||
assert [s["role"] for s in inputs] == ["headline_base", "headline_target", "data"]
|
||||
assert all(s["sealed"] for s in inputs)
|
||||
@@ -157,8 +157,8 @@ async def test_a_metric_comparability_opens_and_rechecks_everywhere(monkeypatch,
|
||||
view = body["chain"][0]["comparability"]
|
||||
# 表里只有周次这一列文字、没有业务日期:窗口长度、整期、星期构成核对不了,不判不可比
|
||||
assert (view["verdict"], view["phrase"], view["reasons"]) == (
|
||||
"comparable", "两期可比,窗口长度、整期、星期构成核对不了", [])
|
||||
assert view["sealed"] is True and view["hash_ok"] is True and view["method"] == "comparable@1"
|
||||
"conditional", "两期有条件可比:窗口长度、整期、星期构成核对不了", [])
|
||||
assert view["sealed"] is True and view["hash_ok"] is True and view["method"] == "comparable@2"
|
||||
steps = view["steps"]
|
||||
assert [s["step"] for s in steps] == ["analysis", "analysis_method", "analysis_calc", "analysis_input",
|
||||
"analysis_input", "analysis_recompute", "analysis_seal"]
|
||||
|
||||
@@ -578,7 +578,7 @@ def test_recheck_with_the_real_operators_says_a_sealed_result_holds():
|
||||
sealed = real_result(q)
|
||||
assert sealed["phrase"] == "销售额下降 194(-8.7%),主要来自华东(-120,占 61.9%)"
|
||||
out = analysis_recheck(sealed, store.get)
|
||||
assert out == {"status": "same", "method": "decompose.additive@1", "text": "复算一致:按封存的输入重新计算,结论逐项相同"}
|
||||
assert out == {"status": "same", "method": "decompose.additive@2", "text": "复算一致:按封存的输入重新计算,结论逐项相同"}
|
||||
assert analysis_recheck({**sealed, "hypothesis": "别的假设"}, store.get)["status"] == "same"
|
||||
|
||||
|
||||
@@ -593,7 +593,7 @@ def test_recheck_follows_the_parent_of_a_drill_down_by_hash():
|
||||
assert analysis_recheck(child, store.get, load_result=store.get)["status"] == "same"
|
||||
# 上一层取不回:照实说取不回哪一份,不说两遍「复算」
|
||||
out = analysis_recheck(child, store.get, load_result=lambda sha: None)
|
||||
assert out == {"status": "failed", "method": "decompose.additive@1", "text": "复算失败:取不回上一层的分析结论 A1"}
|
||||
assert out == {"status": "failed", "method": "decompose.additive@2", "text": "复算失败:取不回上一层的分析结论 A1"}
|
||||
|
||||
|
||||
def test_recheck_says_what_differs_and_what_is_missing():
|
||||
@@ -606,7 +606,7 @@ def test_recheck_says_what_differs_and_what_is_missing():
|
||||
assert out["text"] == "复算不一致:判定语和封存的不同"
|
||||
# 输入快照取不回(不在封存范围、哈希对不上的调用方都交 None)
|
||||
out = analysis_recheck(sealed, lambda sha: None)
|
||||
assert out == {"status": "failed", "method": "decompose.additive@1", "text": "复算失败:取不回 Q2 的查询快照"}
|
||||
assert out == {"status": "failed", "method": "decompose.additive@2", "text": "复算失败:取不回 Q2 的查询快照"}
|
||||
# 输入快照换了一份(同一个 sha256 交回了别的内容):复算照样算,结果和封存的不同
|
||||
other = {**SNAPSHOT, "rows": [["华东", "W41", 820], ["华东", "W42", 650], *SNAPSHOT["rows"][2:]]}
|
||||
assert analysis_recheck(sealed, lambda sha: other)["status"] == "different"
|
||||
@@ -619,7 +619,7 @@ def test_recheck_says_the_phrase_template_was_upgraded(monkeypatch):
|
||||
sealed = {**real_result(q), "phrase_version": 1, "phrase": "旧模板的判定语"}
|
||||
monkeypatch.setattr("app.analysis.phrases.PHRASE_VERSION", 2)
|
||||
out = analysis_recheck(sealed, store.get)
|
||||
assert out == {"status": "same", "method": "decompose.additive@1", "phrase_version": 1, "phrase_now": 2,
|
||||
assert out == {"status": "same", "method": "decompose.additive@2", "phrase_version": 1, "phrase_now": 2,
|
||||
"text": "说法模板升过版(v1 → v2),复算结果相同"}
|
||||
assert "fields" not in out
|
||||
old = {k: v for k, v in sealed.items() if k != "phrase_version"}
|
||||
@@ -632,21 +632,21 @@ def test_recheck_names_both_versions_when_the_method_was_upgraded(monkeypatch):
|
||||
store = Store()
|
||||
q = store.put(SNAPSHOT)
|
||||
sealed = real_result(q)
|
||||
monkeypatch.setitem(contract.METHODS, ("decompose", "additive"), "decompose.additive@2")
|
||||
monkeypatch.setitem(contract.METHODS, ("decompose", "additive"), "decompose.additive@3")
|
||||
out = analysis_recheck(sealed, store.get)
|
||||
assert out == {"status": "same", "method": "decompose.additive@1", "method_now": "decompose.additive@2",
|
||||
"text": "运行时用的是 decompose.additive@1,现在是 decompose.additive@2,复算结果相同"}
|
||||
assert out == {"status": "same", "method": "decompose.additive@2", "method_now": "decompose.additive@3",
|
||||
"text": "运行时用的是 decompose.additive@2,现在是 decompose.additive@3,复算结果相同"}
|
||||
out = analysis_recheck({**sealed, "phrase": "别的判定"}, store.get)
|
||||
assert out["status"] == "different" and out["fields"] == ["phrase"]
|
||||
assert out["text"] == "运行时用的是 decompose.additive@1,现在是 decompose.additive@2,复算结果不同"
|
||||
assert out["text"] == "运行时用的是 decompose.additive@2,现在是 decompose.additive@3,复算结果不同"
|
||||
|
||||
def refuse(result, load, *, load_result=None):
|
||||
raise CallError("unknown_mode", "变化拆解要写模式:additive / ratio / factor")
|
||||
|
||||
monkeypatch.setattr(analysis, "recompute", refuse)
|
||||
out = analysis_recheck(sealed, store.get)
|
||||
assert out["status"] == "failed" and out["method_now"] == "decompose.additive@2"
|
||||
assert out["text"] == ("运行时用的是 decompose.additive@1,现在是 decompose.additive@2:现在的方法按封存的输入复算不了"
|
||||
assert out["status"] == "failed" and out["method_now"] == "decompose.additive@3"
|
||||
assert out["text"] == ("运行时用的是 decompose.additive@2,现在是 decompose.additive@3:现在的方法按封存的输入复算不了"
|
||||
"(变化拆解要写模式:additive / ratio / factor)")
|
||||
|
||||
|
||||
|
||||
@@ -140,7 +140,7 @@ async def test_the_phrase_opens_judgment_method_calculation_inputs_recompute_and
|
||||
assert judged["hypothesis"] == "华东拖累" and judged["hypothesis_label"] == "分析卡上写的『在验证什么』"
|
||||
assert judged["hypothesis_note"].startswith("分析卡上写的『在验证什么』")
|
||||
assert {c["code"] for c in judged["caveats"]} == {"measure_unlabeled", "headline_basis", "comparability"}
|
||||
assert method["method"] == "decompose.additive@1" and method["method_label"] == "变化拆解(可加)"
|
||||
assert method["method"] == "decompose.additive@2" and method["method_label"] == "变化拆解(可加)"
|
||||
assert {s["key"]: s["label"] for s in method["settings"]}["main_share"] == "「主要来源」的占比"
|
||||
assert {s["key"] for s in method["settings"]} >= {"main_share", "concentration", "table_top_k"}
|
||||
assert "k_warn" not in {s["key"] for s in method["settings"]}
|
||||
@@ -154,7 +154,7 @@ async def test_the_phrase_opens_judgment_method_calculation_inputs_recompute_and
|
||||
assert query["alias"] == "Q1" and query["input_of"] == "A1" and query["total_rows"] == 8
|
||||
assert query["highlight"]["cols"] == ["region", "week", "gmv"]
|
||||
# 真的复算:按 sha256 取回封存的输入快照,逐字段和封存的比
|
||||
assert again == {"step": "analysis_recompute", "alias": "A1", "status": "same", "method": "decompose.additive@1",
|
||||
assert again == {"step": "analysis_recompute", "alias": "A1", "status": "same", "method": "decompose.additive@2",
|
||||
"text": "复算一致:按封存的输入重新计算,结论逐项相同"}
|
||||
assert seal["sealed"] is True and seal["ledger"] is True and seal["hash_ok"] is True
|
||||
assert body["seal"]["covered"] is True
|
||||
@@ -260,11 +260,11 @@ async def test_an_input_outside_the_sealed_scope_is_not_recomputed_from(client):
|
||||
|
||||
async def test_an_upgraded_method_names_both_versions(client, monkeypatch):
|
||||
made = await sealed_run(TEXT)
|
||||
monkeypatch.setitem(contract.METHODS, ("decompose", "additive"), "decompose.additive@2")
|
||||
monkeypatch.setitem(contract.METHODS, ("decompose", "additive"), "decompose.additive@3")
|
||||
body = (await client.get(f"/api/runs/{made['run']}/evidence/segments/{seg_id(made['doc'], 'a:A1')}")).json()
|
||||
again = next(s for s in body["chain"] if s["step"] == "analysis_recompute")
|
||||
assert again["method"] == "decompose.additive@1" and again["method_now"] == "decompose.additive@2"
|
||||
assert again["text"] == "运行时用的是 decompose.additive@1,现在是 decompose.additive@2,复算结果相同"
|
||||
assert again["method"] == "decompose.additive@2" and again["method_now"] == "decompose.additive@3"
|
||||
assert again["text"] == "运行时用的是 decompose.additive@2,现在是 decompose.additive@3,复算结果相同"
|
||||
|
||||
|
||||
async def test_recompute_errors_do_not_break_the_panel(client, monkeypatch):
|
||||
@@ -436,7 +436,7 @@ async def test_the_metric_step_opens_the_comparability_result(client):
|
||||
body = (await client.get(f"/api/runs/{made['run']}/evidence/segments/{seg_id(made['doc'], 'm:wow')}")).json()
|
||||
view = body["chain"][0]["comparability"]
|
||||
assert view["verdict"] == result["verdict"]["code"] and view["phrase"] == result["phrase"]
|
||||
assert view["sealed"] is True and view["hash_ok"] is True and view["method"] == "comparable@1"
|
||||
assert view["sealed"] is True and view["hash_ok"] is True and view["method"] == "comparable@2"
|
||||
steps = view["steps"]
|
||||
assert [s["step"] for s in steps] == ["analysis", "analysis_method", "analysis_calc", "analysis_input",
|
||||
"analysis_input", "analysis_recompute", "analysis_seal"]
|
||||
@@ -565,7 +565,7 @@ async def test_the_export_recomputes_each_conclusion_once(client, monkeypatch):
|
||||
"[[table:A1 cols=region,delta,share rows=0-3]]")
|
||||
made = await sealed_run(text, doc_extra=SCOPE)
|
||||
data = data_block((await client.get(f"/api/runs/{made['run']}/report.html")).text)
|
||||
assert calls == ["decompose.additive@1"]
|
||||
assert calls == ["decompose.additive@2"]
|
||||
cells = [s["id"] for s in iter_segments(made["doc"]) if str(s.get("ref") or "").startswith("v:A1.")]
|
||||
assert len(cells) == 12
|
||||
keys = {data["refs"][sid] for sid in [*cells, seg_id(made["doc"], "a:A1"), seg_id(made["doc"], "a:A1.top_share|pct")]}
|
||||
|
||||
@@ -32,7 +32,7 @@ def test_the_judge_gets_the_method_the_judgment_and_the_input_sql():
|
||||
[cand] = request.cands
|
||||
assert cand.aliases == ("A1",) and "A1" in cand.whole and ("A1", 0) in cand.rows and ("A1", "delta") in cand.cols
|
||||
excerpt = request.excerpts["A1"]
|
||||
assert excerpt.startswith("【A1】分析结论(变化拆解(可加),方法 decompose.additive@1)")
|
||||
assert excerpt.startswith("【A1】分析结论(变化拆解(可加),方法 decompose.additive@2)")
|
||||
assert "判定:销售额下降 194 万(-8.7%),主要来自华东(-120 万,占 62%)" in excerpt
|
||||
assert "判定标准:一段占总变化一半以上算主要来源" in excerpt
|
||||
assert "关键数:base_total=2,220 | target_total=2,026 | delta_total=-194" in excerpt
|
||||
|
||||
@@ -213,7 +213,7 @@ async def test_the_bundle_page_shows_the_analysis_detail_and_the_scope():
|
||||
phrase = next(text for text in out["opened"].values() if "出处分析结论 A1 的判定语" in text)
|
||||
assert "核对结果✓ 通过" in phrase and "分析结论里的值「销售额下降 194(-8.7%)" in phrase
|
||||
# 方法写服务端带来的叫法和 id(和导出页同一个写法),不只写 id
|
||||
assert "判定语销售额下降 194(-8.7%)" in phrase and "方法与版本变化拆解(可加) · decompose.additive@1" in phrase
|
||||
assert "判定语销售额下降 194(-8.7%)" in phrase and "方法与版本变化拆解(可加) · decompose.additive@2" in phrase
|
||||
assert "输入数据 · Q1" in phrase # 输入的角色写叫法,不写 data
|
||||
# 这份结论是分析卡做的(台账条目带 item):那一句「在验证什么」写「分析卡上写的『在验证什么』」,不标成 Agent 写的假设
|
||||
assert "分析卡上写的『在验证什么』" in phrase and "Agent 写的假设" not in phrase
|
||||
@@ -247,7 +247,7 @@ def test_the_bundle_page_shows_the_comparability_in_the_metric_detail():
|
||||
out = run_node(RUN_VERIFIER, VERIFIER, {"data": json.dumps(bundle), "seal": bundle["meta"]["manifest_hash"]})
|
||||
assert out["verified"] == ""
|
||||
[detail] = out["opened"].values()
|
||||
for text in ("两期可比性", "判定两期可比", "判定标准只差日期", "方法与版本可比性检查 · comparable@1", "各项检查", "同口径",
|
||||
for text in ("两期可比性", "判定两期有条件可比", "判定标准只差日期", "方法与版本可比性检查 · comparable@2", "各项检查", "同口径",
|
||||
"复算一致:按封存的输入重新计算,结论逐项相同(导出时服务端按封存的输入复算的结论"):
|
||||
assert text in detail, text
|
||||
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
"""验证口径约束、执行身份和调查完成条件,覆盖会造成错误业务结论的输入。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from app.core.artifact_store import execution_hash, graph_hash
|
||||
from app.data.caliber_identity import definition_hash, semantic_hash
|
||||
from app.engine.investigation import InvestigationPolicy, assess
|
||||
from app.engine.metric_spec import MetricContract, contract_findings, metric_expressions
|
||||
from app.engine.validation import metric_quality, validation_report
|
||||
from tests import ledger_samples as S
|
||||
|
||||
|
||||
def test_execution_and_semantic_hashes_distinguish_layout_from_sql():
|
||||
graph = S.caliber_graph()
|
||||
layout = copy.deepcopy(graph)
|
||||
layout["nodes"][1]["position"] = {"x": 999, "y": 777}
|
||||
layout["nodes"].reverse()
|
||||
assert graph_hash(layout) != graph_hash(graph) # 历史封存算法不改写
|
||||
assert execution_hash(layout) == execution_hash(graph)
|
||||
assert semantic_hash(layout, "card") == semantic_hash(graph, "card")
|
||||
sql = copy.deepcopy(graph)
|
||||
fetch = next(n for n in sql["nodes"] if n["id"] == "fetch")
|
||||
fetch["data"]["config"]["args"]["sql"] = "SELECT SUM(amount) AS gmv FROM orders WHERE status='paid'"
|
||||
assert semantic_hash(sql, "card") != semantic_hash(graph, "card")
|
||||
assert execution_hash(sql) != execution_hash(graph)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("key,value", [("grain", "order"), ("filters", ["status=paid"]),
|
||||
("time_field", "paid_at"), ("contract", {"shape": "scalar"})])
|
||||
def test_declared_semantics_are_part_of_definition_identity(key, value):
|
||||
metric = {"id": "n", "expression": "input.n"}
|
||||
assert definition_hash("", "", [metric]) != definition_hash("", "", [{**metric, key: value}])
|
||||
|
||||
|
||||
def test_inactive_formula_is_not_a_rule_dependency():
|
||||
assert metric_expressions({"kind": "rule", "expression": "nodes.old.value",
|
||||
"rule": {"inputs": {"n": "input.n"}}}) == ["input.n"]
|
||||
|
||||
|
||||
def test_contract_catches_multiline_first_cell_and_duplicate_grain():
|
||||
snapshot = {"columns": ["shop", "gmv"], "rows": [["east", 4], ["east", 7]]}
|
||||
metric = {"value": 4}
|
||||
bad = contract_findings({"contract": {"shape": "scalar"}}, metric, [snapshot])
|
||||
assert any("恰好一行" in problem for problem in bad)
|
||||
bad = contract_findings({"contract": {"shape": "table", "grain": ["shop"]}}, metric, [snapshot])
|
||||
assert any("重复行" in problem for problem in bad)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("rows,value,problem", [([[2, 10]], .2, None), ([[2, 0]], 0, "分母"),
|
||||
([[12, 10]], 1.2, "子集"), ([[2, 10]], 20, "倍率"),
|
||||
([[None, 10]], .2, "有限")])
|
||||
def test_ratio_contract_verifies_value_and_subset(rows, value, problem):
|
||||
definition = {"contract": {"ratio": {"numerator": "refunds", "denominator": "orders"}}}
|
||||
found = contract_findings(definition, {"value": value}, [{"columns": ["refunds", "orders"], "rows": rows}])
|
||||
assert any(problem in p for p in found) if problem else not found
|
||||
|
||||
|
||||
def test_contract_rejects_unknown_fields_and_nonfinite_bounds():
|
||||
for raw in ({"minimum": float("nan")}, {"shape": "scalar", "typo": True}, {"minimum": 4, "maximum": 3}):
|
||||
with pytest.raises(ValueError):
|
||||
MetricContract.model_validate(raw)
|
||||
|
||||
|
||||
def test_value_range_contract_can_validate_non_query_inputs():
|
||||
definition = {"expression": "input.score", "contract": {"minimum": 0, "maximum": 100}}
|
||||
assert contract_findings(definition, {"value": 85}, []) == []
|
||||
assert contract_findings(definition, {"value": float("nan")}, [])
|
||||
assert contract_findings({"contract": {"shape": "scalar"}}, {"value": 85}, [])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("flag", ["incomplete", "no_data", "sql_check_failed"])
|
||||
def test_successful_execution_does_not_hide_data_quality_failure(flag):
|
||||
metric = {"id": "n", "status": "ok", "value": 0, flag: True}
|
||||
report = validation_report([metric], execution="succeeded", trusted=True)
|
||||
assert report["status"] == "fail" and report["execution"] == "succeeded"
|
||||
|
||||
|
||||
def test_unknown_comparability_and_unsealed_evidence_never_pass():
|
||||
metric = {"id": "n", "status": "ok", "value": 0}
|
||||
assert validation_report([metric], execution="succeeded", trusted=True)["status"] == "pass"
|
||||
assert validation_report([metric], execution="succeeded", trusted=False)["status"] == "unknown"
|
||||
assert metric_quality({**metric, "comparability": {"verdict": "conditional"}})["status"] == "unknown"
|
||||
assert metric_quality({**metric, "value": float("nan")})["status"] == "fail"
|
||||
|
||||
|
||||
def scope(*results):
|
||||
return SimpleNamespace(queries={"Q1": {}}, results={f"A{i}": r for i, r in enumerate(results)})
|
||||
|
||||
|
||||
def decomposition(*, depth=0, code="driver", caveats=None, share="0.9"):
|
||||
return {"status": "ok", "op": "decompose", "mode": "additive", "params": {
|
||||
"depth": depth, **({"parent": {"of": "A0", "key": ["east"]}} if depth else {})},
|
||||
"table": {"columns": ["region", "share"], "rows": [["east", share]]},
|
||||
"verdict": {"code": code}, "values": {"top_share": share}, "caveats": caveats or []}
|
||||
|
||||
|
||||
def test_investigation_requires_closure_and_drill_before_completion():
|
||||
policy = InvestigationPolicy()
|
||||
question = "销售额为什么下降"
|
||||
assert assess(scope(), question, policy)["stage"] == "decompose"
|
||||
assert assess(scope(decomposition(code="unclosed")), question, policy)["stage"] == "closure"
|
||||
assert assess(scope(decomposition()), question, policy)["stage"] == "drill"
|
||||
done = assess(scope(decomposition(), decomposition(depth=1)), question, policy)
|
||||
assert done["status"] == "complete" and done["coverage"] == pytest.approx(.81)
|
||||
unknown = assess(scope(decomposition(), decomposition(depth=1, caveats=[{"code": "comparability"}])), question, policy)
|
||||
assert unknown["status"] == "partial" and unknown["stage"] == "comparability"
|
||||
|
||||
|
||||
def test_local_drill_coverage_does_not_count_as_coverage_of_the_whole_change():
|
||||
found = assess(scope(decomposition(share="0.6"), decomposition(depth=1)), "销售额为什么下降", InvestigationPolicy())
|
||||
assert found["status"] == "partial" and found["stage"] == "coverage"
|
||||
assert found["coverage"] == pytest.approx(.54)
|
||||
unclosed = assess(scope(decomposition(code="unclosed"), decomposition(depth=1)), "为什么下降", InvestigationPolicy())
|
||||
assert unclosed["stage"] == "closure"
|
||||
unlinked = assess(scope(decomposition(depth=1)), "为什么下降", InvestigationPolicy())
|
||||
assert unlinked["stage"] == "lineage"
|
||||
|
||||
|
||||
def test_anomaly_activates_investigation_but_plain_value_request_does_not():
|
||||
policy = InvestigationPolicy()
|
||||
assert not assess(scope(), "本周销售额是多少", policy)["active"]
|
||||
anomaly = {"op": "baseline", "status": "ok", "verdict": {"code": "outside_band"}}
|
||||
assert assess(scope(anomaly), "本周销售额是多少", policy)["active"]
|
||||
assert not assess(scope(anomaly), "本周销售额是多少", InvestigationPolicy(enabled=False))["active"]
|
||||
|
||||
|
||||
def test_eval_fingerprint_tracks_investigation_policy_changes(monkeypatch):
|
||||
from app.engine import investigation
|
||||
from app.eval.runner import prompt_fingerprint
|
||||
|
||||
before = prompt_fingerprint()
|
||||
monkeypatch.setattr(investigation, "INVESTIGATION_VERSION", investigation.INVESTIGATION_VERSION + 1)
|
||||
assert prompt_fingerprint() != before
|
||||
|
||||
|
||||
def test_investigation_eval_checks_depth_budget_and_stopping_reason():
|
||||
from app.eval.cases import _expectation
|
||||
from app.eval.score import Answer, score_case
|
||||
|
||||
exp = _expectation({"id": "i", "kind": "investigation", "minimum_depth": 1, "max_queries": 4},
|
||||
0, gold_kind=None, taken=set())
|
||||
answer = Answer("run", investigations=[{"status": "complete", "depth": 1, "query_count": 3}])
|
||||
case = {"expectations": [exp]}
|
||||
assert score_case(case, None, answer, path="ran").verdict == "pass"
|
||||
answer.investigations[0]["query_count"] = 5
|
||||
assert score_case(case, None, answer, path="ran").verdict == "fail"
|
||||
answer.investigations = [{"status": "partial", "depth": 0, "query_count": 2, "stop_reason": "evidence_insufficient"}]
|
||||
partial = _expectation({"id": "i", "kind": "investigation", "status": "partial",
|
||||
"stop_reason": "evidence_insufficient"}, 0, gold_kind=None, taken=set())
|
||||
assert score_case({"expectations": [partial]}, None, answer, path="ran").verdict == "pass"
|
||||
@@ -0,0 +1,18 @@
|
||||
from app.apps.impact import dependents
|
||||
from app.db.base import SessionLocal
|
||||
from tests.test_app_versions import chain, node, workflow
|
||||
|
||||
|
||||
async def test_impact_follows_caliber_and_subgraph_dependencies_transitively():
|
||||
root = await workflow({1: "published", 2: "published"}, current=2)
|
||||
one = chain(node("in", "input"), node("card", "metrics", caliber_from={
|
||||
"workflow_id": root, "workflow_version": 1, "node_id": "card"}))
|
||||
parent = await workflow({1: "published"}, current=1, graphs={1: one})
|
||||
two = chain(node("in", "input"), node("child", "subgraph", workflow_id=parent, workflow_version=1))
|
||||
grandparent = await workflow({1: "published"}, current=1, graphs={1: two})
|
||||
async with SessionLocal() as session:
|
||||
impact = await dependents(session, root, 1)
|
||||
unaffected = await dependents(session, root, 2)
|
||||
found = {(r["workflow_id"], r["depth"]) for r in impact["workflows"]}
|
||||
assert (parent, 1) in found and (grandparent, 2) in found
|
||||
assert not unaffected["workflows"]
|
||||
@@ -0,0 +1,52 @@
|
||||
"""真实 Agent 循环:模型提前结束时继续调查,不能补齐证据时明确降级。"""
|
||||
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
|
||||
|
||||
from app.core import artifact_store
|
||||
from app.engine.evidence_reader import read_sealed
|
||||
from app.eval.score import read_answer
|
||||
from app.providers import mock_model
|
||||
from tests.test_analysis_tools import (DECOMPOSE, DECOMPOSE_ARGS, SQL, TOOL, agent_graph, engine_up,
|
||||
events, run_graph, shop)
|
||||
|
||||
|
||||
async def test_runtime_continues_after_premature_answer_then_records_the_remaining_gap(monkeypatch, shop):
|
||||
calls = [(TOOL, {"sql": SQL}), (DECOMPOSE, DECOMPOSE_ARGS)]
|
||||
nudges = []
|
||||
|
||||
def decide(model, messages):
|
||||
continued = [m for m in messages if isinstance(m, HumanMessage) and "调查尚未完成" in str(m.content)]
|
||||
nudges[:] = continued
|
||||
if not continued:
|
||||
return AIMessage(content="销售额下降。")
|
||||
done = sum(isinstance(m, ToolMessage) for m in messages)
|
||||
if done < len(calls):
|
||||
name, args = calls[done]
|
||||
return AIMessage(content="", tool_calls=[{"id": f"c{done}", "name": name, "args": args}])
|
||||
return AIMessage(content="已按地区拆解,仍需核对总数。")
|
||||
|
||||
monkeypatch.setattr(mock_model.MockChatModel, "_decide", decide)
|
||||
row = await run_graph(agent_graph(max_steps=8))
|
||||
sealed = await read_sealed(row.id)
|
||||
bot = artifact_store.load(sealed.outputs["bot"])
|
||||
assert len(nudges) == 2
|
||||
assert [c["tool"] for c in bot["tool_calls"]] == [TOOL, DECOMPOSE]
|
||||
assert bot["investigation"]["status"] == "partial"
|
||||
assert bot["investigation"]["stage"] == "closure"
|
||||
assert bot["investigation"]["stop_reason"] == "evidence_insufficient"
|
||||
assert "现有证据不足以确认根因" in bot["text"]
|
||||
assert any(e.get("code") == "investigation_incomplete" for e in await events(row.id, "log", "bot"))
|
||||
answer = await read_answer(row.id)
|
||||
assert answer.investigations[0]["query_count"] == 1
|
||||
|
||||
|
||||
async def test_unfinished_investigation_cannot_receive_formal_issuance(monkeypatch, shop):
|
||||
monkeypatch.setattr(mock_model.MockChatModel, "_decide", lambda *_: AIMessage(content="尚未取得证据。"))
|
||||
graph = agent_graph(max_steps=4)
|
||||
graph["nodes"].insert(-1, {"id": "card", "type": "metrics", "data": {"config": {
|
||||
"caliber": "调查", "metrics": [{"id": "n", "expression": "1"}]}}})
|
||||
graph["edges"] = [{"source": a["id"], "target": b["id"]}
|
||||
for a, b in zip(graph["nodes"], graph["nodes"][1:])]
|
||||
graph["nodes"][-1]["data"]["config"]["contract"] = {"metrics_from": "card"}
|
||||
row = await run_graph(graph)
|
||||
assert row.output["_issuance"]["tier"] != "formal"
|
||||
assert any("异常调查未完成" in g for g in row.output["_issuance"]["gaps"])
|
||||
@@ -0,0 +1,176 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import copy
|
||||
import json
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import CaliberDraftJob, Workflow
|
||||
from app.ledger import draftplan, jobs
|
||||
from tests.test_ledger_docdraft_api import (client, datasource, documents, extract, fake_model, fake_trial, plan_reply)
|
||||
from tests.test_caliber_regressions import draft
|
||||
|
||||
|
||||
@pytest.mark.sqlite_only
|
||||
async def test_concurrent_draft_edit_is_not_overwritten_by_trial_completion(
|
||||
client, documents, datasource, monkeypatch):
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.api.ledger import _save_trial
|
||||
from app.db.models import CaliberDraft
|
||||
|
||||
body = await draft(client, documents, datasource, monkeypatch)
|
||||
execute = AsyncSession.execute
|
||||
raced = False
|
||||
|
||||
async def concurrent(session, statement, *args, **kwargs):
|
||||
nonlocal raced
|
||||
if (not raced and getattr(statement, "is_update", False)
|
||||
and statement.table.name == CaliberDraft.__tablename__):
|
||||
raced = True
|
||||
async with SessionLocal() as other:
|
||||
row = await other.get(CaliberDraft, body["id"])
|
||||
row.payload = {**row.payload, "guidance": "刚提交的审阅意见"}
|
||||
await other.commit()
|
||||
return await execute(session, statement, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(AsyncSession, "execute", concurrent)
|
||||
assert not await _save_trial(body["id"], {**body["trial"], "error": "较早试跑的结果"})
|
||||
assert raced
|
||||
async with SessionLocal() as session:
|
||||
current = await session.get(CaliberDraft, body["id"])
|
||||
assert current.payload["guidance"] == "刚提交的审阅意见"
|
||||
assert current.payload["trial"] == body["trial"]
|
||||
|
||||
|
||||
async def test_layout_save_keeps_trial_evidence_but_sql_change_invalidates(client, documents, datasource, monkeypatch):
|
||||
body = await draft(client, documents, datasource, monkeypatch)
|
||||
published = await client.post(f"/api/workflows/{body['workflow_id']}/publish", json={"level": "published"})
|
||||
assert published.status_code == 200 and published.json()["ok"]
|
||||
async with SessionLocal() as session:
|
||||
workflow = await session.get(Workflow, body["workflow_id"])
|
||||
graph = copy.deepcopy(workflow.graph)
|
||||
graph["nodes"][0]["position"] = {"x": 100, "y": 200}
|
||||
saved = await client.patch(f"/api/workflows/{body['workflow_id']}", json={"graph": graph})
|
||||
assert saved.status_code == 200
|
||||
assert saved.json()["same_as_published"] is True
|
||||
result = (await client.get(f"/api/ledger/drafts/{body['id']}/trial")).json()
|
||||
assert result["status"] == "succeeded" and result["version"] == 1
|
||||
fetch = next(n for n in graph["nodes"] if n["id"] == "fetch")
|
||||
fetch["data"]["config"]["args"]["sql"] = "SELECT 9 AS refund_rate FROM orders"
|
||||
changed = await client.patch(f"/api/workflows/{body['workflow_id']}", json={"graph": graph})
|
||||
assert changed.json()["same_as_published"] is False
|
||||
result = (await client.get(f"/api/ledger/drafts/{body['id']}/trial")).json()
|
||||
assert result["status"] == "stale" and result["metrics"] == []
|
||||
|
||||
|
||||
async def test_interrupted_authoring_resumes_completed_groups_only_for_same_material(
|
||||
client, documents, datasource, monkeypatch):
|
||||
doc = await documents("architecture-resume")
|
||||
source = await datasource()
|
||||
group = json.loads(plan_reply(f"{doc}-2"))
|
||||
calls = []
|
||||
|
||||
async def interrupted(_session, **kwargs):
|
||||
calls.append(list(kwargs["covered"]))
|
||||
if len(calls) == 1:
|
||||
return {"groups": [group], "uncovered": ["第二个指标"]}
|
||||
raise asyncio.CancelledError
|
||||
|
||||
monkeypatch.setattr(draftplan, "plan_groups", interrupted)
|
||||
made = await extract(client, doc, source)
|
||||
draft_id = made["draft_id"]
|
||||
body = (await client.get(f"/api/ledger/drafts/{draft_id}")).json()
|
||||
assert body["status"] == "failed" and body["job"]["status"] == "interrupted"
|
||||
assert body["groups"] and body["material_hash"]
|
||||
second = copy.deepcopy(group)
|
||||
second["metrics"][0].update(id="m2", name="第二个指标")
|
||||
|
||||
async def resume(_session, **kwargs):
|
||||
assert kwargs["covered"] == [group["metrics"][0]["name"]]
|
||||
return {"groups": [second], "uncovered": []}
|
||||
|
||||
monkeypatch.setattr(draftplan, "plan_groups", resume)
|
||||
reply = await client.post(f"/api/ledger/drafts/{draft_id}/retry", json={})
|
||||
assert reply.status_code == 200, reply.text
|
||||
body = (await client.get(f"/api/ledger/drafts/{draft_id}")).json()
|
||||
assert body["status"] == "ready" and len(body["metrics"]) == 2
|
||||
assert body["job"]["attempt"] == 2 and body["job"]["status"] == "succeeded"
|
||||
assert (await client.post(f"/api/ledger/drafts/{draft_id}/retry", json={})).status_code == 409
|
||||
|
||||
|
||||
async def test_parallel_group_repairs_have_independent_sessions(client, documents, datasource, monkeypatch):
|
||||
doc = await documents("architecture-sessions")
|
||||
source = await datasource()
|
||||
group = json.loads(plan_reply(f"{doc}-2"))
|
||||
broken = [{**copy.deepcopy(group), "sql": "SELECT * FROM unknown_table"} for _ in range(2)]
|
||||
broken[1]["metrics"][0].update(id="m2", name="另一个指标")
|
||||
sessions = []
|
||||
|
||||
async def plan(_session, **kwargs):
|
||||
return {"groups": broken, "uncovered": []}
|
||||
|
||||
async def repair(session, **kwargs):
|
||||
sessions.append(session)
|
||||
await session.execute(select(Workflow.id).limit(1))
|
||||
await asyncio.sleep(.01)
|
||||
return {**kwargs["group"], "sql": group["sql"]}
|
||||
|
||||
monkeypatch.setattr(draftplan, "plan_groups", plan)
|
||||
monkeypatch.setattr(draftplan, "repair_group", repair)
|
||||
made = await extract(client, doc, source)
|
||||
body = (await client.get(f"/api/ledger/drafts/{made['draft_id']}")).json()
|
||||
assert len(sessions) == 2 and sessions[0] is not sessions[1]
|
||||
assert body["status"] == "ready" and len(body["metrics"]) == 2
|
||||
|
||||
|
||||
async def test_startup_marks_orphaned_job_interrupted_and_preserves_progress(client, documents, datasource, monkeypatch):
|
||||
from app.db.models import CaliberDraft
|
||||
|
||||
async with SessionLocal() as session:
|
||||
row = CaliberDraft(title="中断任务", payload={"progress": {"round": 1}}, status="drafting")
|
||||
session.add(row)
|
||||
await session.flush()
|
||||
session.add(CaliberDraftJob(id=row.id, status="running"))
|
||||
await session.commit()
|
||||
draft_id = row.id
|
||||
await jobs.startup()
|
||||
body = (await client.get(f"/api/ledger/drafts/{draft_id}")).json()
|
||||
assert body["status"] == "failed" and body["job"]["status"] == "interrupted"
|
||||
assert body["progress"]["round"] == 1 and body["job"]["retryable"]
|
||||
|
||||
|
||||
async def test_stop_api_cancels_the_worker_and_preserves_checkpoint(client, monkeypatch):
|
||||
from app.db.models import CaliberDraft
|
||||
|
||||
checkpoint = {"groups": [{"metrics": [{"id": "m1", "name": "已完成指标"}]}]}
|
||||
async with SessionLocal() as session:
|
||||
row = CaliberDraft(title="停止任务", payload={"checkpoint": checkpoint}, status="drafting")
|
||||
session.add(row)
|
||||
await session.flush()
|
||||
session.add(CaliberDraftJob(id=row.id))
|
||||
await session.commit()
|
||||
draft_id = row.id
|
||||
entered = asyncio.Event()
|
||||
|
||||
async def generating(_draft_id):
|
||||
entered.set()
|
||||
await asyncio.Event().wait()
|
||||
|
||||
monkeypatch.setattr(jobs, "executor", generating)
|
||||
jobs.manager.start(draft_id)
|
||||
try:
|
||||
await asyncio.wait_for(entered.wait(), 3)
|
||||
edit = await client.patch(f"/api/ledger/drafts/{draft_id}", json={"metrics": []})
|
||||
assert edit.status_code == 409, edit.text
|
||||
result = await client.post(f"/api/ledger/drafts/{draft_id}/stop", json={})
|
||||
assert result.status_code == 200, result.text
|
||||
body = result.json()
|
||||
assert body["status"] == "failed" and body["job"]["status"] == "cancelled"
|
||||
assert body["checkpoint"] == checkpoint and body["job"]["retryable"]
|
||||
assert not jobs.manager.running(draft_id)
|
||||
finally:
|
||||
await jobs.manager.stop(draft_id, "user")
|
||||
@@ -155,7 +155,7 @@ async def test_old_graphs_without_the_mapping_are_unchanged():
|
||||
card = artifact_store.load(entry["artifact"])
|
||||
assert card["source"] == pinned and entry["source"] == pinned
|
||||
assert [m["expression"] for m in card["metrics"]] == ["cell(nodes.fetch,0,'gmv')"]
|
||||
assert set(card) == {"kind", "caliber", "caliber_version", "definition_hash", "metrics", "source"}
|
||||
assert set(card) == {"kind", "caliber", "caliber_version", "definition_hash", "semantic_hash", "validation_version", "metrics", "source"}
|
||||
assert card["definition_hash"]
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,337 @@
|
||||
"""口径审查回归:走真实保存/发布接口和隔离数据库,验证版本、只读试跑、质量门禁和组合复用。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import json
|
||||
import sqlite3
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.api import ledger as api_ledger
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import CaliberDraft, DataSource, Run, Workflow, WorkflowVersion
|
||||
from app.engine.runner import run_manager
|
||||
from app.engine.schema import GraphSpec, validate_graph
|
||||
from app.ledger import draftplan
|
||||
from tests import ledger_samples as S
|
||||
from tests.test_ledger_docdraft_api import (
|
||||
REAL_RUN_TRIAL, client, datasource, documents, extract, fake_model, fake_trial, groups_reply, plan_reply,
|
||||
)
|
||||
from tests.test_runtime_sqlcheck import (
|
||||
FANOUT_SQL, _drop_created_sources, _source, engine_up, output_of,
|
||||
)
|
||||
from tests.test_metric_no_data import plant, board as no_data_board
|
||||
|
||||
|
||||
def cfg(graph, nid):
|
||||
return next(n for n in graph["nodes"] if n["id"] == nid)["data"]["config"]
|
||||
|
||||
|
||||
async def draft(client, documents, datasource, monkeypatch, *, multi=False):
|
||||
doc = await documents("regression-" + uuid.uuid4().hex)
|
||||
source = await datasource()
|
||||
one = json.loads(plan_reply(doc + "-2"))
|
||||
if multi:
|
||||
two = copy.deepcopy(one)
|
||||
two["metrics"][0].update(id="m2", name="订单金额", expression="cell(nodes.fetch, 0, 'gmv')")
|
||||
two["sql"] = "SELECT SUM(amount) AS gmv FROM orders WHERE store = '{{ input.store }}'"
|
||||
reply = groups_reply(one, two)
|
||||
else:
|
||||
reply = json.dumps(one, ensure_ascii=False)
|
||||
fake_model(monkeypatch, reply)
|
||||
created = await extract(client, doc, source)
|
||||
got = await client.get("/api/ledger/drafts/" + created["draft_id"])
|
||||
assert got.status_code == 200, got.text
|
||||
body = got.json()
|
||||
assert body["status"] == "ready", body
|
||||
return body
|
||||
|
||||
|
||||
async def snapshots(wid):
|
||||
async with SessionLocal() as session:
|
||||
workflow = await session.get(Workflow, wid)
|
||||
versions = list((await session.execute(select(WorkflowVersion).where(
|
||||
WorkflowVersion.workflow_id == wid).order_by(WorkflowVersion.version))).scalars())
|
||||
return workflow, versions
|
||||
|
||||
|
||||
async def test_draft_edit_preserves_published_snapshot_and_updates_canvas(client, documents, datasource, monkeypatch):
|
||||
body = await draft(client, documents, datasource, monkeypatch)
|
||||
wid, did = body["workflow_id"], body["id"]
|
||||
published = await client.post(f"/api/workflows/{wid}/publish", json={"level": "published"})
|
||||
assert published.status_code == 200 and published.json()["ok"], published.text
|
||||
_, before = await snapshots(wid)
|
||||
metrics = copy.deepcopy(body["metrics"])
|
||||
metrics[0]["expression"] += " * 100"
|
||||
changed = await client.patch(f"/api/ledger/drafts/{did}", json={"metrics": metrics})
|
||||
assert changed.status_code == 200, changed.text
|
||||
workflow, after = await snapshots(wid)
|
||||
assert [v.version for v in after] == [1, 2]
|
||||
assert after[0].level == "published" and after[0].graph == before[0].graph
|
||||
assert after[0].graph_hash == before[0].graph_hash
|
||||
assert after[1].level is None and workflow.published_version == 1 and workflow.status == "draft"
|
||||
assert cfg(after[1].graph, "card")["metrics"][0]["expression"].endswith("* 100")
|
||||
assert workflow.graph == after[1].graph and changed.json()["version"] == 2
|
||||
|
||||
|
||||
async def test_trial_is_invalidated_by_edit_and_late_results_cannot_replace_new_trial(
|
||||
client, documents, datasource, monkeypatch):
|
||||
body = await draft(client, documents, datasource, monkeypatch)
|
||||
did = body["id"]
|
||||
metrics = copy.deepcopy(body["metrics"])
|
||||
metrics[0]["expression"] += " * 100"
|
||||
changed = (await client.patch(f"/api/ledger/drafts/{did}", json={"metrics": metrics})).json()
|
||||
assert changed["trial"]["status"] == "stale"
|
||||
assert not next(c for c in changed["checks"] if c["key"] == "trial")["ok"]
|
||||
old = (await client.get(f"/api/ledger/drafts/{did}/trial")).json()
|
||||
assert old["status"] == "stale" and old["metrics"] == [] and old["queries"] == []
|
||||
first = (await client.post(f"/api/ledger/drafts/{did}/trial", json={"inputs": {}})).json()
|
||||
latest = (await client.post(f"/api/ledger/drafts/{did}/trial", json={"inputs": {"store": "east"}})).json()
|
||||
assert first["version"] == latest["version"] == 2
|
||||
assert first["inputs_hash"] != latest["inputs_hash"]
|
||||
await api_ledger._save_trial(did, body["trial"])
|
||||
await api_ledger._save_trial(did, first)
|
||||
current = (await client.get(f"/api/ledger/drafts/{did}")).json()
|
||||
assert current["trial"]["run_id"] == latest["run_id"]
|
||||
assert current["trial"]["status"] == "succeeded"
|
||||
assert next(c for c in current["checks"] if c["key"] == "trial")["status"] == "unknown"
|
||||
|
||||
|
||||
async def test_pending_trial_failure_cannot_replace_a_newer_attempt(client, documents, datasource, monkeypatch):
|
||||
body = await draft(client, documents, datasource, monkeypatch)
|
||||
first = {**body["trial"], "trial_id": "first", "run_id": "", "status": "running"}
|
||||
second = {**first, "trial_id": "second"}
|
||||
assert await api_ledger._save_trial(body["id"], first, replace=True)
|
||||
assert await api_ledger._save_trial(body["id"], second, replace=True)
|
||||
assert not await api_ledger._save_trial(body["id"], {**first, "status": "failed"})
|
||||
current = (await client.get(f"/api/ledger/drafts/{body['id']}/trial")).json()
|
||||
assert current["trial_id"] == "second" and current["status"] == "running"
|
||||
|
||||
|
||||
async def test_metric_edits_preserve_canvas_sql_and_node_ids(client, documents, datasource, monkeypatch):
|
||||
body = await draft(client, documents, datasource, monkeypatch, multi=True)
|
||||
wid, did = body["workflow_id"], body["id"]
|
||||
workflow, _ = await snapshots(wid)
|
||||
graph = copy.deepcopy(workflow.graph)
|
||||
cfg(graph, "fetch_2")["args"]["sql"] = "SELECT SUM(amount) * 2 AS gmv FROM orders"
|
||||
cfg(graph, "start")["fields"].append({"name": "region", "type": "string", "required": True})
|
||||
saved = await client.patch(f"/api/workflows/{wid}", json={"graph": graph})
|
||||
assert saved.status_code == 200, saved.text
|
||||
kept = copy.deepcopy(body["metrics"][1])
|
||||
kept["name"] = "编辑第二组"
|
||||
patched = await client.patch(f"/api/ledger/drafts/{did}", json={"metrics": [kept]})
|
||||
assert patched.status_code == 200, patched.text
|
||||
workflow, versions = await snapshots(wid)
|
||||
latest = versions[-1].graph
|
||||
assert workflow.version == 3 and workflow.graph == latest
|
||||
assert cfg(latest, "fetch_2") == cfg(graph, "fetch_2")
|
||||
assert cfg(latest, "start") == cfg(graph, "start")
|
||||
assert cfg(latest, "card")["metrics"][0]["expression"] == kept["expression"]
|
||||
assert validate_graph(GraphSpec.model_validate(latest)).ok
|
||||
assert cfg(latest, "out")["contract"]["required"] == [kept["id"]]
|
||||
empty = await client.patch(f"/api/ledger/drafts/{did}", json={"metrics": []})
|
||||
assert empty.status_code == 200 and empty.json()["metrics"] == [], empty.text
|
||||
|
||||
|
||||
async def test_draft_list_counts_current_canvas_metrics(client, documents, datasource, monkeypatch):
|
||||
body = await draft(client, documents, datasource, monkeypatch)
|
||||
workflow, _ = await snapshots(body["workflow_id"])
|
||||
graph = copy.deepcopy(workflow.graph)
|
||||
metrics = cfg(graph, "card")["metrics"]
|
||||
metrics.append({**metrics[0], "id": "m2", "name": "另一个指标"})
|
||||
saved = await client.patch(f"/api/workflows/{workflow.id}", json={"graph": graph})
|
||||
assert saved.status_code == 200
|
||||
items = (await client.get("/api/ledger/drafts")).json()["drafts"]
|
||||
assert next(item for item in items if item["id"] == body["id"])["metric_count"] == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("draft_trial", [False, True])
|
||||
async def test_trials_never_write_even_when_source_is_writable_and_approval_disabled(
|
||||
client, tmp_path, engine_up, monkeypatch, draft_trial):
|
||||
path = tmp_path / "writable.db"
|
||||
name = await S.fresh_source(path)
|
||||
async with SessionLocal() as session:
|
||||
source = (await session.execute(select(DataSource).where(DataSource.name == name))).scalar_one()
|
||||
source.readonly = False
|
||||
await session.commit()
|
||||
graph = S.caliber_graph(sql="UPDATE orders SET amount = amount + 1 WHERE id = 1 RETURNING amount AS gmv")
|
||||
cfg(graph, "fetch")["approval"] = "never"
|
||||
wid = await S.publish(name="writable caliber " + uuid.uuid4().hex, graph=graph, level="published")
|
||||
published = await client.post(f"/api/workflows/{wid}/publish", json={"level": "published"})
|
||||
assert published.status_code == 200 and published.json()["ok"], published.text
|
||||
if draft_trial:
|
||||
async with SessionLocal() as session:
|
||||
workflow = await session.get(Workflow, wid)
|
||||
workflow.origin = {"kind": "doc_draft"}
|
||||
row = CaliberDraft(title="只读回归", workflow_id=wid, status="ready", payload={"version": 1})
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
did = row.id
|
||||
monkeypatch.setattr(api_ledger, "_run_trial", REAL_RUN_TRIAL)
|
||||
url = f"/api/ledger/drafts/{did}/trial"
|
||||
else:
|
||||
url = f"/api/ledger/calibers/{wid}/1/card/trial"
|
||||
result = await client.post(url, json={"inputs": {"start": "2026-09-01", "end": "2026-09-08"}})
|
||||
assert result.status_code == 200, result.text
|
||||
assert result.json()["status"] == "failed", result.text
|
||||
with sqlite3.connect(path) as db:
|
||||
assert db.execute("SELECT amount FROM orders WHERE id = 1").fetchone()[0] == 100
|
||||
async with SessionLocal() as session:
|
||||
run = await session.get(Run, result.json()["run_id"])
|
||||
assert run.options["read_only_data"] is True
|
||||
|
||||
|
||||
def definition(kind, path="cell(nodes.fetch, 0, 'gmv')"):
|
||||
if kind == "rule":
|
||||
return {"kind": "rule", "rule": {"inputs": {"x": path},
|
||||
"all": [{"input": "x", "op": "gte", "value": 50}], "then": 1, "else": 0}}
|
||||
if kind == "assessment":
|
||||
return {"kind": "assessment", "assessment": {
|
||||
"dimensions": [{"name": "x", "input": path, "min": 0, "max": 100000000}],
|
||||
"levels": [{"name": "normal", "min": 0}]}}
|
||||
return {"expression": path}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind", ["formula", "rule", "assessment"])
|
||||
async def test_all_metric_kinds_degrade_when_the_query_fails_sql_checks(scenic_db, engine_up, kind):
|
||||
_, source = await _source(scenic_db)
|
||||
graph = S.chain(
|
||||
S.node("start", "input"),
|
||||
S.node("fetch", "tool", tool=f"db_query__{source}", args={"sql": FANOUT_SQL}),
|
||||
S.node("card", "metrics", caliber="review", metrics=[{"id": "m", "name": "metric", **definition(kind)}]),
|
||||
S.node("out", "output", fields=[{"name": "value", "value": "{{ nodes.card.text }}"}],
|
||||
contract={"metrics_from": ["card"], "required": ["m"]}),
|
||||
)
|
||||
started = await run_manager.start(graph=graph, input_payload={})
|
||||
run = await S.wait(started.id)
|
||||
assert run.status == "succeeded", run.error
|
||||
metric = (await output_of(run.id, "card"))["metrics"][0]
|
||||
assert metric["sql_check_failed"] is True
|
||||
assert run.output["_issuance"]["tier"] == "degraded"
|
||||
assert any("SQL 检查" in gap for gap in run.output["_issuance"]["gaps"])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("first,second", [(0, 100), (False, True), ("2026-09-01", "2026-10-01")])
|
||||
def test_conflicting_defaults_require_explicit_input_even_after_more_groups(first, second):
|
||||
blocks = [{"entry_fields": [{"name": "scope", "type": "string", "default": value}]}
|
||||
for value in (first, second, second)]
|
||||
fields, notes = draftplan.merge_entry_fields(blocks)
|
||||
assert "default" not in fields[0] and fields[0]["required"] is True
|
||||
assert any("默认值不一致" in note for note in notes)
|
||||
graph = draftplan.build_graph(blocks, name="conflict")
|
||||
assert cfg(graph, "start")["fields"] == fields
|
||||
fields, notes = draftplan.merge_entry_fields([blocks[0], blocks[0]])
|
||||
assert fields[0]["default"] == first and not notes
|
||||
|
||||
|
||||
async def test_real_trial_runs_current_canvas_version(client, documents, datasource, monkeypatch, tmp_path, engine_up):
|
||||
body = await draft(client, documents, datasource, monkeypatch)
|
||||
path = tmp_path / "draft.db"
|
||||
with sqlite3.connect(path) as db:
|
||||
db.execute("CREATE TABLE orders (status TEXT, store TEXT, amount REAL, paid_at TEXT)")
|
||||
db.execute("INSERT INTO orders VALUES ('paid', '', 5, '2026-01-01')")
|
||||
async with SessionLocal() as session:
|
||||
source = await session.get(DataSource, body["datasource_id"])
|
||||
source.database, source.readonly, source.enabled = str(path), True, True
|
||||
await session.commit()
|
||||
workflow, _ = await snapshots(body["workflow_id"])
|
||||
graph = copy.deepcopy(workflow.graph)
|
||||
cfg(graph, "card")["metrics"][0]["expression"] += " * 100"
|
||||
saved = await client.patch("/api/workflows/" + body["workflow_id"], json={"graph": graph})
|
||||
assert saved.status_code == 200 and saved.json()["version"] == 2, saved.text
|
||||
monkeypatch.setattr(api_ledger, "_run_trial", REAL_RUN_TRIAL)
|
||||
result = await client.post("/api/ledger/drafts/" + body["id"] + "/trial", json={"inputs": {}})
|
||||
assert result.status_code == 200 and result.json()["status"] == "succeeded", result.text
|
||||
assert result.json()["metrics"][0]["value"] == 100
|
||||
async with SessionLocal() as session:
|
||||
run = await session.get(Run, result.json()["run_id"])
|
||||
assert run.origin["version"] == result.json()["version"] == 2
|
||||
current = (await client.get("/api/ledger/drafts/" + body["id"])).json()
|
||||
assert next(c for c in current["checks"] if c["key"] == "trial")["status"] == "pass"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind", ["rule", "assessment"])
|
||||
async def test_prefixed_reuse_preserves_nonformula_inputs(kind, tmp_path, engine_up):
|
||||
from app.ledger.derive import assemble, caliber_block, caliber_probe, load_pinned
|
||||
|
||||
await S.fresh_source(tmp_path / (kind + ".db"))
|
||||
graph = S.caliber_graph(sql="SELECT amount AS gmv FROM orders WHERE id = 1")
|
||||
cfg(graph, "card")["metrics"] = [{"id": "m", "name": "metric", **definition(kind)}]
|
||||
# 编辑器切类型会保留旧公式;依赖分析只认当前类型,不把未使用的旧输入也拷进来。
|
||||
cfg(graph, "card")["metrics"][0]["expression"] = "cell(nodes.former_fetch, 0, 'gmv')"
|
||||
original_graph = copy.deepcopy(graph)
|
||||
wid = await S.publish(name="regression " + kind, graph=graph)
|
||||
async with SessionLocal() as session:
|
||||
pinned = await load_pinned(session, wid, 1, "card")
|
||||
inputs = {"start": "2026-09-01", "end": "2026-09-08"}
|
||||
original = await run_manager.start(graph=caliber_probe(pinned).graph, input_payload=inputs)
|
||||
direct = await S.wait(original.id)
|
||||
combined = assemble("caliber_probe", [caliber_block(pinned, prefix="c1_")], fields=pinned.entry_fields)
|
||||
renamed = await run_manager.start(graph=combined.graph, input_payload=inputs)
|
||||
reused = await S.wait(renamed.id)
|
||||
assert direct.status == reused.status == "succeeded", reused.error
|
||||
before = (await output_of(direct.id, "card"))["metrics"][0]
|
||||
after = (await output_of(reused.id, "c1_card"))["metrics"][0]
|
||||
assert after["value"] == before["value"]
|
||||
assert all("c1_fetch" in item["path"] for item in after["inputs"])
|
||||
_, versions = await snapshots(wid)
|
||||
assert versions[0].graph == original_graph
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind", ["rule", "assessment"])
|
||||
async def test_nonformula_metrics_keep_no_data_gaps(kind, plant, engine_up):
|
||||
graph = no_data_board(plant)
|
||||
cfg(graph, "card")["metrics"] = [{"id": "m", "name": "metric",
|
||||
**definition(kind, "cell(nodes.q_kpi, 0, '上周到访人数')")}]
|
||||
cfg(graph, "out")["contract"]["required"] = ["m"]
|
||||
started = await run_manager.start(graph=graph, input_payload={})
|
||||
run = await S.wait(started.id)
|
||||
assert run.status == "succeeded", run.error
|
||||
metric = (await output_of(run.id, "card"))["metrics"][0]
|
||||
assert metric["no_data"] is True and metric["no_data_sources"]
|
||||
assert run.output["_issuance"]["tier"] == "degraded"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind", ["rule", "assessment"])
|
||||
@pytest.mark.parametrize("copied", [False, True])
|
||||
async def test_nonformula_metrics_keep_truncation_gaps(kind, copied, tmp_path, engine_up):
|
||||
await S.fresh_source(tmp_path / "truncated.db")
|
||||
nodes = [S.node("start", "input"),
|
||||
S.node("fetch", "tool", tool=S.tool(), args={"sql": "SELECT id FROM orders ORDER BY id", "limit": 2}),
|
||||
S.node("parse", "transform", mode="json", template="{{ nodes.fetch }}", assign_to="rows")]
|
||||
path = "len(vars.rows.rows)"
|
||||
if copied:
|
||||
nodes.append(S.node("upstream", "metrics", metrics=[{"id": "n", "expression": path}]))
|
||||
path = "nodes.upstream.metrics[0].value"
|
||||
nodes.extend([
|
||||
S.node("card", "metrics", caliber="truncated", metrics=[{"id": "m", "name": "metric", **definition(kind, path)}]),
|
||||
S.node("out", "output", fields=[{"name": "value", "value": "{{ nodes.card.text }}"}],
|
||||
contract={"metrics_from": ["card"], "required": ["m"]}),
|
||||
])
|
||||
started = await run_manager.start(graph=S.chain(*nodes), input_payload={})
|
||||
run = await S.wait(started.id)
|
||||
assert run.status == "succeeded", run.error
|
||||
metric = (await output_of(run.id, "card"))["metrics"][0]
|
||||
assert metric["incomplete"] is True and metric["truncated_sources"]
|
||||
assert run.output["_issuance"]["tier"] == "degraded"
|
||||
|
||||
|
||||
async def test_a_stale_writer_cannot_overwrite_a_new_workflow_revision():
|
||||
from app.apps.versions import WorkflowChanged, save_graph
|
||||
|
||||
graph = S.chain(S.node("start", "input"), S.node("out", "output"))
|
||||
wid = await S.publish(name="concurrent " + uuid.uuid4().hex, graph=graph)
|
||||
async with SessionLocal() as first, SessionLocal() as second:
|
||||
stale = await first.get(Workflow, wid)
|
||||
current = await second.get(Workflow, wid)
|
||||
changed = copy.deepcopy(graph)
|
||||
cfg(changed, "out")["fields"] = [{"name": "value", "value": "latest"}]
|
||||
assert await save_graph(second, current, changed)
|
||||
await second.commit()
|
||||
with pytest.raises(WorkflowChanged):
|
||||
await save_graph(first, stale, {**graph, "viewport": {"x": 1, "y": 1}})
|
||||
await first.rollback()
|
||||
workflow, versions = await snapshots(wid)
|
||||
assert workflow.graph == changed and len(versions) == 2
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
- 分析说法:出口按报告文档记下的那一项独立再核一遍(verify_doc)。strict 有违规记一条缺口(降档),报告节点报上来的同一条
|
||||
不重复收;check 只写进出具声明(analysis_claims,counted 为 false),不影响档位;没记这一项的老文档不核
|
||||
- 可比性:报告引用了的、契约必需的指标两期不可比记缺口(降档),有条件可比的只写进出具声明(comparability)
|
||||
- 可比性:报告引用了的、契约必需的指标两期不可比或有条件可比都记缺口(降档),并写进出具声明(comparability)
|
||||
- 血缘:报告引用的指标([[m:]] 片段、句末 [[see:m:…]])输入追不到原始数据的,受管级别的正式运行记缺口,其余写进出具声明的
|
||||
untraced(counted 为 false)
|
||||
"""
|
||||
@@ -190,7 +190,7 @@ def test_the_exit_policy_takes_the_stricter_of_the_doc_and_the_node():
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_comparability_gaps_only_for_incomparable_metrics():
|
||||
def test_comparability_gaps_for_incomparable_and_conditional_metrics():
|
||||
metrics = [
|
||||
{"id": "wow", "name": "环比增幅", "comparability": {"verdict": "incomparable", "phrase": "两期不可比:本期只有 3 天数据",
|
||||
"reasons": ["本期只有 3 天数据"], "artifact": "c" * 64}},
|
||||
@@ -200,7 +200,7 @@ def test_comparability_gaps_only_for_incomparable_metrics():
|
||||
{"id": "orders", "name": "订单数"},
|
||||
]
|
||||
gaps, declared = _comparability(metrics + metrics[:1])
|
||||
assert gaps == ["指标「环比增幅」两期不可比:本期只有 3 天数据"]
|
||||
assert gaps == ["指标「环比增幅」两期不可比:本期只有 3 天数据", "指标「客单价」两期有条件可比:两段的周末天数不同:4 天对 6 天"]
|
||||
assert [(d["metric"], d["verdict"]) for d in declared] == [("wow", "incomparable"), ("aov", "conditional")]
|
||||
assert declared[0]["reasons"] == ["本期只有 3 天数据"] and declared[0]["artifact"] == "c" * 64
|
||||
|
||||
@@ -222,7 +222,7 @@ def comparability(monkeypatch):
|
||||
return marks
|
||||
|
||||
|
||||
async def test_a_cited_incomparable_metric_degrades_and_a_conditional_one_is_noted(monkeypatch, comparability):
|
||||
async def test_a_cited_incomparable_or_conditional_metric_degrades(monkeypatch, comparability):
|
||||
comparability["wow"] = {"verdict": "incomparable", "phrase": "两期不可比:本期只有 3 天数据",
|
||||
"reasons": ["本期只有 3 天数据"]}
|
||||
writes(monkeypatch, "本周销售额 [[m:gmv]],环比 [[m:wow]]。[[see:m:gmv,m:wow]]")
|
||||
@@ -231,7 +231,7 @@ async def test_a_cited_incomparable_metric_degrades_and_a_conditional_one_is_not
|
||||
assert [c["metric"] for c in issuance["comparability"]] == ["wow"]
|
||||
comparability["wow"] = {"verdict": "conditional", "phrase": "两期有条件可比:两段的周末天数不同"}
|
||||
issuance = await issued(weekly())
|
||||
assert issuance["tier"] == "formal" and issuance["comparability"][0]["verdict"] == "conditional"
|
||||
assert issuance["tier"] == "degraded" and issuance["comparability"][0]["verdict"] == "conditional"
|
||||
|
||||
|
||||
async def test_an_incomparable_metric_nobody_cites_or_requires_is_left_alone(monkeypatch, comparability):
|
||||
|
||||
@@ -368,7 +368,12 @@ async def test_an_estimate_that_cannot_be_computed_still_marks_the_calls_unprice
|
||||
async def no_estimate(*args, **kwargs):
|
||||
return None
|
||||
|
||||
async def no_sources(*args, **kwargs):
|
||||
return []
|
||||
|
||||
monkeypatch.setattr(runner, "estimate", no_estimate)
|
||||
# 这道反问题只验证记账;不把其他测试留下的数据源纳入目录快照,以免先触发数据源数量上限。
|
||||
monkeypatch.setattr(runner.EvalExecution, "_sources", no_sources)
|
||||
before = await eval_spend.daily()
|
||||
suite_id = await es.make_suite("none", cases=[{"question": "问 1", "expectations": [{"kind": "ask"}]}],
|
||||
budget={"max_parallel": 1})
|
||||
|
||||
@@ -77,3 +77,28 @@ def test_a_job_without_a_lane_skips_the_queue():
|
||||
|
||||
run = _NoLane("whatever")
|
||||
assert run.lane_of(object()) is None
|
||||
|
||||
|
||||
def test_starting_an_active_job_twice_does_not_orphan_the_first_task():
|
||||
async def go():
|
||||
calls = []
|
||||
finish = asyncio.Event()
|
||||
|
||||
class Job:
|
||||
def __init__(self, job_id):
|
||||
self.job_id, self.task = job_id, None
|
||||
|
||||
async def run(self):
|
||||
calls.append(self.job_id)
|
||||
await finish.wait()
|
||||
|
||||
manager = job_runtime.JobManager(Job, label="幂等启动测试", name="test_start_once")
|
||||
manager.start("one")
|
||||
manager.start("one")
|
||||
await asyncio.sleep(0)
|
||||
assert calls == ["one"] and manager.running("one")
|
||||
finish.set()
|
||||
await manager.wait("one")
|
||||
assert not manager.running("one")
|
||||
|
||||
asyncio.run(go())
|
||||
|
||||
@@ -143,3 +143,15 @@ async def test_metric_synonyms_reach_the_descriptor(client):
|
||||
[made] = await descriptors(client, wf)
|
||||
[metric] = made["metrics"]
|
||||
assert metric["synonyms"] == ["月活", "MAU"]
|
||||
|
||||
|
||||
async def test_metric_constraints_and_semantic_identity_reach_the_descriptor(client):
|
||||
graph = S.caliber_graph()
|
||||
definition = {"shape": "scalar", "minimum": 0, "unit": "元"}
|
||||
for node in graph["nodes"]:
|
||||
if node["type"] == "metrics":
|
||||
node["data"]["config"]["metrics"][0].update(contract=definition, grain="汇总", time_field="支付时间")
|
||||
[made] = await descriptors(client, await S.publish(graph=graph))
|
||||
[metric] = made["metrics"]
|
||||
assert metric["contract"] == definition and metric["grain"] == "汇总"
|
||||
assert metric["time_field"] == "支付时间" and made["semantic_hash"]
|
||||
|
||||
@@ -57,6 +57,16 @@ def plan_reply(chunk_id: str, *, questions=None, sql: str | None = None, metrics
|
||||
}, ensure_ascii=False)
|
||||
|
||||
|
||||
def groups_reply(*plans: dict) -> str:
|
||||
"""一次交多组:每组沿用 plan_reply 的 sql / entry_fields / metrics。"""
|
||||
return json.dumps({
|
||||
"groups": [{"sql": plan["sql"], "entry_fields": plan["entry_fields"], "metrics": plan["metrics"]}
|
||||
for plan in plans],
|
||||
"open_questions": [],
|
||||
"uncovered": [],
|
||||
}, ensure_ascii=False)
|
||||
|
||||
|
||||
class FakeModel:
|
||||
"""起草模型:ainvoke 返回带 .content 的对象;按顺序交回复,交完重复最后一份。"""
|
||||
|
||||
@@ -96,16 +106,19 @@ async def fake_trial(monkeypatch):
|
||||
"""试跑换成写一行 succeeded 的运行:不启动引擎、不查库,接口和 payload 的形状照真的一样。"""
|
||||
made: list[str] = []
|
||||
|
||||
async def start(*, draft_id, workflow_id, graph, values, started_by, entry_fields=()):
|
||||
async def start(*, draft_id, workflow_id, graph, values, started_by, entry_fields=(), version=1):
|
||||
run_id = f"run-draft-{uuid.uuid4().hex[:10]}"
|
||||
async with SessionLocal() as session:
|
||||
session.add(Run(id=run_id, workflow_id=workflow_id, status="succeeded", graph=graph,
|
||||
input=values, run_class="exploratory", origin_kind="caliber_trial",
|
||||
origin={"kind": "caliber_trial", "draft_id": draft_id,
|
||||
"workflow_id": workflow_id, "version": 1}))
|
||||
"workflow_id": workflow_id, "version": version}))
|
||||
await session.commit()
|
||||
made.append(run_id)
|
||||
return {"run_id": run_id, "status": "succeeded", "error": ""}
|
||||
trial = {**api_ledger._trial_binding(version, graph, values),
|
||||
"run_id": run_id, "status": "succeeded", "error": ""}
|
||||
await api_ledger._save_trial(draft_id, trial, replace=True)
|
||||
return trial
|
||||
|
||||
monkeypatch.setattr(api_ledger, "_run_trial", start)
|
||||
yield made
|
||||
@@ -213,6 +226,10 @@ async def test_extract_produces_a_ready_draft_with_a_graph_and_a_trial(client, d
|
||||
assert made["alignment"]["new"] == ["已支付退款率"]
|
||||
assert made["truncated"] is False and made["model_id"] == "fake-model"
|
||||
assert made["datasource_id"] == source_id
|
||||
# 发布前检查:五项都从图和试跑状态算,不另起一份判断
|
||||
assert {check["key"] for check in made["checks"]} == {"graph", "sql", "entries", "trial", "citations"}
|
||||
assert all(check["ok"] for check in made["checks"] if check["key"] != "trial"), made["checks"]
|
||||
assert next(check for check in made["checks"] if check["key"] == "trial")["status"] == "unknown"
|
||||
|
||||
# 工作流和 v1 已建:合法的草稿图,取数指向这个数据源
|
||||
async with SessionLocal() as session:
|
||||
@@ -225,10 +242,27 @@ async def test_extract_produces_a_ready_draft_with_a_graph_and_a_trial(client, d
|
||||
assert [n["id"] for n in workflow.graph["nodes"]] == ["start", "fetch", "card", "write", "out"]
|
||||
assert validate_graph(GraphSpec.model_validate(workflow.graph)).ok
|
||||
|
||||
# 试跑只读接口:契约里的形状
|
||||
# 试跑只读接口:契约里的形状(queries 是取数节点交回的查询快照;这次是假试跑,没有快照)
|
||||
trial = (await client.get(f"/api/ledger/drafts/{body['draft_id']}/trial")).json()
|
||||
assert set(trial) == {"run_id", "status", "metrics", "error"}
|
||||
assert set(trial) == {"run_id", "status", "metrics", "queries", "error", "version", "graph_hash", "execution_hash", "inputs_hash", "validation"}
|
||||
assert trial["run_id"] == made["trial"]["run_id"] and trial["status"] == "succeeded"
|
||||
assert trial["queries"] == []
|
||||
|
||||
|
||||
async def test_publish_checks_point_at_an_entry_field_the_sql_never_uses(client, documents, datasource, monkeypatch):
|
||||
"""发布前检查照实说:声明了入口但 SQL 没用,也算不一致;其他项不跟着判失败。"""
|
||||
doc = await documents("dd-checks")
|
||||
source_id = await datasource()
|
||||
plan = json.loads(plan_reply(f"{doc}-2", sql="SELECT COUNT(*) * 1.0 AS refund_rate FROM orders "
|
||||
"WHERE status = 'paid'"))
|
||||
fake_model(monkeypatch, json.dumps(plan, ensure_ascii=False))
|
||||
draft_id = (await extract(client, doc, source_id))["draft_id"]
|
||||
|
||||
made = (await client.get(f"/api/ledger/drafts/{draft_id}")).json()
|
||||
by_key = {check["key"]: check for check in made["checks"]}
|
||||
assert by_key["entries"]["ok"] is False and "store" in by_key["entries"]["detail"]
|
||||
assert by_key["graph"]["ok"] and by_key["sql"]["ok"] and by_key["citations"]["ok"]
|
||||
assert by_key["trial"]["status"] == "unknown"
|
||||
|
||||
|
||||
async def test_open_questions_keep_the_draft_in_questions_open_but_still_run_it(
|
||||
@@ -240,6 +274,7 @@ async def test_open_questions_keep_the_draft_in_questions_open_but_still_run_it(
|
||||
body = await extract(client, doc, source_id)
|
||||
made = (await client.get(f"/api/ledger/drafts/{body['draft_id']}")).json()
|
||||
assert made["status"] == "questions_open"
|
||||
assert made["stage"] == "review"
|
||||
assert made["open_questions"] == ["退款算在哪一天"]
|
||||
assert made["workflow_id"] and made["sql"]
|
||||
assert made["trial"]["status"] == "succeeded"
|
||||
@@ -318,6 +353,15 @@ async def test_patch_replaces_metrics_and_clears_resolved_questions(client, docu
|
||||
assert [m["name"] for m in made["metrics"]] == ["已支付退款率(按支付日)"]
|
||||
assert made["open_questions"] == [] and made["question_count"] == 0
|
||||
|
||||
# PATCH 和画布一样新增版本,读取新版本而不是覆盖 v1
|
||||
async with SessionLocal() as session:
|
||||
draft_row = await session.get(CaliberDraft, draft_id)
|
||||
version_row = (await session.execute(select(WorkflowVersion).where(
|
||||
WorkflowVersion.workflow_id == draft_row.workflow_id,
|
||||
WorkflowVersion.version == made["version"]))).scalars().one()
|
||||
card = next(n for n in version_row.graph["nodes"] if n["type"] == "metrics")
|
||||
assert [m["name"] for m in card["data"]["config"]["metrics"]] == ["已支付退款率(按支付日)"]
|
||||
|
||||
# 结构不对的指标 422,草稿不动
|
||||
bad = await client.patch(f"/api/ledger/drafts/{draft_id}", json={"metrics": [{"synonyms": ["x"]}]})
|
||||
assert bad.status_code == 422
|
||||
@@ -330,6 +374,94 @@ async def test_patch_replaces_metrics_and_clears_resolved_questions(client, docu
|
||||
assert res.json()["status"] == "questions_open" and res.json()["question_count"] == 1
|
||||
|
||||
|
||||
async def test_draft_view_reads_the_graph_so_canvas_edits_show_up(client, documents, datasource, monkeypatch):
|
||||
"""画布改了指标名和取数 SQL 之后,草稿页从图里读最新的那几项,不再显示起草时的 payload 快照。"""
|
||||
doc = await documents("dd-live")
|
||||
source_id = await datasource()
|
||||
fake_model(monkeypatch, plan_reply(f"{doc}-2"))
|
||||
draft_id = (await extract(client, doc, source_id))["draft_id"]
|
||||
|
||||
# 必须走画布的真实保存接口:它新增 v2,不能直接覆盖 v1 来模拟
|
||||
async with SessionLocal() as session:
|
||||
draft = await session.get(CaliberDraft, draft_id)
|
||||
row = (await session.execute(select(WorkflowVersion).where(
|
||||
WorkflowVersion.workflow_id == draft.workflow_id,
|
||||
WorkflowVersion.version == 1))).scalars().one()
|
||||
graph = copy.deepcopy(row.graph)
|
||||
for node in graph["nodes"]:
|
||||
if node["type"] == "metrics":
|
||||
node["data"]["config"]["metrics"][0]["name"] = "已支付退款率(画布改过)"
|
||||
if node["type"] == "tool" and (node["data"]["config"].get("args") or {}).get("sql"):
|
||||
node["data"]["config"]["args"]["sql"] = "SELECT 1 AS refund_rate"
|
||||
workflow_id = draft.workflow_id
|
||||
saved = await client.patch(f"/api/workflows/{workflow_id}", json={"graph": graph})
|
||||
assert saved.status_code == 200 and saved.json()["version"] == 2, saved.text
|
||||
|
||||
made = (await client.get(f"/api/ledger/drafts/{draft_id}")).json()
|
||||
assert made["metrics"][0]["name"] == "已支付退款率(画布改过)"
|
||||
assert made["sql"] == "SELECT 1 AS refund_rate"
|
||||
assert [group["sql"] for group in made["groups"]] == ["SELECT 1 AS refund_rate"]
|
||||
assert made["version"] == 2 and made["trial"]["status"] == "stale"
|
||||
|
||||
|
||||
async def test_resolving_a_metric_question_writes_it_back_to_the_graph(client, documents, datasource, monkeypatch):
|
||||
"""答清一条指标自己的未决问题之后,图里那条指标的 open_questions 也要去掉:草稿页从图读,不能又显示出来。"""
|
||||
doc = await documents("dd-resolve")
|
||||
source_id = await datasource()
|
||||
plan = json.loads(plan_reply(f"{doc}-2"))
|
||||
plan["metrics"][0]["open_questions"] = ["按哪个门店口径"]
|
||||
fake_model(monkeypatch, json.dumps(plan, ensure_ascii=False))
|
||||
draft_id = (await extract(client, doc, source_id))["draft_id"]
|
||||
|
||||
made = (await client.get(f"/api/ledger/drafts/{draft_id}")).json()
|
||||
assert made["status"] == "questions_open"
|
||||
assert made["metrics"][0]["open_questions"] == ["按哪个门店口径"]
|
||||
|
||||
res = await client.patch(f"/api/ledger/drafts/{draft_id}", json={"resolved": ["按哪个门店口径"]})
|
||||
assert res.status_code == 200, res.text
|
||||
fixed = res.json()
|
||||
assert fixed["status"] == "ready" and fixed["question_count"] == 0
|
||||
assert fixed["metrics"][0]["open_questions"] == []
|
||||
|
||||
async with SessionLocal() as session:
|
||||
draft = await session.get(CaliberDraft, draft_id)
|
||||
row = (await session.execute(select(WorkflowVersion).where(
|
||||
WorkflowVersion.workflow_id == draft.workflow_id,
|
||||
WorkflowVersion.version == fixed["version"]))).scalars().one()
|
||||
card = next(n for n in row.graph["nodes"] if n["type"] == "metrics")
|
||||
assert not card["data"]["config"]["metrics"][0].get("open_questions")
|
||||
|
||||
|
||||
async def test_trial_validates_against_graph_entry_fields_after_canvas_edit(
|
||||
client, documents, datasource, monkeypatch):
|
||||
"""画布上给入口加了必填字段之后,试跑按图里的字段校验;payload 的旧入口字段不能把新字段漏掉。"""
|
||||
doc = await documents("dd-entry-live")
|
||||
source_id = await datasource()
|
||||
fake_model(monkeypatch, plan_reply(f"{doc}-2"))
|
||||
draft_id = (await extract(client, doc, source_id))["draft_id"]
|
||||
|
||||
async with SessionLocal() as session:
|
||||
draft = await session.get(CaliberDraft, draft_id)
|
||||
row = (await session.execute(select(WorkflowVersion).where(
|
||||
WorkflowVersion.workflow_id == draft.workflow_id,
|
||||
WorkflowVersion.version == 1))).scalars().one()
|
||||
graph = copy.deepcopy(row.graph)
|
||||
for node in graph["nodes"]:
|
||||
if node["type"] == "input":
|
||||
node["data"]["config"]["fields"].append(
|
||||
{"name": "region", "type": "string", "required": True, "default": "",
|
||||
"description": "区域"})
|
||||
workflow_id = draft.workflow_id
|
||||
saved = await client.patch(f"/api/workflows/{workflow_id}", json={"graph": graph})
|
||||
assert saved.status_code == 200 and saved.json()["version"] == 2, saved.text
|
||||
|
||||
missing = await client.post(f"/api/ledger/drafts/{draft_id}/trial", json={"inputs": {}})
|
||||
assert missing.status_code == 422 and "region" in missing.text
|
||||
ok = await client.post(f"/api/ledger/drafts/{draft_id}/trial", json={"inputs": {"region": "华东"}})
|
||||
assert ok.status_code == 200, ok.text and ok.text[:200]
|
||||
assert ok.json()["status"] == "succeeded"
|
||||
|
||||
|
||||
async def test_dismiss_marks_rejected_and_writes_an_audit(client, documents, datasource, monkeypatch):
|
||||
doc = await documents("dd-7")
|
||||
source_id = await datasource()
|
||||
@@ -337,7 +469,7 @@ async def test_dismiss_marks_rejected_and_writes_an_audit(client, documents, dat
|
||||
draft_id = (await extract(client, doc, source_id))["draft_id"]
|
||||
|
||||
res = await client.post(f"/api/ledger/drafts/{draft_id}/dismiss", json={"note": "已经有口径了"})
|
||||
assert res.status_code == 200 and res.json()["status"] == "rejected"
|
||||
assert res.status_code == 200 and res.json()["status"] == "rejected" and res.json()["stage"] == "discarded"
|
||||
async with SessionLocal() as session:
|
||||
event = (await session.execute(select(AuditEvent).where(
|
||||
AuditEvent.action == "ledger.docdraft.dismiss", AuditEvent.subject_id == draft_id))).scalars().one()
|
||||
@@ -358,14 +490,20 @@ async def test_list_counts_orders_by_last_change_and_filters_by_status(client, d
|
||||
listed = (await client.get("/api/ledger/drafts")).json()["drafts"]
|
||||
assert [d["id"] for d in listed] == [two, one]
|
||||
latest, earlier = listed
|
||||
assert (latest["title"], latest["status"], latest["metric_count"], latest["question_count"]) == \
|
||||
("二.md", "ready", 1, 0)
|
||||
assert (earlier["status"], earlier["metric_count"], earlier["question_count"]) == ("questions_open", 1, 1)
|
||||
assert (latest["title"], latest["status"], latest["stage"], latest["metric_count"], latest["question_count"]) == \
|
||||
("二.md", "ready", "review", 1, 0)
|
||||
assert (earlier["status"], earlier["stage"], earlier["metric_count"], earlier["question_count"]) == \
|
||||
("questions_open", "review", 1, 1)
|
||||
assert earlier["workflow_id"] and earlier["created_at"] and earlier["error"] is None
|
||||
|
||||
only = (await client.get("/api/ledger/drafts", params={"status": "questions_open"})).json()["drafts"]
|
||||
assert [d["id"] for d in only] == [one]
|
||||
# 界面按阶段筛:ready 和 questions_open 都是「待审」
|
||||
review = (await client.get("/api/ledger/drafts", params={"stage": "review"})).json()["drafts"]
|
||||
assert {d["id"] for d in review} == {one, two}
|
||||
assert (await client.get("/api/ledger/drafts", params={"stage": "discarded"})).json()["drafts"] == []
|
||||
assert (await client.get("/api/ledger/drafts", params={"status": "nope"})).status_code == 422
|
||||
assert (await client.get("/api/ledger/drafts", params={"stage": "nope"})).status_code == 422
|
||||
assert (await client.get("/api/ledger/drafts/d-nope")).status_code == 404
|
||||
|
||||
|
||||
@@ -382,9 +520,11 @@ async def test_trial_endpoint_returns_metric_values_from_the_run(client, documen
|
||||
res = await client.get(f"/api/ledger/drafts/{draft_id}/trial")
|
||||
assert res.status_code == 200, res.text
|
||||
body = res.json()
|
||||
assert body == {"run_id": body["run_id"], "status": "succeeded",
|
||||
assert body == {"run_id": body["run_id"], "status": "succeeded", "version": 1,
|
||||
"graph_hash": body["graph_hash"], "execution_hash": body["execution_hash"],
|
||||
"validation": body["validation"], "inputs_hash": body["inputs_hash"],
|
||||
"metrics": [{"id": "m1", "name": "已支付退款率", "rendered": "3.2%", "value": 0.032}],
|
||||
"error": ""}
|
||||
"queries": [], "error": ""}
|
||||
|
||||
# 没有试跑的老草稿
|
||||
async with SessionLocal() as session:
|
||||
@@ -489,6 +629,14 @@ async def test_a_real_trial_run_feeds_the_trial_endpoint(client, documents, tmp_
|
||||
assert [m["id"] for m in trial["metrics"]] == ["m1"]
|
||||
assert trial["metrics"][0]["name"] == "已支付销售额"
|
||||
assert float(trial["metrics"][0]["value"]) == 156.5
|
||||
# 试跑结果集:取数节点交回的查询快照,给列和行、不给 SQL
|
||||
assert [query["node_id"] for query in trial["queries"]] == ["fetch"]
|
||||
query = trial["queries"][0]
|
||||
assert set(query) == {"node_id", "tool", "source", "columns", "rows", "total_rows",
|
||||
"truncated", "masked", "source_truncated", "preview_truncated"}
|
||||
assert query["columns"] == ["gmv"] and query["rows"] == [[156.5]]
|
||||
assert query["total_rows"] == 1 and query["truncated"] is False and query["masked"] == []
|
||||
assert "sql" not in query
|
||||
async with SessionLocal() as session:
|
||||
run = await session.get(Run, run_id)
|
||||
assert run.origin_kind == "caliber_trial" and run.run_class == "exploratory"
|
||||
@@ -510,6 +658,64 @@ async def test_a_real_trial_run_feeds_the_trial_endpoint(client, documents, tmp_
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def test_one_reply_can_bring_several_independent_groups(client, documents, datasource, monkeypatch):
|
||||
"""模型一次交两组:只调一次模型,两组各自成块,不再一组一轮。"""
|
||||
doc = await documents("dd-groups")
|
||||
source_id = await datasource()
|
||||
first = json.loads(plan_reply(f"{doc}-2"))
|
||||
second = json.loads(plan_reply(f"{doc}-2"))
|
||||
second["sql"] = ("SELECT COUNT(*) AS orders FROM orders WHERE status = 'paid' "
|
||||
"AND store = '{{ input.store }}'")
|
||||
second["metrics"] = [{**second["metrics"][0], "id": "m1", "name": "已支付订单数",
|
||||
"expression": "cell(nodes.fetch, 0, 'orders')"}]
|
||||
model = fake_model(monkeypatch, groups_reply(first, second))
|
||||
|
||||
body = await extract(client, doc, source_id)
|
||||
made = (await client.get(f"/api/ledger/drafts/{body['draft_id']}")).json()
|
||||
assert made["status"] == "ready" and made["blocks"] == 2 and made["metric_count"] == 2
|
||||
assert [group["sql"] for group in made["groups"]] == [first["sql"], second["sql"]]
|
||||
assert [m["id"] for m in made["metrics"]] == ["m1", "m2"]
|
||||
assert made["progress"] == {"round": 1, "covered": 2, "uncovered": 0}
|
||||
assert len(model.calls) == 1
|
||||
|
||||
|
||||
async def test_one_bad_group_is_repaired_without_failing_the_others(client, documents, datasource, monkeypatch):
|
||||
"""一组 SQL 不合格只重写这一组;修好后照常落图,好的那组不受影响。"""
|
||||
doc = await documents("dd-groups-bad")
|
||||
source_id = await datasource()
|
||||
good = json.loads(plan_reply(f"{doc}-2"))
|
||||
bad = json.loads(plan_reply(f"{doc}-2", sql="SELECT * FROM secrets"))
|
||||
bad["metrics"] = [{**bad["metrics"][0], "id": "m1", "name": "已支付订单数",
|
||||
"expression": "cell(nodes.fetch, 0, 'orders')"}]
|
||||
fixed = {**bad, "sql": ("SELECT COUNT(*) AS orders FROM orders WHERE status = 'paid' "
|
||||
"AND store = '{{ input.store }}'")}
|
||||
model = fake_model(monkeypatch, groups_reply(good, bad), groups_reply(fixed))
|
||||
|
||||
body = await extract(client, doc, source_id)
|
||||
made = (await client.get(f"/api/ledger/drafts/{body['draft_id']}")).json()
|
||||
assert made["status"] == "ready" and made["blocks"] == 2 and made["metric_count"] == 2
|
||||
assert made["workflow_id"] and len(model.calls) == 2
|
||||
assert [group["sql"] for group in made["groups"]] == [good["sql"], fixed["sql"]]
|
||||
assert not any("secrets" in item["reason"] for item in made["dropped"])
|
||||
|
||||
|
||||
async def test_a_bad_group_that_still_cannot_be_repaired_is_dropped(client, documents, datasource, monkeypatch):
|
||||
"""重写也没修好:只丢这一组,问题照实写下来,不把整份草稿判死。"""
|
||||
doc = await documents("dd-groups-unfixable")
|
||||
source_id = await datasource()
|
||||
good = json.loads(plan_reply(f"{doc}-2"))
|
||||
bad = json.loads(plan_reply(f"{doc}-2", sql="SELECT * FROM secrets"))
|
||||
bad["metrics"] = [{**bad["metrics"][0], "id": "m1", "name": "已支付订单数",
|
||||
"expression": "cell(nodes.fetch, 0, 'orders')"}]
|
||||
model = fake_model(monkeypatch, groups_reply(good, bad))
|
||||
|
||||
body = await extract(client, doc, source_id)
|
||||
made = (await client.get(f"/api/ledger/drafts/{body['draft_id']}")).json()
|
||||
assert made["status"] == "ready" and made["blocks"] == 1 and made["metric_count"] == 1
|
||||
assert len(model.calls) == 2
|
||||
assert any("secrets" in item["reason"] for item in made["dropped"])
|
||||
|
||||
|
||||
async def test_a_big_document_is_drafted_in_several_rounds_onto_one_multi_fetch_graph(
|
||||
client, documents, datasource, monkeypatch):
|
||||
"""第一轮交一组、uncovered 还剩一条:续跑第二轮,收成一张两个取数节点、卡上两条指标的图。"""
|
||||
@@ -531,6 +737,13 @@ async def test_a_big_document_is_drafted_in_several_rounds_onto_one_multi_fetch_
|
||||
assert made["status"] == "ready"
|
||||
assert [m["name"] for m in made["metrics"]] == ["已支付退款率", "已支付订单数"]
|
||||
assert made["blocks"] == 2
|
||||
# groups 逐组给出 SQL、入口字段和指标 id:页面按组画「这条 SQL 出哪几个指标」,不必从拼接的 sql 里猜边界
|
||||
assert [set(group) for group in made["groups"]] == [{"sql", "entry_fields", "metric_ids"}] * 2
|
||||
assert [group["sql"] for group in made["groups"]] == [first["sql"], second["sql"]]
|
||||
assert [group["metric_ids"] for group in made["groups"]] == [["m1"], ["m2"]]
|
||||
# 各组都从 m1 起草,拼图前重排:payload 的指标 id 和 groups 指同一组指标,页面按 id 配试跑值才不会张冠李戴
|
||||
assert [m["id"] for m in made["metrics"]] == ["m1", "m2"]
|
||||
assert [f["name"] for f in made["groups"][0]["entry_fields"]] == ["store"]
|
||||
assert made["progress"] == {"round": 2, "covered": 2, "uncovered": 0}
|
||||
assert made["truncated"] is False and made["doc_truncated"] is False and made["catalog_truncated"] is False
|
||||
# 第二轮提示词写明上一轮起草过什么:模型据此不再重复起草
|
||||
@@ -641,6 +854,8 @@ async def test_a_real_trial_run_reads_values_from_both_fetch_nodes(client, docum
|
||||
trial = (await client.get(f"/api/ledger/drafts/{draft_id}/trial")).json()
|
||||
assert trial["status"] == "succeeded" and trial["error"] == ""
|
||||
assert {m["id"]: float(m["value"]) for m in trial["metrics"]} == {"m1": 156.5, "m2": 4.0}
|
||||
assert [query["node_id"] for query in trial["queries"]] == ["fetch_1", "fetch_2"]
|
||||
assert [query["columns"] for query in trial["queries"]] == [["gmv"], ["orders"]]
|
||||
async with SessionLocal() as session:
|
||||
run = await session.get(Run, run_id)
|
||||
tools = [n["id"] for n in (run.graph or {}).get("nodes") or [] if n.get("type") == "tool"]
|
||||
@@ -712,25 +927,26 @@ async def test_running_the_trial_with_inputs_typed_on_the_page(client, documents
|
||||
"AND paid_at >= '{{ input.start }}'")
|
||||
fake_model(monkeypatch, json.dumps(reply, ensure_ascii=False))
|
||||
|
||||
async def waiting(*, draft_id, workflow_id, graph, values, started_by, entry_fields=()):
|
||||
return {"run_id": "", "status": "needs_input", "error": "缺少必填输入:start"}
|
||||
async def waiting(*, draft_id, workflow_id, graph, values, started_by, entry_fields=(), version=1):
|
||||
return {**api_ledger._trial_binding(version, graph, values),
|
||||
"run_id": "", "status": "needs_input", "error": "缺少必填输入:start"}
|
||||
|
||||
monkeypatch.setattr(api_ledger, "_run_trial", waiting)
|
||||
body = await extract(client, doc, source_id)
|
||||
made = (await client.get(f"/api/ledger/drafts/{body['draft_id']}")).json()
|
||||
assert made["trial"]["status"] == "needs_input"
|
||||
trial = (await client.get(f"/api/ledger/drafts/{body['draft_id']}/trial")).json()
|
||||
assert trial["status"] == "needs_input" and trial["metrics"] == []
|
||||
assert trial["status"] == "needs_input" and trial["metrics"] == [] and trial["queries"] == []
|
||||
|
||||
missing = await client.post(f"/api/ledger/drafts/{body['draft_id']}/trial", json={"inputs": {}})
|
||||
assert missing.status_code == 422
|
||||
|
||||
seen: dict[str, object] = {}
|
||||
|
||||
async def started(*, draft_id, workflow_id, graph, values, started_by, entry_fields=()):
|
||||
async def started(*, draft_id, workflow_id, graph, values, started_by, entry_fields=(), version=1):
|
||||
seen.clear()
|
||||
seen.update(values)
|
||||
return {"run_id": "", "status": "running", "error": ""}
|
||||
return {**api_ledger._trial_binding(version, graph, values), "run_id": "", "status": "running", "error": ""}
|
||||
|
||||
monkeypatch.setattr(api_ledger, "_run_trial", started)
|
||||
res = await client.post(f"/api/ledger/drafts/{body['draft_id']}/trial", json={"inputs": {"start": "2026-01-01"}})
|
||||
@@ -746,13 +962,13 @@ async def test_extract_hands_the_fixed_filters_to_every_round(client, documents,
|
||||
source_id = await datasource()
|
||||
fake_model(monkeypatch, plan_reply(f"{doc}-2"))
|
||||
seen: dict[str, object] = {}
|
||||
real_plan = draftplan.plan
|
||||
real_plan = draftplan.plan_groups
|
||||
|
||||
async def spy(session, **kwargs):
|
||||
seen.update(kwargs)
|
||||
return await real_plan(session, **kwargs)
|
||||
|
||||
monkeypatch.setattr(draftplan, "plan", spy)
|
||||
monkeypatch.setattr(draftplan, "plan_groups", spy)
|
||||
body = await client.post("/api/ledger/drafts/extract", json={
|
||||
"documents": [doc], "datasource_id": source_id, "fixed_filters": "line_code = 'FOAM01'"})
|
||||
made = (await client.get(f"/api/ledger/drafts/{body.json()['draft_id']}")).json()
|
||||
|
||||
@@ -135,6 +135,33 @@ def test_metric_ids_are_unique_and_entry_fields_are_normalized():
|
||||
assert plan["entry_fields"][0]["required"] is True
|
||||
|
||||
|
||||
def test_document_drafting_fields_are_kept_and_normalized():
|
||||
"""文档起草里的粒度、时间字段、分子分母、维度、过滤条件不能在校验时丢掉:草稿页要逐行展示。"""
|
||||
metric = {**METRIC, "grain": " 日 ", "time_field": " 支付时间 ", "numerator": "退款订单数",
|
||||
"denominator": "已支付订单数", "dimensions": ["门店", "", 3, " 区域 "],
|
||||
"filters": ["只算已支付", " ", None]}
|
||||
plan, problems = draftplan.normalize_plan(raw(metrics=[metric]), CHUNKS, allowed_tables=TABLES)
|
||||
got = plan["metrics"][0]
|
||||
assert got["grain"] == "日" and got["time_field"] == "支付时间"
|
||||
assert got["numerator"] == "退款订单数" and got["denominator"] == "已支付订单数"
|
||||
assert got["dimensions"] == ["门店", "区域"] and got["filters"] == ["只算已支付"]
|
||||
# METRIC_FIELDS 要列全:校验后的指标带着这些业务字段(前端按它画)
|
||||
assert set(draftplan.METRIC_FIELDS) >= {"grain", "time_field", "numerator", "denominator",
|
||||
"dimensions", "filters"}
|
||||
assert set(got) >= set(draftplan.METRIC_FIELDS)
|
||||
assert not [p for p in problems if p["name"] == "销售额"]
|
||||
|
||||
|
||||
def test_empty_document_drafting_fields_do_not_leave_empty_string_keys():
|
||||
"""没写的业务字段不留空字符串:没有内容就没有这个键(dimensions、filters 里的空项也丢掉)。"""
|
||||
metric = {**METRIC, "grain": " ", "time_field": "", "numerator": None, "denominator": [],
|
||||
"dimensions": ["", " "], "filters": []}
|
||||
plan, _problems = draftplan.normalize_plan(raw(metrics=[metric]), CHUNKS, allowed_tables=TABLES)
|
||||
got = plan["metrics"][0]
|
||||
assert not [key for key in ("grain", "time_field", "numerator", "denominator", "dimensions", "filters")
|
||||
if key in got]
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# build_graph
|
||||
# ==========================================================================
|
||||
|
||||
@@ -40,6 +40,15 @@ async def test_trial_runs_the_pinned_caliber_and_returns_its_metrics(client):
|
||||
assert [m["id"] for m in body["metrics"]] == ["gmv"]
|
||||
assert float(body["metrics"][0]["value"]) == 170.5
|
||||
assert body["metrics"][0]["unit"] == "元"
|
||||
# 试跑结果集:取数节点交回的查询快照,给列和行、不给 SQL
|
||||
assert [query["node_id"] for query in body["queries"]] == ["fetch"]
|
||||
query = body["queries"][0]
|
||||
assert set(query) == {"node_id", "tool", "source", "columns", "rows", "total_rows",
|
||||
"truncated", "masked", "source_truncated", "preview_truncated"}
|
||||
assert query["tool"] == S.tool() and query["source"] == S.CURRENT["source"]
|
||||
assert query["columns"] == ["gmv"] and query["rows"] == [[170.5]]
|
||||
assert query["total_rows"] == 1 and query["truncated"] is False and query["masked"] == []
|
||||
assert "sql" not in query
|
||||
async with SessionLocal() as session:
|
||||
run = await session.get(Run, body["run_id"])
|
||||
assert run is not None and run.origin_kind == "caliber_trial" and run.run_class == "exploratory"
|
||||
@@ -55,3 +64,33 @@ async def test_trial_refuses_missing_required_inputs_and_unknown_fields(client):
|
||||
unknown = await client.post(f"/api/ledger/calibers/{wf}/1/card/trial",
|
||||
json={"inputs": {"start": "2026-09-01", "end": "2026-09-08", "nope": 1}})
|
||||
assert unknown.status_code == 422 and "nope" in unknown.text
|
||||
|
||||
|
||||
async def test_trial_queries_mask_the_configured_columns_and_never_return_sql(client):
|
||||
"""试跑结果集把数据源设了遮罩的列换成「已遮罩」(改了名字的结果列也认);响应里不出现 SQL,SQL 里可能写着遮罩值。"""
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.db.models import DataSource
|
||||
|
||||
async with SessionLocal() as session:
|
||||
source = (await session.execute(select(DataSource).where(
|
||||
DataSource.name == S.CURRENT["source"]))).scalar_one()
|
||||
source.options = {"mask_columns": ["phone"]}
|
||||
await session.commit()
|
||||
wf = await S.publish(graph=S.caliber_graph(sql="SELECT phone AS tel FROM orders ORDER BY id LIMIT 3",
|
||||
expression="cell(nodes.fetch, 0, 'tel')"))
|
||||
res = await client.post(f"/api/ledger/calibers/{wf}/1/card/trial",
|
||||
json={"inputs": {"start": "2026-09-01", "end": "2026-09-08"}})
|
||||
assert res.status_code == 200, res.text
|
||||
body = res.json()
|
||||
assert body["status"] == "succeeded", body
|
||||
query = body["queries"][0]
|
||||
assert query["columns"] == ["tel"] and query["masked"] == ["tel"]
|
||||
assert query["rows"] == [["已遮罩"]] * 3
|
||||
assert "value" not in body["metrics"][0]
|
||||
assert body["metrics"][0]["rendered"] == "已遮罩"
|
||||
assert body["metrics"][0]["masked"] is True
|
||||
assert body["validation"]["status"] == "unknown"
|
||||
assert body.get("output") is None
|
||||
assert query["total_rows"] == 3 and query["truncated"] is False
|
||||
assert "sql" not in query
|
||||
|
||||
@@ -4,6 +4,8 @@
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from app.engine.schema import GraphSpec, validate_graph
|
||||
from tests import ledger_samples as S
|
||||
|
||||
@@ -208,3 +210,21 @@ def test_an_assessment_metric_is_labelled_assessed():
|
||||
assert label["kind"] == "assessed" and label["queries"] == ["q1"]
|
||||
assert label["assessment"]["level"] == "B" and label["definition_hash"] == "abc"
|
||||
assert "19900000501" not in str(label)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("raw", ["nan", "inf", "-inf"])
|
||||
async def test_nonfinite_assessment_input_cannot_be_clamped_into_a_normal_score(raw):
|
||||
from app.engine.runner import run_manager
|
||||
|
||||
await run_manager.setup()
|
||||
try:
|
||||
graph = S.chain(S.node("start", "input"), S.node("card", "metrics", caliber="评估", metrics=[{
|
||||
"id": "quality", "kind": "assessment", "assessment": {
|
||||
"dimensions": [{"name": "得分", "input": "input.raw", "weight": 1}],
|
||||
"levels": [{"name": "合格", "min": 60}, {"name": "不合格", "min": 0}],
|
||||
}}]), S.node("out", "output", fields=[{"name": "v", "value": "{{ nodes.card.text }}"}]))
|
||||
started = await run_manager.start(graph=graph, input_payload={"raw": raw})
|
||||
row = await S.wait(started.id)
|
||||
assert row.status == "failed" and "输入必须是有限数值" in row.error
|
||||
finally:
|
||||
await run_manager.shutdown()
|
||||
|
||||
@@ -86,6 +86,11 @@ def without_new_keys(card: dict) -> dict:
|
||||
"""去掉这个功能之前没有的两个键:value_artifact 和 definition_hash(口径的内容指纹)。"""
|
||||
out = without_value_artifacts(card)
|
||||
out.pop("definition_hash", None)
|
||||
out.pop("semantic_hash", None)
|
||||
out.pop("validation_version", None)
|
||||
for metric in out["metrics"]:
|
||||
metric.pop("quality", None)
|
||||
metric.pop("contract_findings", None)
|
||||
return out
|
||||
|
||||
|
||||
|
||||
@@ -7,7 +7,9 @@ selftest 是部署脚本用的:真起一台 VM,起得来才把 /dev/kvm 交
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from types import ModuleType
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -22,6 +24,8 @@ from app.sandbox.local_sandbox import LocalSandbox
|
||||
def linux(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path:
|
||||
"""在 Linux 上、运行时已装好、镜像已缓存;返回假的 /dev/kvm 路径,默认不存在。"""
|
||||
kvm = tmp_path / "kvm"
|
||||
# 此组模拟运行时已经就绪,SDK 是否安装不应影响 KVM 分支的测试。
|
||||
monkeypatch.setitem(sys.modules, "microsandbox", ModuleType("microsandbox"))
|
||||
monkeypatch.setattr(mv, "_on_linux", lambda: True)
|
||||
monkeypatch.setattr(mv, "_KVM_DEVICE", kvm)
|
||||
monkeypatch.setattr(mv.MicroVMSandbox, "_runtime_installed", staticmethod(lambda: True))
|
||||
@@ -35,6 +39,12 @@ def test_no_kvm_device_means_unavailable(linux: Path) -> None:
|
||||
assert "/dev/kvm" in reason and "挂进容器" in reason
|
||||
|
||||
|
||||
def test_missing_sdk_returns_a_result_without_import_crashing(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setitem(sys.modules, "microsandbox", None)
|
||||
result = asyncio.run(mv.MicroVMSandbox().run("print(1)"))
|
||||
assert not result.ok and "未安装 microsandbox" in result.error
|
||||
|
||||
|
||||
def test_kvm_without_permission_means_unavailable(linux: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
linux.touch()
|
||||
monkeypatch.setattr(mv.os, "access", lambda path, mode: False)
|
||||
|
||||
@@ -145,7 +145,7 @@ SHAPES: dict[str, list[tuple[str, object]]] = {
|
||||
"LocateProblem": [("code", REQ), ("detail", "")],
|
||||
"SumEqRule": [("table", REQ), ("total", REQ), ("parts", REQ), ("types", REQ)],
|
||||
"RelatedCheckPlan": [("check", REQ), ("rule", REQ)],
|
||||
"CompileFacts": [("tables", REQ), ("result_rows", REQ), ("yields", REQ)],
|
||||
"CompileFacts": [("tables", REQ), ("result_rows", REQ), ("yields", REQ), ("selects", 1)],
|
||||
}
|
||||
RESPONSE = ("ReportRef", "CellRef", "Reason", "Alert", "TableRef", "PeriodView", "AcceptanceView", "StateNote",
|
||||
"PartView", "VersionView", "FromCell", "YearSource", "CanonicalView", "Recheck", "CellSource",
|
||||
|
||||
@@ -351,14 +351,15 @@ async def test_rehash_is_false_without_a_hash_or_a_file(snapshot):
|
||||
])
|
||||
async def test_compile_facts_reports_tables_and_result_rows(snapshot, sql, tables, rows, yields):
|
||||
facts = await provenance_db.compile_facts(await _view(snapshot), sql)
|
||||
assert facts == CompileFacts(tables=tables, result_rows=rows, yields=yields)
|
||||
selects = 3 if 'UNION ALL' in sql else 2 if ' IN (SELECT' in sql else 1
|
||||
assert facts == CompileFacts(tables=tables, result_rows=rows, yields=yields, selects=selects)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("op", ["UNION", "INTERSECT", "EXCEPT"])
|
||||
async def test_compile_facts_sees_yield_in_set_operations(snapshot, op):
|
||||
async def test_compile_facts_sees_multiple_selects_in_set_operations(snapshot, op):
|
||||
facts = await provenance_db.compile_facts(
|
||||
await _view(snapshot), f'SELECT "日期","全日客流" FROM "日客流" {op} SELECT "日期","分区甲" FROM "日客流"')
|
||||
assert facts.yields >= 1
|
||||
assert facts.selects >= 2 # 不依赖 SQLite 是否将复合查询编译成 Yield 协程
|
||||
|
||||
|
||||
async def test_compile_facts_runs_in_a_thread_on_an_immutable_read_only_connection(snapshot, monkeypatch):
|
||||
|
||||
@@ -495,7 +495,7 @@ def test_analysis_signals_are_registered_as_gaps_that_only_need_a_note():
|
||||
("report_analysis_claims", rv.DEGRADED), ("analysis_refused", rv.DEGRADED),
|
||||
("metric_comparability_skipped", rv.DEGRADED)]
|
||||
# NOTE_ONLY 另有一种不是日志代码的:分析调用写错了一直没改对(analysis_call_failed,见 tests/test_review_analysis.py)
|
||||
assert {s.kind for s in signals} == rv.NOTE_ONLY - {rv.ANALYSIS_CALL_FAILED}
|
||||
assert {s.kind for s in signals} == rv.NOTE_ONLY - {rv.ANALYSIS_CALL_FAILED, "investigation_incomplete"}
|
||||
assert not rv.should_retry(signals) and not rv.NOTE_ONLY & (rv.RETRYABLE | rv.REBUILD)
|
||||
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import re
|
||||
import sys
|
||||
import types
|
||||
|
||||
import pytest
|
||||
@@ -72,6 +73,10 @@ def test_a_process_that_cannot_spawn_has_no_class_name(backend, monkeypatch) ->
|
||||
|
||||
|
||||
def _microvm(monkeypatch) -> MicroVMSandbox:
|
||||
sdk = types.ModuleType("microsandbox")
|
||||
sdk.ExecTimeoutError = TimeoutError
|
||||
monkeypatch.setitem(sys.modules, "microsandbox", sdk)
|
||||
|
||||
async def nothing(*args, **kwargs): # noqa: ANN002, ANN003, ANN202
|
||||
return None
|
||||
|
||||
|
||||
Reference in New Issue
Block a user