复盘修正:试运行不计入统计、评估标签遮罩、规则/评估卡可组合

This commit is contained in:
ThearchyHelios
2026-11-03 21:23:08 +08:00
parent 4bf9e56979
commit 753c35918d
27 changed files with 240 additions and 86 deletions
+3 -1
View File
@@ -330,6 +330,8 @@ async def trial_caliber(workflow_id: str, version: int, node_id: str, payload: T
pinned = await load_pinned(session, workflow_id, int(version), node_id)
except NotDerivable as e:
raise HTTPException(409, e.message) from None
if not pinned.published or pinned.retired:
raise HTTPException(409, "这一版没有发布或已退役,不能试运行")
fields = [dict(f) for f in pinned.entry_fields]
known = {str(f.get("name")) for f in fields}
unknown = sorted(str(k) for k in payload.inputs if str(k) not in known)
@@ -343,7 +345,7 @@ async def trial_caliber(workflow_id: str, version: int, node_id: str, payload: T
graph = caliber_probe(pinned).graph
principal = current_principal()
run = await run_manager.start(
graph=graph, input_payload=values, run_class="exploratory", origin_kind="trial",
graph=graph, input_payload=values, run_class="exploratory", origin_kind="caliber_trial",
origin={"kind": "caliber_trial", "workflow_id": workflow_id, "version": int(version),
"node_id": node_id, "card": pinned.node_id},
started_by=principal.username)
+1 -1
View File
@@ -15,7 +15,7 @@ ROUND_IN_SOURCE = "caliber.round_in_source"
ZERO_DEFAULT = "caliber.zero_default"
ROUND_MESSAGE = (
"取数 SQL 里写了 ROUND(...):取整交给口径卡的小数位和展示;取数 SQL 里写 ROUND 会让未取整的查询"
"取数 SQL 里写了取整函数:取整交给口径卡的小数位和展示;取数 SQL 里写取整会让未取整的查询"
"永远认不成等价(口径账本只认规范化后完全相同的结果)"
)
ZERO_MESSAGE = (
+4 -1
View File
@@ -108,8 +108,11 @@ def run_fixtures(rule: Mapping[str, Any], table: str, fixtures: Iterable[Mapping
columns: dict[str, str] = {}
for item in items:
for row in item.get("rows") or []:
if not isinstance(row, Mapping):
raise RuleInvalid("fixture 的 rows 需要是对象")
for name, value in row.items():
columns.setdefault(str(name), "INTEGER" if isinstance(value, int) and not isinstance(value, bool)
key = _ident(name, "fixture 列名")
columns.setdefault(key, "INTEGER" if isinstance(value, int) and not isinstance(value, bool)
else "REAL" if isinstance(value, float) else "TEXT")
if not columns:
raise RuleInvalid("fixture 需要至少一行 rows")
+3
View File
@@ -512,6 +512,9 @@ async def _plan_metric(state: GraphState, metric: dict[str, Any], options: _Opti
"status": "pending", "original": {"value": metric.get("value"), "rendered": metric.get("rendered")},
"blind": {"value": None, "rendered": None}, "delta_pct": None, "inputs": [], "differing": [],
}
if metric.get("rule") is not None or metric.get("assessment") is not None or metric.get("external") is not None:
row.update(status=UNCHECKED, reason="这个指标不是按表达式算的,不参与双盲复算")
return row, {}
tops = _tops(metric)
# masks 是每份用到的快照要遮的列(cell() 的、按位置取的格落在的),queries 只是 cell() 用到的那几份:重算时容器
# 输入按内容对快照只对这几份。按位置取的格落在的快照算遮罩只为认这一格,不当成指标用到的结果——放进去的话,同一个
+39 -12
View File
@@ -120,18 +120,31 @@ async def run_metrics(state: GraphState, ctx: NodeContext) -> dict[str, Any]:
value: Any = None
details: list[dict[str, Any]] = []
level: str | None = None
out_of_range: list[str] = []
if not missing:
total = sum(float(d.get("weight", 1)) for d in dimensions)
points = 0.0
bad: str | None = None
for d in dimensions:
low, high = float(d.get("min", 0)), float(d.get("max", 100))
raw = float(values_by_name[str(d["name"])])
try:
raw = float(values_by_name[str(d["name"])])
except (TypeError, ValueError):
bad = f"维度「{d['name']}」的输入不是数({values_by_name[str(d['name'])]!r})"
break
normalized = max(0.0, min(100.0, (raw - low) / (high - low) * 100))
weight = float(d.get("weight", 1))
points += normalized * weight
details.append({"name": d["name"], "value": raw, "normalized": normalized, "weight": weight})
value = round_metric(points / total, display["decimals"])
level = next((str(l["name"]) for l in assessment["levels"] if value >= float(l["min"])), None)
if raw < low or raw > high:
out_of_range.append(str(d["name"]))
details.append({"name": d["name"], "value": raw, "normalized": normalized, "weight": weight,
"out_of_range": raw < low or raw > high})
if bad:
errors.append(f"指标「{metric_id}」:{bad}")
continue
raw_score = points / total
value = round_metric(raw_score, display["decimals"])
level = next((str(l["name"]) for l in assessment["levels"] if raw_score >= float(l["min"])), None)
inputs = [_input_of(str(d["input"]), values_by_name[str(d["name"])], state, ctx, tctx)
for d in dimensions]
metrics.append({
@@ -145,10 +158,14 @@ async def run_metrics(state: GraphState, ctx: NodeContext) -> dict[str, Any]:
"rendered": _rendered(value, definition, display),
"inputs": inputs,
"substituted": "加权评分",
"recompute_ok": True,
"recompute_ok": None,
"recompute_reason": "这个指标不按表达式复算",
"status": "ok" if value is not None else "missing_input",
"assessment": {"score": value, "level": level, "dimensions": details,
"levels": assessment["levels"], **({"missing": missing} if missing else {})},
"levels": assessment["levels"],
**({"missing": missing} if missing else {}),
**({"out_of_range": out_of_range} if out_of_range else {}),
**({"unlevelled": True} if value is not None and level is None else {})},
})
continue
if str(definition.get("kind") or "formula") == "rule":
@@ -179,7 +196,8 @@ async def run_metrics(state: GraphState, ctx: NodeContext) -> dict[str, Any]:
"rendered": _rendered(value, definition, display),
"inputs": inputs,
"substituted": _rule_text(rule),
"recompute_ok": True,
"recompute_ok": None,
"recompute_reason": "这个指标不按表达式复算",
"status": "ok" if value is not None else "missing_input",
"rule": rule,
})
@@ -202,7 +220,8 @@ async def run_metrics(state: GraphState, ctx: NodeContext) -> dict[str, Any]:
"rendered": _rendered(value, definition, display),
"inputs": [],
"substituted": str(value),
"recompute_ok": True,
"recompute_ok": None,
"recompute_reason": "这个指标不按表达式复算",
"status": "ok",
"external": external,
})
@@ -605,10 +624,18 @@ def _rule_matches(values: dict[str, Any], rule: dict[str, Any]) -> bool:
return False
continue
b = cond.get("value")
try:
ok = {"eq": a == b, "ne": a != b, "gt": a > b, "gte": a >= b, "lt": a < b, "lte": a <= b}.get(op, False)
except TypeError:
ok = False
# 类型对不上(字符串和数)按条件不成立,不静默走 ne;大小比较只对数
if op in ("eq", "ne"):
comparable = (isinstance(a, (int, float)) and not isinstance(a, bool)
and isinstance(b, (int, float)) and not isinstance(b, bool)) or type(a) is type(b)
if not comparable:
return False
equals = a == b
ok = equals if op == "eq" else not equals
else:
if not all(isinstance(v, (int, float)) and not isinstance(v, bool) for v in (a, b)):
return False
ok = {"gt": a > b, "gte": a >= b, "lt": a < b, "lte": a <= b}.get(op, False)
if not ok:
return False
return True
+2 -1
View File
@@ -29,6 +29,7 @@ ORIGIN_KINDS = (
"ledger_issue", # 问数据的答案按已治理口径正式出具(正式运行)
"reconcile", # 口径对账:这次的取数和口径在同一份数据上各算一次(探索运行)
"recall", # 口径召回:新旧两版口径在同一份数据上各算一次(探索运行)
"caliber_trial", # 口径目录里的试跑(探索运行,不是问数回答,也不进统计/召回)
)
#: 客户端在 POST /api/runs 里能声明的来源
@@ -38,7 +39,7 @@ CLIENT_ORIGINS = ("chat", "canvas", "launcher", "api")
#: 不是有人在用。它们是完整的运行(证据、封存都在、点得开),但不进使用统计——探索问题聚类、工具白名单衰减、数据目录的
#: 使用次数和「被哪些运行用过」、审阅队列的使用权重、以后的账本统计——否则一跑评测,治理信号和目录的使用热度就失真了。
#: 记录页默认也不列(筛选里能选,列表照实写有几次没列出)。ledger_issue 是替用户正式出具,照常计入
SYSTEM_ORIGINS = ("eval", "reconcile", "recall")
SYSTEM_ORIGINS = ("eval", "reconcile", "recall", "caliber_trial")
def counted_runs() -> Any:
+17 -6
View File
@@ -1086,8 +1086,8 @@ def _check_challenge(node: GraphNode, spec: GraphSpec, result: ValidationResult)
field="model", code="challenge.same_model")
#: 指标定义的类型。现在只有公式(表达式)这一种在运行时能算;规则、评估、外部常量还没实现,
#: 先在校验里挡住,免得写进图里以后被当成能算的指标
#: 指标定义的类型:公式(表达式)、外部常量、单元格条件规则、评估评分。四种都在运行时能算;
#: 不认识的类型仍在校验里挡住,免得被当成能算的指标
METRIC_KINDS = ("formula", "external", "rule", "assessment")
#: 定义卡参数的标量类型(和入口表单的字段类型同一套子集)
METRIC_PARAM_TYPES = ("string", "number", "integer", "boolean")
@@ -1110,8 +1110,8 @@ def metric_definition_issues(node: GraphNode, metric: Any, index: int) -> list[V
add(f"指标定义缺少 ID:{field}", code="metrics.id_missing")
kind = str(metric.get("kind") or "formula")
if kind not in METRIC_KINDS:
add(f"指标「{metric.get('id') or index + 1}」的类型「{kind}」还没有实现:现在支持「公式」(formula)和"
"「外部常量」(external)。规则、评估请先在定义卡里写成说明,等对应的执行方式上线后再发布",
add(f"指标「{metric.get('id') or index + 1}」的类型「{kind}」不认识:现在支持「公式」(formula)、"
"「外部常量」(external)、「规则」(rule)、「评估」(assessment)",
f"{field}.kind", code="metrics.kind_unsupported")
if kind == "external":
external = metric.get("external")
@@ -1151,6 +1151,10 @@ def metric_definition_issues(node: GraphNode, metric: Any, index: int) -> list[V
if not isinstance(name, str) or not name.strip() or not isinstance(path, str) \
or not path.strip():
add("规则的每个输入都要有名字和取值路径", f"{field}.rule.inputs", code="metrics.rule_invalid")
elif not any(token in path for token in ("cell(", "nodes.", "vars.", "input.")):
add(f"规则输入「{name}」的取值路径要写成 cell(nodes.…)、vars.… 或 input.…;"
"表列名是规则 DSL 编译接口的写法,不能直接写进口径卡",
f"{field}.rule.inputs.{name}", code="metrics.rule_invalid")
conditions = rule.get("all")
if not isinstance(conditions, list) or not conditions:
add("规则需要 all:至少一条条件", f"{field}.rule.all", code="metrics.rule_invalid")
@@ -1163,8 +1167,8 @@ def metric_definition_issues(node: GraphNode, metric: Any, index: int) -> list[V
if cond.get("op") not in RULE_OPS:
add(f"条件运算符只能是 {'、'.join(RULE_OPS)}", f"{key}.op", code="metrics.rule_invalid")
if cond.get("op") in ("in", "not_in"):
if not isinstance(cond.get("values"), list):
add("in / not_in 需要 values 列表", f"{key}.values", code="metrics.rule_invalid")
if not isinstance(cond.get("values"), list) or not cond["values"]:
add("in / not_in 需要非空的 values 列表", f"{key}.values", code="metrics.rule_invalid")
elif "value" not in cond:
add("条件需要 value", f"{key}.value", code="metrics.rule_invalid")
for key in ("then", "else"):
@@ -1194,6 +1198,9 @@ def metric_definition_issues(node: GraphNode, metric: Any, index: int) -> list[V
names.add(name)
if not isinstance(dimension.get("input"), str) or not dimension["input"].strip():
add("评估维度需要 input(取值路径)", f"{key}.input", code="metrics.assessment_invalid")
elif not any(token in dimension["input"] for token in ("cell(", "nodes.", "vars.", "input.")):
add("评估维度的 input 要写成 cell(nodes.…)、vars.… 或 input.…",
f"{key}.input", code="metrics.assessment_invalid")
weight = dimension.get("weight", 1)
if isinstance(weight, bool) or not isinstance(weight, (int, float)) or weight <= 0:
add("评估维度的 weight 需要是正数", f"{key}.weight", code="metrics.assessment_invalid")
@@ -1220,6 +1227,10 @@ def metric_definition_issues(node: GraphNode, metric: Any, index: int) -> list[V
add("评估等级要按 min 从高到低排列", f"{key}.min", code="metrics.assessment_invalid")
else:
previous = float(minimum)
if levels and isinstance(levels[-1], dict) and isinstance(levels[-1].get("min"), (int, float)) \
and not isinstance(levels[-1]["min"], bool) and levels[-1]["min"] > 0:
add("最后一个评估等级的 min 要不大于 0,否则低分没有等级可落",
f"{field}.assessment.levels[-1].min", code="metrics.assessment_invalid")
elif not metric.get("expression"):
add(f"指标定义缺少表达式:{metric.get('id') or '(空)'}", code="metrics.expression_missing")
+15 -2
View File
@@ -77,6 +77,17 @@ async def _describe(session: AsyncSession, workflow: Workflow, row: Any, node_id
cfg = card["config"]
level = str(row.level or "")
_, reason = card_inputs(graph, node_id)
if reason is None:
from app.ledger.derive import template_problem
for nid, node in nodes.items():
tool = str((node.get("config") or {}).get("tool") or "")
sql = ((node.get("config") or {}).get("args") or {}).get("sql") \
if isinstance((node.get("config") or {}).get("args"), dict) else None
if node.get("type") == "tool" and tool.startswith(QUERY_PREFIX) and isinstance(sql, str) and sql.strip():
if problem := template_problem(str(nid), node.get("config") or {}):
reason = problem
break
key = (workflow.id, int(row.version))
if key not in side_cache:
side_cache[key] = await graph_side_effect(session, graph, level=level or None)
@@ -86,7 +97,9 @@ async def _describe(session: AsyncSession, workflow: Workflow, row: Any, node_id
if level != "governed":
reasons.append("这一版是已发布级别,只支持一键出具")
if side_effect is not None:
reasons.append(f"这一版{side_effect},不能替人重跑")
reasons.append(f"这一版{side_effect},不能替人再次执行")
if not sources:
reasons.append("这一版没有取数节点,账本不能按口径证明等价")
return {
"workflow_id": workflow.id,
"workflow_name": workflow.name,
@@ -103,7 +116,7 @@ async def _describe(session: AsyncSession, workflow: Workflow, row: Any, node_id
"compose_reason": reason,
"dependencies": {"sources": sorted(sources), "tables": sorted(tables),
"catalog_versions": row.catalog_versions, "schema_versions": row.schema_versions},
"issuance": {"level": level, "auto_issue": level == "governed" and side_effect is None,
"issuance": {"level": level, "auto_issue": level == "governed" and side_effect is None and bool(sources),
"requires_independent_proof": True, "side_effects": side_effect, "reasons": reasons},
}
+26 -6
View File
@@ -1,8 +1,11 @@
"""逐个数标口径(docs/design/next-round/02-口径账本.md 做法 3):报告里每个引用了查询格、口径卡指标的数,来自哪一份口径。
标签四种(KINDS):
标签(KINDS):
- caliber:来自口径「X」vN。how 是怎么认的:independent「等价已证明」(这次 Agent 自己写的 SQL 被证明和口径的取数等价)、
composed「助手选用」(助手 use_caliber 按口径组合出来的取数,证明了和那一版一致:ledger/match 的 composed)。
- external:外部常量(口径卡上的 kind=external),带来源、生效日期、是否过期。
- derived:规则型指标(kind=rule),数由结构化规则从输入算出,不是某条查询与口径等价。
- assessed:评估型指标(kind=assessment),按评分协议算出的评估值,不是测量值。
- related:相关口径——同一个数据源、查了同样的表的已发布口径,但不等价;附差在哪(explain_sql_diff 的条目)。
- reasoning:这次的推理——没有相关的已发布口径。
- unknown:没比对、比对没做完(超时、解析失败)、快照读不出来。
@@ -309,13 +312,26 @@ def metric_label(card: Mapping[str, Any] | None, metric_id: Any, queries: Mappin
inputs = [i for i in metric.get("inputs") or [] if isinstance(i, dict)]
from_queries = [i for i in inputs if i.get("artifact") and i.get("via") in _SNAPSHOT_VIAS]
if metric.get("rule") is not None:
# 规则型指标:数由规则从输入算出,不是某条查询与口径等价;输入仍然逐格可追溯
return {"kind": "derived", "rule": metric.get("rule"),
# 规则型指标:数由规则从输入算出,不是某条查询与口径等价;输入仍然逐格可追溯。
# 标签里只给结构(输入名、条件条数、then/else),不给条件里的字面量——那可能是遮罩列的取值
rule = metric.get("rule") if isinstance(metric.get("rule"), dict) else {}
return {"kind": "derived",
"rule": {"inputs": sorted(str(k) for k in (rule.get("inputs") or {})),
"conditions": len(rule.get("all") or []),
"then": rule.get("then", 1), "else": rule.get("else", 0)},
"queries": [str(i["artifact"]) for i in from_queries],
**({"definition_hash": card.get("definition_hash")} if card.get("definition_hash") else {})}
if metric.get("assessment") is not None:
# 评估值:按评分协议从输入算出,不是测量值,也不参与自动出具
return {"kind": "assessed", "assessment": metric.get("assessment"),
# 评估值:按评分协议从输入算出,不是测量值,也不参与自动出具。标签里不带各维度的原始取值
# (可能是遮罩列的取值),只给得分、等级和维度名/权重
assessment = metric.get("assessment") if isinstance(metric.get("assessment"), dict) else {}
dimensions = [{"name": d.get("name"), "weight": d.get("weight")}
for d in assessment.get("dimensions") or [] if isinstance(d, dict)]
return {"kind": "assessed",
"assessment": {"score": assessment.get("score"), "level": assessment.get("level"),
"dimensions": dimensions,
"levels": [{"name": l.get("name"), "min": l.get("min")}
for l in assessment.get("levels") or [] if isinstance(l, dict)]},
"queries": [str(i["artifact"]) for i in from_queries],
**({"definition_hash": card.get("definition_hash")} if card.get("definition_hash") else {})}
if not from_queries:
@@ -445,7 +461,11 @@ def whole_of(labels: Mapping[str, Mapping[str, Any]], calibers: Mapping[str, Map
keys = {label.get("caliber") for label in labels.values() if label.get("kind") == "caliber"}
if counts["caliber"] != len(labels):
# 逐格判之后常见「一部分数来自口径、另几个只是相关」:照实说是不一致,不说成这次的推理
reason = ("有的数没有比对" if counts["unknown"] else "没有数来自口径" if not counts["caliber"]
reason = ("有的数没有比对" if counts["unknown"]
else "有的数是外部常量,不是测量值" if counts["external"]
else "有的数是规则派生的,不是测量值" if counts["derived"]
else "有的数是评估值,不是测量值" if counts["assessed"]
else "没有数来自口径" if not counts["caliber"]
else "有的数和相关口径不一致,不来自口径" if counts["related"] else "有的数是这次的推理")
return {"equivalent": False, "reason": reason, "counts": counts}
if len(keys) != 1:
+2 -1
View File
@@ -34,7 +34,8 @@ FORMAT = "agentlab.caliber_match/1"
#: 3:整条比不等价时按报告引用的列投影再比(ledger/match 的 _projected),候选记 projection;没投影的照旧
#: 4:按全部引用列投影仍没证明的查询,被引用的每一列逐格各比一次(queries[].cells、one_row),每个数按它那一格标;老工件
#: 没有 cells,照旧按整条查询读
ALGO = "4"
#: 5:候选记定义指纹(caliber_hash / definition_hashes),标签新增 external / derived / assessed;老工件没有这些键照旧读
ALGO = "5"
#: 触发方式(见模块说明)。certified(第 3 波):从问数据固化出的口径发布之后,对来源运行补比的(ledger/certify)
TRIGGERS = ("sealed", "backfill", "on_demand", "certified")
+47 -21
View File
@@ -106,33 +106,59 @@ def card_inputs(graph: dict[str, Any], card_id: str) -> tuple[list[dict[str, Any
return [], CHAINED_REASON
found: dict[str, dict[str, Any]] = {}
for metric in card["config"].get("metrics") or []:
try:
tree = parse_expression(str((metric or {}).get("expression") or ""))[0]
except ExpressionError:
paths = _metric_paths(metric)
if paths is None:
return [], NOT_TOOL_REASON
for path in leaf_refs(tree):
parts = cell_parts(path)
if parts is None:
return [], NOT_TOOL_REASON
chain, row, column = parts
node_id = chain[len("nodes."):] if chain.startswith("nodes.") else ""
tool = str((nodes.get(node_id) or {}).get("config", {}).get("tool") or "")
if (nodes.get(node_id) or {}).get("type") != "tool" or not tool.startswith(QUERY_PREFIX):
return [], NOT_TOOL_REASON
if row.strip() != "0":
return [], ROW_REASON
for path in paths:
try:
col = json.loads(column.replace("'", '"')) if column[:1] in ("'", '"') else None
except ValueError:
col = None
if not isinstance(col, str):
tree = parse_expression(path)[0]
except ExpressionError:
return [], NOT_TOOL_REASON
entry = found.setdefault(node_id, {"node": node_id, "tool": tool, "columns": []})
if col not in entry["columns"]:
entry["columns"].append(col)
for leaf in leaf_refs(tree):
parts = cell_parts(leaf)
if parts is None:
return [], NOT_TOOL_REASON
chain, row, column = parts
node_id = chain[len("nodes."):] if chain.startswith("nodes.") else ""
tool = str((nodes.get(node_id) or {}).get("config", {}).get("tool") or "")
if (nodes.get(node_id) or {}).get("type") != "tool" or not tool.startswith(QUERY_PREFIX):
return [], NOT_TOOL_REASON
if row.strip() != "0":
return [], ROW_REASON
try:
col = json.loads(column.replace("'", '"')) if column[:1] in ("'", '"') else None
except ValueError:
col = None
if not isinstance(col, str):
return [], NOT_TOOL_REASON
entry = found.setdefault(node_id, {"node": node_id, "tool": tool, "columns": []})
if col not in entry["columns"]:
entry["columns"].append(col)
return list(found.values()), None
def _metric_paths(metric: Any) -> list[str] | None:
"""一个指标读的取值路径:公式的表达式、规则各输入、评估各维度。外部常量没有输入。认不出为 None。"""
if not isinstance(metric, dict):
return None
paths: list[str] = []
if metric.get("expression"):
paths.append(str(metric["expression"]))
rule = metric.get("rule")
if isinstance(rule, dict):
inputs = rule.get("inputs")
if not isinstance(inputs, dict):
return None
paths.extend(str(path) for path in inputs.values())
assessment = metric.get("assessment")
if isinstance(assessment, dict):
dimensions = assessment.get("dimensions")
if not isinstance(dimensions, list):
return None
paths.extend(str(d.get("input")) for d in dimensions if isinstance(d, dict) and d.get("input"))
return paths
def views_of(workflow_id: str, workflow_name: str, version: Any) -> list[CaliberView]:
"""一个已发布版本里每张口径卡(源头的)的每个指标。"""
from app.apps.tree import node_map
+1 -1
View File
@@ -264,7 +264,7 @@ async def test_last_weeks_sales_labels_only_the_sales_number_as_from_the_caliber
row = await S.explore(monkeypatch, ANSWER, origin="chat", text=(
"上周销售额 [[v:Q1.r0.销售额]] 元,共 [[v:Q1.r0.订单数]] 单([[v:Q1.r0.区间首日]] 至 [[v:Q1.r0.区间末日]])。"))
content = (await store.latest(row.id)).content
assert content["algo"] == store.ALGO == "4"
assert content["algo"] == store.ALGO == "5"
[query] = content["queries"]
assert query["status"] == "related" and set(query["cells"]) == set(COLUMNS)
[labels] = [r["labels"] for r in content["reports"].values()]
+1 -1
View File
@@ -253,7 +253,7 @@ async def test_an_answer_built_with_use_caliber_is_labelled_composed(client, mon
await ledger_match.drain()
recorded = await store.latest(row.id)
content = recorded.content
assert content["algo"] == store.ALGO == "4"
assert content["algo"] == store.ALGO == "5"
[query] = content["queries"]
assert query["status"] == "composed" and query["node_id"] == "c1_fetch"
[cand] = query["candidates"]
+14
View File
@@ -117,3 +117,17 @@ async def test_metric_definition_and_params_are_described(client):
[m] = made["metrics"]
assert m["definition"]["text"] == "已支付订单的销售额" and m["definition"]["owner"] == "运营"
assert m["params"] == [{"name": "store", "type": "string", "default": "all"}]
async def test_an_external_only_card_is_not_promised_automatic_issuance(client):
graph = S.chain(
S.node("start", "input"),
S.node("card", "metrics", caliber="年度目标", caliber_version="v1", metrics=[{
"id": "target", "name": "年度目标", "unit": "元", "kind": "external",
"external": {"value": 100, "source": "预算", "as_of": "2026-01-01"}}]),
S.node("out", "output", fields=[{"name": "v", "value": "{{ nodes.card.metrics[0].value }}"}]))
wf = await S.publish(name="只有外部常量", graph=graph)
[made] = await descriptors(client, wf)
assert made["composable"] is True
assert made["issuance"]["auto_issue"] is False
assert any("没有取数节点" in r for r in made["issuance"]["reasons"])
+3 -1
View File
@@ -42,8 +42,10 @@ async def test_trial_runs_the_pinned_caliber_and_returns_its_metrics(client):
assert body["metrics"][0]["unit"] == "元"
async with SessionLocal() as session:
run = await session.get(Run, body["run_id"])
assert run is not None and run.origin_kind == "trial" and run.run_class == "exploratory"
assert run is not None and run.origin_kind == "caliber_trial" and run.run_class == "exploratory"
assert run.origin["kind"] == "caliber_trial" and run.origin["workflow_id"] == wf
# 试运行不是问数回答:不参与账本比对、不计入使用统计/召回
assert not ledger_match.wanted(run.run_class, run.origin_kind)
async def test_trial_refuses_missing_required_inputs_and_unknown_fields(client):
+10 -6
View File
@@ -1,7 +1,6 @@
"""指标定义卡(definition)与参数(params)的最小结构校验。
"""指标定义卡(definition)与参数(params):结构校验、四种 kind(公式/外部常量/规则/评估)的执行与标签。
执行还是公式这一种;定义文本、参数先作为语义元数据校验、进内容指纹、进 descriptor。规则/评估/外部常量没实现前,
写进图里要按错误挡住,免得被当成能算的指标。
定义文本、参数进内容指纹和 descriptor;不认识的 kind 写进图里要按错误挡住,免得被当成能算的指标。
"""
from __future__ import annotations
@@ -30,7 +29,7 @@ def test_a_formula_metric_can_carry_a_definition_and_params():
def test_an_unimplemented_kind_is_refused():
errors = _errors(_graph(kind="normative"))
assert any(i.code == "metrics.kind_unsupported" and "还没有实现" in i.message for i in errors)
assert any(i.code == "metrics.kind_unsupported" and "不认识" in i.message for i in errors)
def test_params_must_be_named_and_typed_once_each():
@@ -72,6 +71,7 @@ async def test_an_external_constant_metric_runs_without_a_query():
metric = card["metrics"][0]
assert metric["external"]["source"] == "2026 年度预算决议"
assert metric["external"]["stale"] is False
assert metric["recompute_ok"] is None and metric["recompute_reason"]
finally:
await run_manager.shutdown()
@@ -147,7 +147,8 @@ def test_a_rule_metric_is_labelled_derived_with_its_input_queries():
}]}
label = metric_label(card, "big", {}, {})
assert label["kind"] == "derived" and label["queries"] == ["q1"]
assert label["definition_hash"] == "abc" and label["rule"]["all"][0]["op"] == "eq"
assert label["definition_hash"] == "abc" and label["rule"]["conditions"] == 1
assert "paid" not in str(label)
def test_an_assessment_metric_needs_unique_dimensions_and_descending_levels():
@@ -198,9 +199,12 @@ def test_an_assessment_metric_is_labelled_assessed():
from app.ledger.labels import metric_label
card = {"definition_hash": "abc", "metrics": [{
"id": "quality", "assessment": {"score": 80, "level": "B"},
"id": "quality",
"assessment": {"score": 80, "level": "B",
"dimensions": [{"name": "手机号", "weight": 1, "value": "19900000501"}]},
"inputs": [{"artifact": "q1", "via": "tool_cell"}],
}]}
label = metric_label(card, "quality", {}, {})
assert label["kind"] == "assessed" and label["queries"] == ["q1"]
assert label["assessment"]["level"] == "B" and label["definition_hash"] == "abc"
assert "19900000501" not in str(label)
+3 -2
View File
@@ -130,7 +130,8 @@
剖析范围选「取值观察」或定时维护时,还会观察取值:行数、码值分布(包括人工确认过的码值列)、最新业务日期、最近 35 天每天的行数。
- 和同一口径(同样的取样方式)的历史比,比出新码值、比例剧变、断档、缺天、量跌、行数大起大落,记进变化记录(见下文)。
- 标了「已列全」的码值表出现目录里没有的取值时,系统撤销「已列全」,目录升一版。这是机器唯一会自己写进目录的事;新取值的含义仍要人填。
- 标了「已列全」的码值表出现目录里没有的取值时,系统撤销「已列全」,目录升一版。新取值的含义仍要人填。
- 新设遮罩列时,系统会清掉这些列在目录里留下的码值、含义、备注和确认依据,目录升一版。这是机器自己写目录的另一种情况。
- 遮罩列不观察。
## 版本与提醒
@@ -185,7 +186,7 @@
- 数据目录的「变化」视图按「要处理 / 提醒 / 说明」分组;数据源卡片上的角标是要处理的条数;表详情里有「结构与取值变化」一节。
- 每条写明影响面:说到这些列的目录项和它们的状态、写死了这一列的已发布模板、哪条 SQL 现在会报列不存在。
- 处理动作:「清理目录项」「加入码值」「重新起草」「重新剖析」「知道了」「误报」(误报必须写理由)。动作都落到目录已有的写入上,记下目录版本或建的任务。
- 机器自己只做一件事:数据证明码值表不再列全时,撤销「已列全」。
- 机器自己写目录只有两种情况:数据证明码值表不再列全时撤销「已列全」;新设遮罩列时清掉该列的码值、含义、备注和确认依据。
- 有「要处理」的变化时,可以推通知「数据源有变化要处理」,消息里只有源名、条数和链接,不放表名、列名和取值(见[通知](schedules.md#通知))。也可以作为定时任务的触发事件。
### 定时维护
+8 -1
View File
@@ -75,9 +75,16 @@
|---|---|
| 「等价已证明」 | SQL 被证明和口径等价 |
| 「助手选用」 | 助手选用口径拼出来的 |
| 「外部常量」 | 口径卡上的外部常量,带来源和生效日期;过期会降档 |
| 「规则派生」 | 规则型指标从查询格按结构化条件算出,不是某条查询与口径等价 |
| 「评估值」 | 评估型指标按评分协议算出的值,不是测量值 |
| 「相关口径」 | 用到同一张表却不等价,写明差在哪 |
| 「这次的推理」 | 和哪份口径都不沾边 |
后三种(外部常量、规则派生、评估值)不会让整题变成「等价已证明」,不自动正式出具;证据面板会照实写出处。
口径卡上的指标现在有四种:公式(`formula`)、外部常量(`external`)、规则(`rule`)、评估(`assessment`)。定义文本和参数随版本固定,并在「口径」目录页(`/ledger/calibers`)里能看、能按参数试运行;试运行是只读探索运行,不进使用统计,也不是正式出具。
- 证据面板的出处链最上面多一步「口径」;问数据每一轮底下有一行摘要。比对还没出结果时,写「正在比对口径」并按退避重取。
- 比对结果是一条封存后追加的事件 `ledger.matched`,出现在时间线和可验证报告的「封存后追加」里,不在封存范围内。启动时会补比最近 7 天的运行(跟着 `AGENTLAB_SCHEDULER` 开关)。
- 逐个数判:报告引用了一条查询的几列(例如销售额、订单数、首末日)时,每个数按它那一列单独和口径的指标列比,多查的列不影响这个数。只在去掉别的列不会改变这个数时才这样比;有 DISTINCT、按别的列排序后截断、多出来分组列的,照旧整条比。
@@ -85,7 +92,7 @@
- 整题要每个来自数据的数都被证明来自同一份口径,才自动出具。
- 分析结论里的数(占比、变化、波动范围)是从查询结果派生的,不比对、不标口径,摘要行也不计它们。
写口径的取数 SQL 时,最好不要取整(`ROUND(SUM(x), 2)`),也不要套 `COALESCE(…, 0)`。取整和缺省值交给口径卡的小数位和展示;否则问数据里没取整的写法永远认不成等价(差在哪会照实写出来)。
写口径的取数 SQL 时,不要取整(`ROUND(SUM(x), 2)`),也不要套 `COALESCE(…, 0)`。取整和缺省值交给口径卡的小数位和展示;否则问数据里没取整的写法永远认不成等价(差在哪会照实写出来)。受管级别发布时,这两类写法会直接挡住发布;已发布级别只提示。
### 按口径正式出具
+1 -1
View File
@@ -67,7 +67,7 @@ export const PAGES: PageDef[] = [
// 评测(评测-D8):搭建者以上可见(使用者的页面表在 visiblePages 里收窄,用不着另判),同样排在设置前面;快捷键用最后一个空着的 ⌥0
{ to: '/evals', label: EVAL_TEXT.nav, hint: EVAL_TEXT.hint, icon: Gauge, shortcut: 'Alt+0', keywords: 'evals eval evaluation 评测 回归 题集 准确率 A/B 口径命中 重复度' },
// 口径目录:已发布口径的入口、依赖和试跑。后加的页,快捷键用没占用的 Alt+L
{ to: '/ledger/calibers', label: '口径', hint: '查看已发布口径的入口、依赖,并按参数试跑', icon: Layers, shortcut: 'Alt+L', keywords: 'caliber ledger 口径 指标 试跑 入口' },
{ to: '/ledger/calibers', label: '口径', hint: '查看已发布口径的入口、依赖,并按参数试运行', icon: Layers, shortcut: 'Alt+L', keywords: 'caliber ledger 口径 指标 试运行 入口' },
{ to: '/settings', label: '设置', hint: '模型接入、偏好、运行环境', icon: Settings, shortcut: 'Alt+7', keywords: 'settings provider 模型 署名 主题 偏好' },
]
+1
View File
@@ -5482,6 +5482,7 @@ export const ORIGIN_LABEL: Record<OriginKind, string> = {
ledger_issue: '按口径正式出具',
reconcile: '口径对账',
recall: '口径召回',
caliber_trial: '口径试运行',
}
export const ORIGIN_TEXT = {
+5 -1
View File
@@ -153,12 +153,16 @@ export const LEDGER_UI = {
pending: '正在比对口径…',
timeout: '比对口径还没有结果',
/** 一行摘要:几个数来自哪份口径、几个是这次的推理、几个没有比对 */
summary: (parts: { caliber: number; calibers: string[]; reasoning: number; related: number; unknown: number }) => {
summary: (parts: { caliber: number; calibers: string[]; reasoning: number; related: number; unknown: number;
external?: number; derived?: number; assessed?: number }) => {
const out: string[] = []
if (parts.caliber) {
out.push(parts.calibers.length === 1 ? `${formatNumber(parts.caliber)} 个数来自口径${parts.calibers[0]}`
: `${formatNumber(parts.caliber)} 个数来自 ${formatNumber(parts.calibers.length)} 份口径`)
}
if (parts.external) out.push(`${formatNumber(parts.external)} 个数是外部常量`)
if (parts.derived) out.push(`${formatNumber(parts.derived)} 个数是规则派生`)
if (parts.assessed) out.push(`${formatNumber(parts.assessed)} 个数是评估值`)
const reasoning = parts.reasoning + parts.related
if (reasoning) {
out.push(`${formatNumber(reasoning)} 个数是这次的推理${parts.related ? `(其中 ${formatNumber(parts.related)} 个有相关口径)` : ''}`)
+15 -8
View File
@@ -65,9 +65,9 @@ export function CaliberCatalog() {
try {
const out = await api.caliberLedger.trial(d.workflow_id, d.version, d.card.node_id, values[key] ?? {})
setTrials((t) => ({ ...t, [key]: out }))
if (out.status === 'succeeded') toast.ok('试跑完成')
else if (out.status === 'running') toast.warn('试跑还在运行,打开运行记录看结果')
else toast.error(out.error || '试跑没有成功')
if (out.status === 'succeeded') toast.ok('试运行完成')
else if (out.status === 'running') toast.warn('试运行还在进行,打开运行记录看结果')
else toast.error(out.error || '试运行没有成功')
} catch (e) {
toast.error(e)
} finally {
@@ -77,7 +77,7 @@ export function CaliberCatalog() {
return (
<div className="flex h-full flex-col" data-caliber-catalog="">
<PageHeader icon={<Layers size={14} />} title="口径" subtitle="已发布口径的入口、指标、依赖与试跑"
<PageHeader icon={<Layers size={14} />} title="口径" subtitle="已发布口径的入口、指标、依赖与试运行"
actions={<button type="button" className="btn btn-xs btn-ghost" onClick={load} title="重新加载">
<RefreshCw size={12} aria-hidden /> 刷新
</button>} />
@@ -164,17 +164,24 @@ export function CaliberCatalog() {
<h3 className="mb-1 text-2xs font-semibold text-dim">入口字段</h3>
{d.entry_schema.properties && Object.keys(d.entry_schema.properties).length > 0 ? (
<SchemaForm schema={d.entry_schema} value={values[key] ?? {}}
onChange={(v) => setValues((all) => ({ ...all, [key]: v }))}
onChange={(v) => {
setValues((all) => ({ ...all, [key]: v }))
setTrials((all) => {
const next = { ...all }
delete next[key]
return next
})
}}
idPrefix={`caliber-${key}`} />
) : <p className="text-2xs text-faint">这份口径不需要参数。</p>}
</div>
<div className="flex flex-wrap items-center gap-2">
<button type="button" className="btn btn-primary btn-sm" disabled={busy === key}
onClick={() => void trial(d)} data-caliber-trial="">
{busy === key ? <Spinner size={12} /> : <Play size={12} aria-hidden />} 试跑
{busy === key ? <Spinner size={12} /> : <Play size={12} aria-hidden />} 试运行
</button>
<Link to={`/studio/${d.workflow_id}`} className="btn btn-sm btn-ghost">打开工作流</Link>
<span className="text-2xs text-faint">试跑是只读探索运行,不是正式出具</span>
<span className="text-2xs text-faint">试运行是只读探索运行,不是正式出具</span>
</div>
{result && (
<div className="rounded-md border bg-elev px-3 py-2 text-xs" data-caliber-trial-result={result.status}>
@@ -189,7 +196,7 @@ export function CaliberCatalog() {
</ul>
) : (
<Notice tone={result.status === 'running' ? 'info' : 'err'}>
{result.status === 'running' ? '还在运行:' : '试跑没有成功:'}
{result.status === 'running' ? '还在运行:' : '试运行没有成功:'}
{result.error || result.status}
{' '}<Link className="underline" to={`/runs/${result.run_id}`}>看运行</Link>
</Notice>
+1 -1
View File
@@ -364,7 +364,7 @@ export function LedgerPart({ runId, runClass, report, segId, segText, segLedger,
<p className="text-2xs text-dim">
数由规则从查询结果算出,不是某条查询与口径等价;输入仍然逐格可追溯。
</p>
{label.rule?.all?.length ? <p className="text-2xs text-faint">条件:{label.rule.all.length} 条</p> : null}
{label.rule?.conditions ? <p className="text-2xs text-faint">条件:{label.rule.conditions} 条</p> : null}
</div>
)}
{label.kind === 'assessed' && (
+5 -3
View File
@@ -21,8 +21,8 @@ import { ReconcileDialog, ReconcileResults } from './LedgerReconcile'
* 能不能自动、不能的原因都是服务端给的;摘要只是把主报告里服务端标好的数按标签数一数、写出是哪份口径。
*/
/** 摘要认的四种标签。服务端不给别的标签(分析结论的数不和口径比对,00-总方案 D11),以后多了也不计、不出 NaN */
const COUNTED_KINDS = ['caliber', 'related', 'reasoning', 'unknown'] as const
/** 摘要认的标签。分析结论的数不和口径比对(00-总方案 D11),但外部常量、规则派生、评估值要照实计数 */
const COUNTED_KINDS = ['caliber', 'external', 'derived', 'assessed', 'related', 'reasoning', 'unknown'] as const
type CountedKind = (typeof COUNTED_KINDS)[number]
const counted = (kind: unknown): kind is CountedKind => (COUNTED_KINDS as readonly unknown[]).includes(kind)
@@ -33,7 +33,9 @@ const counted = (kind: unknown): kind is CountedKind => (COUNTED_KINDS as readon
function summaryOf(data: RunLedger) {
const labels = Object.values((data.primary ? data.reports?.[data.primary]?.labels : undefined) ?? {})
.filter((l) => !!l && typeof l === 'object' && counted(l.kind))
const counts: Record<CountedKind, number> = { caliber: 0, related: 0, reasoning: 0, unknown: 0 }
const counts: Record<CountedKind, number> = {
caliber: 0, external: 0, derived: 0, assessed: 0, related: 0, reasoning: 0, unknown: 0,
}
const calibers: string[] = []
for (const l of labels) {
counts[l.kind as CountedKind] += 1
+1 -1
View File
@@ -822,7 +822,7 @@ export interface NotifyEventRow {
export type OriginKind =
| 'chat' | 'canvas' | 'launcher' | 'schedule' | 'app_entry' | 'app_view' | 'app_backfill' | 'hook' | 'api'
| 'trial' | 'personal_app'
| 'eval' | 'ledger_issue' | 'reconcile' | 'recall'
| 'eval' | 'ledger_issue' | 'reconcile' | 'recall' | 'caliber_trial'
/**
* GET /api/runs/unlisted:同样的筛选下,记录页因为「系统派生的运行默认不列」而没列出的次数。系统派生的是评测、口径对账、
+8 -3
View File
@@ -58,8 +58,13 @@ export interface LedgerLabel {
source?: string
as_of?: string
stale?: boolean
rule?: { inputs?: Record<string, string>; all?: unknown[]; then?: number; else?: number }
assessment?: { score?: number | null; level?: string | null; dimensions?: unknown[]; levels?: unknown[] }
rule?: { inputs?: string[]; conditions?: number; then?: number; else?: number }
assessment?: {
score?: number | null
level?: string | null
dimensions?: { name?: string; weight?: number }[]
levels?: { name?: string; min?: number }[]
}
caliber?: string
workflow_id?: string
version?: number
@@ -385,7 +390,7 @@ export interface CaliberMetric {
unit?: string
decimals?: number
format?: string
/** 公式(formula);规则、评估、外部常量还没实现 */
/** 公式(formula)、外部常量(external)、单元格条件规则(rule)、评估(assessment) */
kind?: string
/** 定义卡:文字定义、范围、排除项、负责人、生效日期、引用 */
definition?: { text?: string; scope?: string; exclusions?: string[]; owner?: string; effective_from?: string; citations?: unknown[] }
+4 -4
View File
@@ -147,11 +147,11 @@ await section('导航', '=== 导航 ===', async () => {
const labels = await nav(page).locator('li > a[aria-keyshortcuts] > span:not([aria-hidden])').allInnerTexts()
// 定时任务是后加的,排在设置前面;设置留在最底下,快捷键还是 ⌥7(定时任务用 ⌥8)。应用(⌥9)排在编排后面。
// 评测(next-round 评测-D8,搭建者以上)也排在设置前面,用最后一个空着的 ⌥0
check('导航十项:问数据、编排、应用、记录、数据、工具、知识、定时任务、评测、设置',
labels.join(',') === '问数据,编排,应用,记录,数据,工具,知识,定时任务,评测,设置', labels.join(','))
check('导航十一项:问数据、编排、应用、记录、数据、工具、知识、定时任务、评测、口径、设置',
labels.join(',') === '问数据,编排,应用,记录,数据,工具,知识,定时任务,评测,口径,设置', labels.join(','))
const keys = await nav(page).locator('li > a[aria-keyshortcuts]').evaluateAll((els) => els.map((el) => el.getAttribute('aria-keyshortcuts')))
check('原有快捷键都没动:问数据 ⌥1、编排 ⌥2、记录 ⌥3 … 设置 ⌥7、定时任务 ⌥8;应用是 ⌥9、评测是 ⌥0',
keys.join(',') === 'Alt+1,Alt+2,Alt+9,Alt+3,Alt+4,Alt+5,Alt+6,Alt+8,Alt+0,Alt+7', keys.join(','))
check('原有快捷键都没动:问数据 ⌥1、编排 ⌥2、记录 ⌥3 … 设置 ⌥7、定时任务 ⌥8;应用是 ⌥9、评测是 ⌥0;口径是 ⌥L',
keys.join(',') === 'Alt+1,Alt+2,Alt+9,Alt+3,Alt+4,Alt+5,Alt+6,Alt+8,Alt+0,Alt+L,Alt+7', keys.join(','))
const fontSize = await nav(page).locator('li > a[aria-keyshortcuts] > span:not([aria-hidden])').first()
.evaluate((el) => getComputedStyle(el).fontSize)
check('导航标签 11px', fontSize === '11px', fontSize)