mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 00:01:58 +00:00
复盘修正:试运行不计入统计、评估标签遮罩、规则/评估卡可组合
This commit is contained in:
@@ -330,6 +330,8 @@ async def trial_caliber(workflow_id: str, version: int, node_id: str, payload: T
|
||||
pinned = await load_pinned(session, workflow_id, int(version), node_id)
|
||||
except NotDerivable as e:
|
||||
raise HTTPException(409, e.message) from None
|
||||
if not pinned.published or pinned.retired:
|
||||
raise HTTPException(409, "这一版没有发布或已退役,不能试运行")
|
||||
fields = [dict(f) for f in pinned.entry_fields]
|
||||
known = {str(f.get("name")) for f in fields}
|
||||
unknown = sorted(str(k) for k in payload.inputs if str(k) not in known)
|
||||
@@ -343,7 +345,7 @@ async def trial_caliber(workflow_id: str, version: int, node_id: str, payload: T
|
||||
graph = caliber_probe(pinned).graph
|
||||
principal = current_principal()
|
||||
run = await run_manager.start(
|
||||
graph=graph, input_payload=values, run_class="exploratory", origin_kind="trial",
|
||||
graph=graph, input_payload=values, run_class="exploratory", origin_kind="caliber_trial",
|
||||
origin={"kind": "caliber_trial", "workflow_id": workflow_id, "version": int(version),
|
||||
"node_id": node_id, "card": pinned.node_id},
|
||||
started_by=principal.username)
|
||||
|
||||
@@ -15,7 +15,7 @@ ROUND_IN_SOURCE = "caliber.round_in_source"
|
||||
ZERO_DEFAULT = "caliber.zero_default"
|
||||
|
||||
ROUND_MESSAGE = (
|
||||
"取数 SQL 里写了 ROUND(...):取整交给口径卡的小数位和展示;取数 SQL 里写 ROUND 会让未取整的查询"
|
||||
"取数 SQL 里写了取整函数:取整交给口径卡的小数位和展示;取数 SQL 里写取整会让未取整的查询"
|
||||
"永远认不成等价(口径账本只认规范化后完全相同的结果)"
|
||||
)
|
||||
ZERO_MESSAGE = (
|
||||
|
||||
@@ -108,8 +108,11 @@ def run_fixtures(rule: Mapping[str, Any], table: str, fixtures: Iterable[Mapping
|
||||
columns: dict[str, str] = {}
|
||||
for item in items:
|
||||
for row in item.get("rows") or []:
|
||||
if not isinstance(row, Mapping):
|
||||
raise RuleInvalid("fixture 的 rows 需要是对象")
|
||||
for name, value in row.items():
|
||||
columns.setdefault(str(name), "INTEGER" if isinstance(value, int) and not isinstance(value, bool)
|
||||
key = _ident(name, "fixture 列名")
|
||||
columns.setdefault(key, "INTEGER" if isinstance(value, int) and not isinstance(value, bool)
|
||||
else "REAL" if isinstance(value, float) else "TEXT")
|
||||
if not columns:
|
||||
raise RuleInvalid("fixture 需要至少一行 rows")
|
||||
|
||||
@@ -512,6 +512,9 @@ async def _plan_metric(state: GraphState, metric: dict[str, Any], options: _Opti
|
||||
"status": "pending", "original": {"value": metric.get("value"), "rendered": metric.get("rendered")},
|
||||
"blind": {"value": None, "rendered": None}, "delta_pct": None, "inputs": [], "differing": [],
|
||||
}
|
||||
if metric.get("rule") is not None or metric.get("assessment") is not None or metric.get("external") is not None:
|
||||
row.update(status=UNCHECKED, reason="这个指标不是按表达式算的,不参与双盲复算")
|
||||
return row, {}
|
||||
tops = _tops(metric)
|
||||
# masks 是每份用到的快照要遮的列(cell() 的、按位置取的格落在的),queries 只是 cell() 用到的那几份:重算时容器
|
||||
# 输入按内容对快照只对这几份。按位置取的格落在的快照算遮罩只为认这一格,不当成指标用到的结果——放进去的话,同一个
|
||||
|
||||
@@ -120,18 +120,31 @@ async def run_metrics(state: GraphState, ctx: NodeContext) -> dict[str, Any]:
|
||||
value: Any = None
|
||||
details: list[dict[str, Any]] = []
|
||||
level: str | None = None
|
||||
out_of_range: list[str] = []
|
||||
if not missing:
|
||||
total = sum(float(d.get("weight", 1)) for d in dimensions)
|
||||
points = 0.0
|
||||
bad: str | None = None
|
||||
for d in dimensions:
|
||||
low, high = float(d.get("min", 0)), float(d.get("max", 100))
|
||||
raw = float(values_by_name[str(d["name"])])
|
||||
try:
|
||||
raw = float(values_by_name[str(d["name"])])
|
||||
except (TypeError, ValueError):
|
||||
bad = f"维度「{d['name']}」的输入不是数({values_by_name[str(d['name'])]!r})"
|
||||
break
|
||||
normalized = max(0.0, min(100.0, (raw - low) / (high - low) * 100))
|
||||
weight = float(d.get("weight", 1))
|
||||
points += normalized * weight
|
||||
details.append({"name": d["name"], "value": raw, "normalized": normalized, "weight": weight})
|
||||
value = round_metric(points / total, display["decimals"])
|
||||
level = next((str(l["name"]) for l in assessment["levels"] if value >= float(l["min"])), None)
|
||||
if raw < low or raw > high:
|
||||
out_of_range.append(str(d["name"]))
|
||||
details.append({"name": d["name"], "value": raw, "normalized": normalized, "weight": weight,
|
||||
"out_of_range": raw < low or raw > high})
|
||||
if bad:
|
||||
errors.append(f"指标「{metric_id}」:{bad}")
|
||||
continue
|
||||
raw_score = points / total
|
||||
value = round_metric(raw_score, display["decimals"])
|
||||
level = next((str(l["name"]) for l in assessment["levels"] if raw_score >= float(l["min"])), None)
|
||||
inputs = [_input_of(str(d["input"]), values_by_name[str(d["name"])], state, ctx, tctx)
|
||||
for d in dimensions]
|
||||
metrics.append({
|
||||
@@ -145,10 +158,14 @@ async def run_metrics(state: GraphState, ctx: NodeContext) -> dict[str, Any]:
|
||||
"rendered": _rendered(value, definition, display),
|
||||
"inputs": inputs,
|
||||
"substituted": "加权评分",
|
||||
"recompute_ok": True,
|
||||
"recompute_ok": None,
|
||||
"recompute_reason": "这个指标不按表达式复算",
|
||||
"status": "ok" if value is not None else "missing_input",
|
||||
"assessment": {"score": value, "level": level, "dimensions": details,
|
||||
"levels": assessment["levels"], **({"missing": missing} if missing else {})},
|
||||
"levels": assessment["levels"],
|
||||
**({"missing": missing} if missing else {}),
|
||||
**({"out_of_range": out_of_range} if out_of_range else {}),
|
||||
**({"unlevelled": True} if value is not None and level is None else {})},
|
||||
})
|
||||
continue
|
||||
if str(definition.get("kind") or "formula") == "rule":
|
||||
@@ -179,7 +196,8 @@ async def run_metrics(state: GraphState, ctx: NodeContext) -> dict[str, Any]:
|
||||
"rendered": _rendered(value, definition, display),
|
||||
"inputs": inputs,
|
||||
"substituted": _rule_text(rule),
|
||||
"recompute_ok": True,
|
||||
"recompute_ok": None,
|
||||
"recompute_reason": "这个指标不按表达式复算",
|
||||
"status": "ok" if value is not None else "missing_input",
|
||||
"rule": rule,
|
||||
})
|
||||
@@ -202,7 +220,8 @@ async def run_metrics(state: GraphState, ctx: NodeContext) -> dict[str, Any]:
|
||||
"rendered": _rendered(value, definition, display),
|
||||
"inputs": [],
|
||||
"substituted": str(value),
|
||||
"recompute_ok": True,
|
||||
"recompute_ok": None,
|
||||
"recompute_reason": "这个指标不按表达式复算",
|
||||
"status": "ok",
|
||||
"external": external,
|
||||
})
|
||||
@@ -605,10 +624,18 @@ def _rule_matches(values: dict[str, Any], rule: dict[str, Any]) -> bool:
|
||||
return False
|
||||
continue
|
||||
b = cond.get("value")
|
||||
try:
|
||||
ok = {"eq": a == b, "ne": a != b, "gt": a > b, "gte": a >= b, "lt": a < b, "lte": a <= b}.get(op, False)
|
||||
except TypeError:
|
||||
ok = False
|
||||
# 类型对不上(字符串和数)按条件不成立,不静默走 ne;大小比较只对数
|
||||
if op in ("eq", "ne"):
|
||||
comparable = (isinstance(a, (int, float)) and not isinstance(a, bool)
|
||||
and isinstance(b, (int, float)) and not isinstance(b, bool)) or type(a) is type(b)
|
||||
if not comparable:
|
||||
return False
|
||||
equals = a == b
|
||||
ok = equals if op == "eq" else not equals
|
||||
else:
|
||||
if not all(isinstance(v, (int, float)) and not isinstance(v, bool) for v in (a, b)):
|
||||
return False
|
||||
ok = {"gt": a > b, "gte": a >= b, "lt": a < b, "lte": a <= b}.get(op, False)
|
||||
if not ok:
|
||||
return False
|
||||
return True
|
||||
|
||||
@@ -29,6 +29,7 @@ ORIGIN_KINDS = (
|
||||
"ledger_issue", # 问数据的答案按已治理口径正式出具(正式运行)
|
||||
"reconcile", # 口径对账:这次的取数和口径在同一份数据上各算一次(探索运行)
|
||||
"recall", # 口径召回:新旧两版口径在同一份数据上各算一次(探索运行)
|
||||
"caliber_trial", # 口径目录里的试跑(探索运行,不是问数回答,也不进统计/召回)
|
||||
)
|
||||
|
||||
#: 客户端在 POST /api/runs 里能声明的来源
|
||||
@@ -38,7 +39,7 @@ CLIENT_ORIGINS = ("chat", "canvas", "launcher", "api")
|
||||
#: 不是有人在用。它们是完整的运行(证据、封存都在、点得开),但不进使用统计——探索问题聚类、工具白名单衰减、数据目录的
|
||||
#: 使用次数和「被哪些运行用过」、审阅队列的使用权重、以后的账本统计——否则一跑评测,治理信号和目录的使用热度就失真了。
|
||||
#: 记录页默认也不列(筛选里能选,列表照实写有几次没列出)。ledger_issue 是替用户正式出具,照常计入
|
||||
SYSTEM_ORIGINS = ("eval", "reconcile", "recall")
|
||||
SYSTEM_ORIGINS = ("eval", "reconcile", "recall", "caliber_trial")
|
||||
|
||||
|
||||
def counted_runs() -> Any:
|
||||
|
||||
@@ -1086,8 +1086,8 @@ def _check_challenge(node: GraphNode, spec: GraphSpec, result: ValidationResult)
|
||||
field="model", code="challenge.same_model")
|
||||
|
||||
|
||||
#: 指标定义的类型。现在只有公式(表达式)这一种在运行时能算;规则、评估、外部常量还没实现,
|
||||
#: 先在校验里挡住,免得写进图里以后被当成能算的指标
|
||||
#: 指标定义的类型:公式(表达式)、外部常量、单元格条件规则、评估评分。四种都在运行时能算;
|
||||
#: 不认识的类型仍在校验里挡住,免得被当成能算的指标
|
||||
METRIC_KINDS = ("formula", "external", "rule", "assessment")
|
||||
#: 定义卡参数的标量类型(和入口表单的字段类型同一套子集)
|
||||
METRIC_PARAM_TYPES = ("string", "number", "integer", "boolean")
|
||||
@@ -1110,8 +1110,8 @@ def metric_definition_issues(node: GraphNode, metric: Any, index: int) -> list[V
|
||||
add(f"指标定义缺少 ID:{field}", code="metrics.id_missing")
|
||||
kind = str(metric.get("kind") or "formula")
|
||||
if kind not in METRIC_KINDS:
|
||||
add(f"指标「{metric.get('id') or index + 1}」的类型「{kind}」还没有实现:现在支持「公式」(formula)和"
|
||||
"「外部常量」(external)。规则、评估请先在定义卡里写成说明,等对应的执行方式上线后再发布",
|
||||
add(f"指标「{metric.get('id') or index + 1}」的类型「{kind}」不认识:现在支持「公式」(formula)、"
|
||||
"「外部常量」(external)、「规则」(rule)、「评估」(assessment)",
|
||||
f"{field}.kind", code="metrics.kind_unsupported")
|
||||
if kind == "external":
|
||||
external = metric.get("external")
|
||||
@@ -1151,6 +1151,10 @@ def metric_definition_issues(node: GraphNode, metric: Any, index: int) -> list[V
|
||||
if not isinstance(name, str) or not name.strip() or not isinstance(path, str) \
|
||||
or not path.strip():
|
||||
add("规则的每个输入都要有名字和取值路径", f"{field}.rule.inputs", code="metrics.rule_invalid")
|
||||
elif not any(token in path for token in ("cell(", "nodes.", "vars.", "input.")):
|
||||
add(f"规则输入「{name}」的取值路径要写成 cell(nodes.…)、vars.… 或 input.…;"
|
||||
"表列名是规则 DSL 编译接口的写法,不能直接写进口径卡",
|
||||
f"{field}.rule.inputs.{name}", code="metrics.rule_invalid")
|
||||
conditions = rule.get("all")
|
||||
if not isinstance(conditions, list) or not conditions:
|
||||
add("规则需要 all:至少一条条件", f"{field}.rule.all", code="metrics.rule_invalid")
|
||||
@@ -1163,8 +1167,8 @@ def metric_definition_issues(node: GraphNode, metric: Any, index: int) -> list[V
|
||||
if cond.get("op") not in RULE_OPS:
|
||||
add(f"条件运算符只能是 {'、'.join(RULE_OPS)}", f"{key}.op", code="metrics.rule_invalid")
|
||||
if cond.get("op") in ("in", "not_in"):
|
||||
if not isinstance(cond.get("values"), list):
|
||||
add("in / not_in 需要 values 列表", f"{key}.values", code="metrics.rule_invalid")
|
||||
if not isinstance(cond.get("values"), list) or not cond["values"]:
|
||||
add("in / not_in 需要非空的 values 列表", f"{key}.values", code="metrics.rule_invalid")
|
||||
elif "value" not in cond:
|
||||
add("条件需要 value", f"{key}.value", code="metrics.rule_invalid")
|
||||
for key in ("then", "else"):
|
||||
@@ -1194,6 +1198,9 @@ def metric_definition_issues(node: GraphNode, metric: Any, index: int) -> list[V
|
||||
names.add(name)
|
||||
if not isinstance(dimension.get("input"), str) or not dimension["input"].strip():
|
||||
add("评估维度需要 input(取值路径)", f"{key}.input", code="metrics.assessment_invalid")
|
||||
elif not any(token in dimension["input"] for token in ("cell(", "nodes.", "vars.", "input.")):
|
||||
add("评估维度的 input 要写成 cell(nodes.…)、vars.… 或 input.…",
|
||||
f"{key}.input", code="metrics.assessment_invalid")
|
||||
weight = dimension.get("weight", 1)
|
||||
if isinstance(weight, bool) or not isinstance(weight, (int, float)) or weight <= 0:
|
||||
add("评估维度的 weight 需要是正数", f"{key}.weight", code="metrics.assessment_invalid")
|
||||
@@ -1220,6 +1227,10 @@ def metric_definition_issues(node: GraphNode, metric: Any, index: int) -> list[V
|
||||
add("评估等级要按 min 从高到低排列", f"{key}.min", code="metrics.assessment_invalid")
|
||||
else:
|
||||
previous = float(minimum)
|
||||
if levels and isinstance(levels[-1], dict) and isinstance(levels[-1].get("min"), (int, float)) \
|
||||
and not isinstance(levels[-1]["min"], bool) and levels[-1]["min"] > 0:
|
||||
add("最后一个评估等级的 min 要不大于 0,否则低分没有等级可落",
|
||||
f"{field}.assessment.levels[-1].min", code="metrics.assessment_invalid")
|
||||
elif not metric.get("expression"):
|
||||
add(f"指标定义缺少表达式:{metric.get('id') or '(空)'}", code="metrics.expression_missing")
|
||||
|
||||
|
||||
@@ -77,6 +77,17 @@ async def _describe(session: AsyncSession, workflow: Workflow, row: Any, node_id
|
||||
cfg = card["config"]
|
||||
level = str(row.level or "")
|
||||
_, reason = card_inputs(graph, node_id)
|
||||
if reason is None:
|
||||
from app.ledger.derive import template_problem
|
||||
|
||||
for nid, node in nodes.items():
|
||||
tool = str((node.get("config") or {}).get("tool") or "")
|
||||
sql = ((node.get("config") or {}).get("args") or {}).get("sql") \
|
||||
if isinstance((node.get("config") or {}).get("args"), dict) else None
|
||||
if node.get("type") == "tool" and tool.startswith(QUERY_PREFIX) and isinstance(sql, str) and sql.strip():
|
||||
if problem := template_problem(str(nid), node.get("config") or {}):
|
||||
reason = problem
|
||||
break
|
||||
key = (workflow.id, int(row.version))
|
||||
if key not in side_cache:
|
||||
side_cache[key] = await graph_side_effect(session, graph, level=level or None)
|
||||
@@ -86,7 +97,9 @@ async def _describe(session: AsyncSession, workflow: Workflow, row: Any, node_id
|
||||
if level != "governed":
|
||||
reasons.append("这一版是已发布级别,只支持一键出具")
|
||||
if side_effect is not None:
|
||||
reasons.append(f"这一版{side_effect},不能替人重跑")
|
||||
reasons.append(f"这一版{side_effect},不能替人再次执行")
|
||||
if not sources:
|
||||
reasons.append("这一版没有取数节点,账本不能按口径证明等价")
|
||||
return {
|
||||
"workflow_id": workflow.id,
|
||||
"workflow_name": workflow.name,
|
||||
@@ -103,7 +116,7 @@ async def _describe(session: AsyncSession, workflow: Workflow, row: Any, node_id
|
||||
"compose_reason": reason,
|
||||
"dependencies": {"sources": sorted(sources), "tables": sorted(tables),
|
||||
"catalog_versions": row.catalog_versions, "schema_versions": row.schema_versions},
|
||||
"issuance": {"level": level, "auto_issue": level == "governed" and side_effect is None,
|
||||
"issuance": {"level": level, "auto_issue": level == "governed" and side_effect is None and bool(sources),
|
||||
"requires_independent_proof": True, "side_effects": side_effect, "reasons": reasons},
|
||||
}
|
||||
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
"""逐个数标口径(docs/design/next-round/02-口径账本.md 做法 3):报告里每个引用了查询格、口径卡指标的数,来自哪一份口径。
|
||||
|
||||
标签四种(KINDS):
|
||||
标签(KINDS):
|
||||
- caliber:来自口径「X」vN。how 是怎么认的:independent「等价已证明」(这次 Agent 自己写的 SQL 被证明和口径的取数等价)、
|
||||
composed「助手选用」(助手 use_caliber 按口径组合出来的取数,证明了和那一版一致:ledger/match 的 composed)。
|
||||
- external:外部常量(口径卡上的 kind=external),带来源、生效日期、是否过期。
|
||||
- derived:规则型指标(kind=rule),数由结构化规则从输入算出,不是某条查询与口径等价。
|
||||
- assessed:评估型指标(kind=assessment),按评分协议算出的评估值,不是测量值。
|
||||
- related:相关口径——同一个数据源、查了同样的表的已发布口径,但不等价;附差在哪(explain_sql_diff 的条目)。
|
||||
- reasoning:这次的推理——没有相关的已发布口径。
|
||||
- unknown:没比对、比对没做完(超时、解析失败)、快照读不出来。
|
||||
@@ -309,13 +312,26 @@ def metric_label(card: Mapping[str, Any] | None, metric_id: Any, queries: Mappin
|
||||
inputs = [i for i in metric.get("inputs") or [] if isinstance(i, dict)]
|
||||
from_queries = [i for i in inputs if i.get("artifact") and i.get("via") in _SNAPSHOT_VIAS]
|
||||
if metric.get("rule") is not None:
|
||||
# 规则型指标:数由规则从输入算出,不是某条查询与口径等价;输入仍然逐格可追溯
|
||||
return {"kind": "derived", "rule": metric.get("rule"),
|
||||
# 规则型指标:数由规则从输入算出,不是某条查询与口径等价;输入仍然逐格可追溯。
|
||||
# 标签里只给结构(输入名、条件条数、then/else),不给条件里的字面量——那可能是遮罩列的取值
|
||||
rule = metric.get("rule") if isinstance(metric.get("rule"), dict) else {}
|
||||
return {"kind": "derived",
|
||||
"rule": {"inputs": sorted(str(k) for k in (rule.get("inputs") or {})),
|
||||
"conditions": len(rule.get("all") or []),
|
||||
"then": rule.get("then", 1), "else": rule.get("else", 0)},
|
||||
"queries": [str(i["artifact"]) for i in from_queries],
|
||||
**({"definition_hash": card.get("definition_hash")} if card.get("definition_hash") else {})}
|
||||
if metric.get("assessment") is not None:
|
||||
# 评估值:按评分协议从输入算出,不是测量值,也不参与自动出具
|
||||
return {"kind": "assessed", "assessment": metric.get("assessment"),
|
||||
# 评估值:按评分协议从输入算出,不是测量值,也不参与自动出具。标签里不带各维度的原始取值
|
||||
# (可能是遮罩列的取值),只给得分、等级和维度名/权重
|
||||
assessment = metric.get("assessment") if isinstance(metric.get("assessment"), dict) else {}
|
||||
dimensions = [{"name": d.get("name"), "weight": d.get("weight")}
|
||||
for d in assessment.get("dimensions") or [] if isinstance(d, dict)]
|
||||
return {"kind": "assessed",
|
||||
"assessment": {"score": assessment.get("score"), "level": assessment.get("level"),
|
||||
"dimensions": dimensions,
|
||||
"levels": [{"name": l.get("name"), "min": l.get("min")}
|
||||
for l in assessment.get("levels") or [] if isinstance(l, dict)]},
|
||||
"queries": [str(i["artifact"]) for i in from_queries],
|
||||
**({"definition_hash": card.get("definition_hash")} if card.get("definition_hash") else {})}
|
||||
if not from_queries:
|
||||
@@ -445,7 +461,11 @@ def whole_of(labels: Mapping[str, Mapping[str, Any]], calibers: Mapping[str, Map
|
||||
keys = {label.get("caliber") for label in labels.values() if label.get("kind") == "caliber"}
|
||||
if counts["caliber"] != len(labels):
|
||||
# 逐格判之后常见「一部分数来自口径、另几个只是相关」:照实说是不一致,不说成这次的推理
|
||||
reason = ("有的数没有比对" if counts["unknown"] else "没有数来自口径" if not counts["caliber"]
|
||||
reason = ("有的数没有比对" if counts["unknown"]
|
||||
else "有的数是外部常量,不是测量值" if counts["external"]
|
||||
else "有的数是规则派生的,不是测量值" if counts["derived"]
|
||||
else "有的数是评估值,不是测量值" if counts["assessed"]
|
||||
else "没有数来自口径" if not counts["caliber"]
|
||||
else "有的数和相关口径不一致,不来自口径" if counts["related"] else "有的数是这次的推理")
|
||||
return {"equivalent": False, "reason": reason, "counts": counts}
|
||||
if len(keys) != 1:
|
||||
|
||||
@@ -34,7 +34,8 @@ FORMAT = "agentlab.caliber_match/1"
|
||||
#: 3:整条比不等价时按报告引用的列投影再比(ledger/match 的 _projected),候选记 projection;没投影的照旧
|
||||
#: 4:按全部引用列投影仍没证明的查询,被引用的每一列逐格各比一次(queries[].cells、one_row),每个数按它那一格标;老工件
|
||||
#: 没有 cells,照旧按整条查询读
|
||||
ALGO = "4"
|
||||
#: 5:候选记定义指纹(caliber_hash / definition_hashes),标签新增 external / derived / assessed;老工件没有这些键照旧读
|
||||
ALGO = "5"
|
||||
#: 触发方式(见模块说明)。certified(第 3 波):从问数据固化出的口径发布之后,对来源运行补比的(ledger/certify)
|
||||
TRIGGERS = ("sealed", "backfill", "on_demand", "certified")
|
||||
|
||||
|
||||
@@ -106,33 +106,59 @@ def card_inputs(graph: dict[str, Any], card_id: str) -> tuple[list[dict[str, Any
|
||||
return [], CHAINED_REASON
|
||||
found: dict[str, dict[str, Any]] = {}
|
||||
for metric in card["config"].get("metrics") or []:
|
||||
try:
|
||||
tree = parse_expression(str((metric or {}).get("expression") or ""))[0]
|
||||
except ExpressionError:
|
||||
paths = _metric_paths(metric)
|
||||
if paths is None:
|
||||
return [], NOT_TOOL_REASON
|
||||
for path in leaf_refs(tree):
|
||||
parts = cell_parts(path)
|
||||
if parts is None:
|
||||
return [], NOT_TOOL_REASON
|
||||
chain, row, column = parts
|
||||
node_id = chain[len("nodes."):] if chain.startswith("nodes.") else ""
|
||||
tool = str((nodes.get(node_id) or {}).get("config", {}).get("tool") or "")
|
||||
if (nodes.get(node_id) or {}).get("type") != "tool" or not tool.startswith(QUERY_PREFIX):
|
||||
return [], NOT_TOOL_REASON
|
||||
if row.strip() != "0":
|
||||
return [], ROW_REASON
|
||||
for path in paths:
|
||||
try:
|
||||
col = json.loads(column.replace("'", '"')) if column[:1] in ("'", '"') else None
|
||||
except ValueError:
|
||||
col = None
|
||||
if not isinstance(col, str):
|
||||
tree = parse_expression(path)[0]
|
||||
except ExpressionError:
|
||||
return [], NOT_TOOL_REASON
|
||||
entry = found.setdefault(node_id, {"node": node_id, "tool": tool, "columns": []})
|
||||
if col not in entry["columns"]:
|
||||
entry["columns"].append(col)
|
||||
for leaf in leaf_refs(tree):
|
||||
parts = cell_parts(leaf)
|
||||
if parts is None:
|
||||
return [], NOT_TOOL_REASON
|
||||
chain, row, column = parts
|
||||
node_id = chain[len("nodes."):] if chain.startswith("nodes.") else ""
|
||||
tool = str((nodes.get(node_id) or {}).get("config", {}).get("tool") or "")
|
||||
if (nodes.get(node_id) or {}).get("type") != "tool" or not tool.startswith(QUERY_PREFIX):
|
||||
return [], NOT_TOOL_REASON
|
||||
if row.strip() != "0":
|
||||
return [], ROW_REASON
|
||||
try:
|
||||
col = json.loads(column.replace("'", '"')) if column[:1] in ("'", '"') else None
|
||||
except ValueError:
|
||||
col = None
|
||||
if not isinstance(col, str):
|
||||
return [], NOT_TOOL_REASON
|
||||
entry = found.setdefault(node_id, {"node": node_id, "tool": tool, "columns": []})
|
||||
if col not in entry["columns"]:
|
||||
entry["columns"].append(col)
|
||||
return list(found.values()), None
|
||||
|
||||
|
||||
def _metric_paths(metric: Any) -> list[str] | None:
|
||||
"""一个指标读的取值路径:公式的表达式、规则各输入、评估各维度。外部常量没有输入。认不出为 None。"""
|
||||
if not isinstance(metric, dict):
|
||||
return None
|
||||
paths: list[str] = []
|
||||
if metric.get("expression"):
|
||||
paths.append(str(metric["expression"]))
|
||||
rule = metric.get("rule")
|
||||
if isinstance(rule, dict):
|
||||
inputs = rule.get("inputs")
|
||||
if not isinstance(inputs, dict):
|
||||
return None
|
||||
paths.extend(str(path) for path in inputs.values())
|
||||
assessment = metric.get("assessment")
|
||||
if isinstance(assessment, dict):
|
||||
dimensions = assessment.get("dimensions")
|
||||
if not isinstance(dimensions, list):
|
||||
return None
|
||||
paths.extend(str(d.get("input")) for d in dimensions if isinstance(d, dict) and d.get("input"))
|
||||
return paths
|
||||
|
||||
|
||||
def views_of(workflow_id: str, workflow_name: str, version: Any) -> list[CaliberView]:
|
||||
"""一个已发布版本里每张口径卡(源头的)的每个指标。"""
|
||||
from app.apps.tree import node_map
|
||||
|
||||
@@ -264,7 +264,7 @@ async def test_last_weeks_sales_labels_only_the_sales_number_as_from_the_caliber
|
||||
row = await S.explore(monkeypatch, ANSWER, origin="chat", text=(
|
||||
"上周销售额 [[v:Q1.r0.销售额]] 元,共 [[v:Q1.r0.订单数]] 单([[v:Q1.r0.区间首日]] 至 [[v:Q1.r0.区间末日]])。"))
|
||||
content = (await store.latest(row.id)).content
|
||||
assert content["algo"] == store.ALGO == "4"
|
||||
assert content["algo"] == store.ALGO == "5"
|
||||
[query] = content["queries"]
|
||||
assert query["status"] == "related" and set(query["cells"]) == set(COLUMNS)
|
||||
[labels] = [r["labels"] for r in content["reports"].values()]
|
||||
|
||||
@@ -253,7 +253,7 @@ async def test_an_answer_built_with_use_caliber_is_labelled_composed(client, mon
|
||||
await ledger_match.drain()
|
||||
recorded = await store.latest(row.id)
|
||||
content = recorded.content
|
||||
assert content["algo"] == store.ALGO == "4"
|
||||
assert content["algo"] == store.ALGO == "5"
|
||||
[query] = content["queries"]
|
||||
assert query["status"] == "composed" and query["node_id"] == "c1_fetch"
|
||||
[cand] = query["candidates"]
|
||||
|
||||
@@ -117,3 +117,17 @@ async def test_metric_definition_and_params_are_described(client):
|
||||
[m] = made["metrics"]
|
||||
assert m["definition"]["text"] == "已支付订单的销售额" and m["definition"]["owner"] == "运营"
|
||||
assert m["params"] == [{"name": "store", "type": "string", "default": "all"}]
|
||||
|
||||
|
||||
async def test_an_external_only_card_is_not_promised_automatic_issuance(client):
|
||||
graph = S.chain(
|
||||
S.node("start", "input"),
|
||||
S.node("card", "metrics", caliber="年度目标", caliber_version="v1", metrics=[{
|
||||
"id": "target", "name": "年度目标", "unit": "元", "kind": "external",
|
||||
"external": {"value": 100, "source": "预算", "as_of": "2026-01-01"}}]),
|
||||
S.node("out", "output", fields=[{"name": "v", "value": "{{ nodes.card.metrics[0].value }}"}]))
|
||||
wf = await S.publish(name="只有外部常量", graph=graph)
|
||||
[made] = await descriptors(client, wf)
|
||||
assert made["composable"] is True
|
||||
assert made["issuance"]["auto_issue"] is False
|
||||
assert any("没有取数节点" in r for r in made["issuance"]["reasons"])
|
||||
|
||||
@@ -42,8 +42,10 @@ async def test_trial_runs_the_pinned_caliber_and_returns_its_metrics(client):
|
||||
assert body["metrics"][0]["unit"] == "元"
|
||||
async with SessionLocal() as session:
|
||||
run = await session.get(Run, body["run_id"])
|
||||
assert run is not None and run.origin_kind == "trial" and run.run_class == "exploratory"
|
||||
assert run is not None and run.origin_kind == "caliber_trial" and run.run_class == "exploratory"
|
||||
assert run.origin["kind"] == "caliber_trial" and run.origin["workflow_id"] == wf
|
||||
# 试运行不是问数回答:不参与账本比对、不计入使用统计/召回
|
||||
assert not ledger_match.wanted(run.run_class, run.origin_kind)
|
||||
|
||||
|
||||
async def test_trial_refuses_missing_required_inputs_and_unknown_fields(client):
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
"""指标定义卡(definition)与参数(params)的最小结构校验。
|
||||
"""指标定义卡(definition)与参数(params):结构校验、四种 kind(公式/外部常量/规则/评估)的执行与标签。
|
||||
|
||||
执行还是公式这一种;定义文本、参数先作为语义元数据校验、进内容指纹、进 descriptor。规则/评估/外部常量没实现前,
|
||||
写进图里要按错误挡住,免得被当成能算的指标。
|
||||
定义文本、参数进内容指纹和 descriptor;不认识的 kind 写进图里要按错误挡住,免得被当成能算的指标。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -30,7 +29,7 @@ def test_a_formula_metric_can_carry_a_definition_and_params():
|
||||
|
||||
def test_an_unimplemented_kind_is_refused():
|
||||
errors = _errors(_graph(kind="normative"))
|
||||
assert any(i.code == "metrics.kind_unsupported" and "还没有实现" in i.message for i in errors)
|
||||
assert any(i.code == "metrics.kind_unsupported" and "不认识" in i.message for i in errors)
|
||||
|
||||
|
||||
def test_params_must_be_named_and_typed_once_each():
|
||||
@@ -72,6 +71,7 @@ async def test_an_external_constant_metric_runs_without_a_query():
|
||||
metric = card["metrics"][0]
|
||||
assert metric["external"]["source"] == "2026 年度预算决议"
|
||||
assert metric["external"]["stale"] is False
|
||||
assert metric["recompute_ok"] is None and metric["recompute_reason"]
|
||||
finally:
|
||||
await run_manager.shutdown()
|
||||
|
||||
@@ -147,7 +147,8 @@ def test_a_rule_metric_is_labelled_derived_with_its_input_queries():
|
||||
}]}
|
||||
label = metric_label(card, "big", {}, {})
|
||||
assert label["kind"] == "derived" and label["queries"] == ["q1"]
|
||||
assert label["definition_hash"] == "abc" and label["rule"]["all"][0]["op"] == "eq"
|
||||
assert label["definition_hash"] == "abc" and label["rule"]["conditions"] == 1
|
||||
assert "paid" not in str(label)
|
||||
|
||||
|
||||
def test_an_assessment_metric_needs_unique_dimensions_and_descending_levels():
|
||||
@@ -198,9 +199,12 @@ def test_an_assessment_metric_is_labelled_assessed():
|
||||
from app.ledger.labels import metric_label
|
||||
|
||||
card = {"definition_hash": "abc", "metrics": [{
|
||||
"id": "quality", "assessment": {"score": 80, "level": "B"},
|
||||
"id": "quality",
|
||||
"assessment": {"score": 80, "level": "B",
|
||||
"dimensions": [{"name": "手机号", "weight": 1, "value": "19900000501"}]},
|
||||
"inputs": [{"artifact": "q1", "via": "tool_cell"}],
|
||||
}]}
|
||||
label = metric_label(card, "quality", {}, {})
|
||||
assert label["kind"] == "assessed" and label["queries"] == ["q1"]
|
||||
assert label["assessment"]["level"] == "B" and label["definition_hash"] == "abc"
|
||||
assert "19900000501" not in str(label)
|
||||
|
||||
@@ -130,7 +130,8 @@
|
||||
剖析范围选「取值观察」或定时维护时,还会观察取值:行数、码值分布(包括人工确认过的码值列)、最新业务日期、最近 35 天每天的行数。
|
||||
|
||||
- 和同一口径(同样的取样方式)的历史比,比出新码值、比例剧变、断档、缺天、量跌、行数大起大落,记进变化记录(见下文)。
|
||||
- 标了「已列全」的码值表出现目录里没有的取值时,系统撤销「已列全」,目录升一版。这是机器唯一会自己写进目录的事;新取值的含义仍要人填。
|
||||
- 标了「已列全」的码值表出现目录里没有的取值时,系统撤销「已列全」,目录升一版。新取值的含义仍要人填。
|
||||
- 新设遮罩列时,系统会清掉这些列在目录里留下的码值、含义、备注和确认依据,目录升一版。这是机器自己写目录的另一种情况。
|
||||
- 遮罩列不观察。
|
||||
|
||||
## 版本与提醒
|
||||
@@ -185,7 +186,7 @@
|
||||
- 数据目录的「变化」视图按「要处理 / 提醒 / 说明」分组;数据源卡片上的角标是要处理的条数;表详情里有「结构与取值变化」一节。
|
||||
- 每条写明影响面:说到这些列的目录项和它们的状态、写死了这一列的已发布模板、哪条 SQL 现在会报列不存在。
|
||||
- 处理动作:「清理目录项」「加入码值」「重新起草」「重新剖析」「知道了」「误报」(误报必须写理由)。动作都落到目录已有的写入上,记下目录版本或建的任务。
|
||||
- 机器自己只做一件事:数据证明码值表不再列全时,撤销「已列全」。
|
||||
- 机器自己写目录只有两种情况:数据证明码值表不再列全时撤销「已列全」;新设遮罩列时清掉该列的码值、含义、备注和确认依据。
|
||||
- 有「要处理」的变化时,可以推通知「数据源有变化要处理」,消息里只有源名、条数和链接,不放表名、列名和取值(见[通知](schedules.md#通知))。也可以作为定时任务的触发事件。
|
||||
|
||||
### 定时维护
|
||||
|
||||
@@ -75,9 +75,16 @@
|
||||
|---|---|
|
||||
| 「等价已证明」 | SQL 被证明和口径等价 |
|
||||
| 「助手选用」 | 助手选用口径拼出来的 |
|
||||
| 「外部常量」 | 口径卡上的外部常量,带来源和生效日期;过期会降档 |
|
||||
| 「规则派生」 | 规则型指标从查询格按结构化条件算出,不是某条查询与口径等价 |
|
||||
| 「评估值」 | 评估型指标按评分协议算出的值,不是测量值 |
|
||||
| 「相关口径」 | 用到同一张表却不等价,写明差在哪 |
|
||||
| 「这次的推理」 | 和哪份口径都不沾边 |
|
||||
|
||||
后三种(外部常量、规则派生、评估值)不会让整题变成「等价已证明」,不自动正式出具;证据面板会照实写出处。
|
||||
|
||||
口径卡上的指标现在有四种:公式(`formula`)、外部常量(`external`)、规则(`rule`)、评估(`assessment`)。定义文本和参数随版本固定,并在「口径」目录页(`/ledger/calibers`)里能看、能按参数试运行;试运行是只读探索运行,不进使用统计,也不是正式出具。
|
||||
|
||||
- 证据面板的出处链最上面多一步「口径」;问数据每一轮底下有一行摘要。比对还没出结果时,写「正在比对口径」并按退避重取。
|
||||
- 比对结果是一条封存后追加的事件 `ledger.matched`,出现在时间线和可验证报告的「封存后追加」里,不在封存范围内。启动时会补比最近 7 天的运行(跟着 `AGENTLAB_SCHEDULER` 开关)。
|
||||
- 逐个数判:报告引用了一条查询的几列(例如销售额、订单数、首末日)时,每个数按它那一列单独和口径的指标列比,多查的列不影响这个数。只在去掉别的列不会改变这个数时才这样比;有 DISTINCT、按别的列排序后截断、多出来分组列的,照旧整条比。
|
||||
@@ -85,7 +92,7 @@
|
||||
- 整题要每个来自数据的数都被证明来自同一份口径,才自动出具。
|
||||
- 分析结论里的数(占比、变化、波动范围)是从查询结果派生的,不比对、不标口径,摘要行也不计它们。
|
||||
|
||||
写口径的取数 SQL 时,最好不要取整(`ROUND(SUM(x), 2)`),也不要套 `COALESCE(…, 0)`。取整和缺省值交给口径卡的小数位和展示;否则问数据里没取整的写法永远认不成等价(差在哪会照实写出来)。
|
||||
写口径的取数 SQL 时,不要取整(`ROUND(SUM(x), 2)`),也不要套 `COALESCE(…, 0)`。取整和缺省值交给口径卡的小数位和展示;否则问数据里没取整的写法永远认不成等价(差在哪会照实写出来)。受管级别发布时,这两类写法会直接挡住发布;已发布级别只提示。
|
||||
|
||||
### 按口径正式出具
|
||||
|
||||
|
||||
@@ -67,7 +67,7 @@ export const PAGES: PageDef[] = [
|
||||
// 评测(评测-D8):搭建者以上可见(使用者的页面表在 visiblePages 里收窄,用不着另判),同样排在设置前面;快捷键用最后一个空着的 ⌥0
|
||||
{ to: '/evals', label: EVAL_TEXT.nav, hint: EVAL_TEXT.hint, icon: Gauge, shortcut: 'Alt+0', keywords: 'evals eval evaluation 评测 回归 题集 准确率 A/B 口径命中 重复度' },
|
||||
// 口径目录:已发布口径的入口、依赖和试跑。后加的页,快捷键用没占用的 Alt+L
|
||||
{ to: '/ledger/calibers', label: '口径', hint: '查看已发布口径的入口、依赖,并按参数试跑', icon: Layers, shortcut: 'Alt+L', keywords: 'caliber ledger 口径 指标 试跑 入口' },
|
||||
{ to: '/ledger/calibers', label: '口径', hint: '查看已发布口径的入口、依赖,并按参数试运行', icon: Layers, shortcut: 'Alt+L', keywords: 'caliber ledger 口径 指标 试运行 入口' },
|
||||
{ to: '/settings', label: '设置', hint: '模型接入、偏好、运行环境', icon: Settings, shortcut: 'Alt+7', keywords: 'settings provider 模型 署名 主题 偏好' },
|
||||
]
|
||||
|
||||
|
||||
@@ -5482,6 +5482,7 @@ export const ORIGIN_LABEL: Record<OriginKind, string> = {
|
||||
ledger_issue: '按口径正式出具',
|
||||
reconcile: '口径对账',
|
||||
recall: '口径召回',
|
||||
caliber_trial: '口径试运行',
|
||||
}
|
||||
|
||||
export const ORIGIN_TEXT = {
|
||||
|
||||
@@ -153,12 +153,16 @@ export const LEDGER_UI = {
|
||||
pending: '正在比对口径…',
|
||||
timeout: '比对口径还没有结果',
|
||||
/** 一行摘要:几个数来自哪份口径、几个是这次的推理、几个没有比对 */
|
||||
summary: (parts: { caliber: number; calibers: string[]; reasoning: number; related: number; unknown: number }) => {
|
||||
summary: (parts: { caliber: number; calibers: string[]; reasoning: number; related: number; unknown: number;
|
||||
external?: number; derived?: number; assessed?: number }) => {
|
||||
const out: string[] = []
|
||||
if (parts.caliber) {
|
||||
out.push(parts.calibers.length === 1 ? `${formatNumber(parts.caliber)} 个数来自口径${parts.calibers[0]}`
|
||||
: `${formatNumber(parts.caliber)} 个数来自 ${formatNumber(parts.calibers.length)} 份口径`)
|
||||
}
|
||||
if (parts.external) out.push(`${formatNumber(parts.external)} 个数是外部常量`)
|
||||
if (parts.derived) out.push(`${formatNumber(parts.derived)} 个数是规则派生`)
|
||||
if (parts.assessed) out.push(`${formatNumber(parts.assessed)} 个数是评估值`)
|
||||
const reasoning = parts.reasoning + parts.related
|
||||
if (reasoning) {
|
||||
out.push(`${formatNumber(reasoning)} 个数是这次的推理${parts.related ? `(其中 ${formatNumber(parts.related)} 个有相关口径)` : ''}`)
|
||||
|
||||
@@ -65,9 +65,9 @@ export function CaliberCatalog() {
|
||||
try {
|
||||
const out = await api.caliberLedger.trial(d.workflow_id, d.version, d.card.node_id, values[key] ?? {})
|
||||
setTrials((t) => ({ ...t, [key]: out }))
|
||||
if (out.status === 'succeeded') toast.ok('试跑完成')
|
||||
else if (out.status === 'running') toast.warn('试跑还在运行,打开运行记录看结果')
|
||||
else toast.error(out.error || '试跑没有成功')
|
||||
if (out.status === 'succeeded') toast.ok('试运行完成')
|
||||
else if (out.status === 'running') toast.warn('试运行还在进行,打开运行记录看结果')
|
||||
else toast.error(out.error || '试运行没有成功')
|
||||
} catch (e) {
|
||||
toast.error(e)
|
||||
} finally {
|
||||
@@ -77,7 +77,7 @@ export function CaliberCatalog() {
|
||||
|
||||
return (
|
||||
<div className="flex h-full flex-col" data-caliber-catalog="">
|
||||
<PageHeader icon={<Layers size={14} />} title="口径" subtitle="已发布口径的入口、指标、依赖与试跑"
|
||||
<PageHeader icon={<Layers size={14} />} title="口径" subtitle="已发布口径的入口、指标、依赖与试运行"
|
||||
actions={<button type="button" className="btn btn-xs btn-ghost" onClick={load} title="重新加载">
|
||||
<RefreshCw size={12} aria-hidden /> 刷新
|
||||
</button>} />
|
||||
@@ -164,17 +164,24 @@ export function CaliberCatalog() {
|
||||
<h3 className="mb-1 text-2xs font-semibold text-dim">入口字段</h3>
|
||||
{d.entry_schema.properties && Object.keys(d.entry_schema.properties).length > 0 ? (
|
||||
<SchemaForm schema={d.entry_schema} value={values[key] ?? {}}
|
||||
onChange={(v) => setValues((all) => ({ ...all, [key]: v }))}
|
||||
onChange={(v) => {
|
||||
setValues((all) => ({ ...all, [key]: v }))
|
||||
setTrials((all) => {
|
||||
const next = { ...all }
|
||||
delete next[key]
|
||||
return next
|
||||
})
|
||||
}}
|
||||
idPrefix={`caliber-${key}`} />
|
||||
) : <p className="text-2xs text-faint">这份口径不需要参数。</p>}
|
||||
</div>
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<button type="button" className="btn btn-primary btn-sm" disabled={busy === key}
|
||||
onClick={() => void trial(d)} data-caliber-trial="">
|
||||
{busy === key ? <Spinner size={12} /> : <Play size={12} aria-hidden />} 试跑
|
||||
{busy === key ? <Spinner size={12} /> : <Play size={12} aria-hidden />} 试运行
|
||||
</button>
|
||||
<Link to={`/studio/${d.workflow_id}`} className="btn btn-sm btn-ghost">打开工作流</Link>
|
||||
<span className="text-2xs text-faint">试跑是只读探索运行,不是正式出具</span>
|
||||
<span className="text-2xs text-faint">试运行是只读探索运行,不是正式出具</span>
|
||||
</div>
|
||||
{result && (
|
||||
<div className="rounded-md border bg-elev px-3 py-2 text-xs" data-caliber-trial-result={result.status}>
|
||||
@@ -189,7 +196,7 @@ export function CaliberCatalog() {
|
||||
</ul>
|
||||
) : (
|
||||
<Notice tone={result.status === 'running' ? 'info' : 'err'}>
|
||||
{result.status === 'running' ? '还在运行:' : '试跑没有成功:'}
|
||||
{result.status === 'running' ? '还在运行:' : '试运行没有成功:'}
|
||||
{result.error || result.status}
|
||||
{' '}<Link className="underline" to={`/runs/${result.run_id}`}>看运行</Link>
|
||||
</Notice>
|
||||
|
||||
@@ -364,7 +364,7 @@ export function LedgerPart({ runId, runClass, report, segId, segText, segLedger,
|
||||
<p className="text-2xs text-dim">
|
||||
数由规则从查询结果算出,不是某条查询与口径等价;输入仍然逐格可追溯。
|
||||
</p>
|
||||
{label.rule?.all?.length ? <p className="text-2xs text-faint">条件:{label.rule.all.length} 条</p> : null}
|
||||
{label.rule?.conditions ? <p className="text-2xs text-faint">条件:{label.rule.conditions} 条</p> : null}
|
||||
</div>
|
||||
)}
|
||||
{label.kind === 'assessed' && (
|
||||
|
||||
@@ -21,8 +21,8 @@ import { ReconcileDialog, ReconcileResults } from './LedgerReconcile'
|
||||
* 能不能自动、不能的原因都是服务端给的;摘要只是把主报告里服务端标好的数按标签数一数、写出是哪份口径。
|
||||
*/
|
||||
|
||||
/** 摘要认的四种标签。服务端不给别的标签(分析结论的数不和口径比对,00-总方案 D11),以后多了也不计、不出 NaN */
|
||||
const COUNTED_KINDS = ['caliber', 'related', 'reasoning', 'unknown'] as const
|
||||
/** 摘要认的标签。分析结论的数不和口径比对(00-总方案 D11),但外部常量、规则派生、评估值要照实计数 */
|
||||
const COUNTED_KINDS = ['caliber', 'external', 'derived', 'assessed', 'related', 'reasoning', 'unknown'] as const
|
||||
type CountedKind = (typeof COUNTED_KINDS)[number]
|
||||
const counted = (kind: unknown): kind is CountedKind => (COUNTED_KINDS as readonly unknown[]).includes(kind)
|
||||
|
||||
@@ -33,7 +33,9 @@ const counted = (kind: unknown): kind is CountedKind => (COUNTED_KINDS as readon
|
||||
function summaryOf(data: RunLedger) {
|
||||
const labels = Object.values((data.primary ? data.reports?.[data.primary]?.labels : undefined) ?? {})
|
||||
.filter((l) => !!l && typeof l === 'object' && counted(l.kind))
|
||||
const counts: Record<CountedKind, number> = { caliber: 0, related: 0, reasoning: 0, unknown: 0 }
|
||||
const counts: Record<CountedKind, number> = {
|
||||
caliber: 0, external: 0, derived: 0, assessed: 0, related: 0, reasoning: 0, unknown: 0,
|
||||
}
|
||||
const calibers: string[] = []
|
||||
for (const l of labels) {
|
||||
counts[l.kind as CountedKind] += 1
|
||||
|
||||
@@ -822,7 +822,7 @@ export interface NotifyEventRow {
|
||||
export type OriginKind =
|
||||
| 'chat' | 'canvas' | 'launcher' | 'schedule' | 'app_entry' | 'app_view' | 'app_backfill' | 'hook' | 'api'
|
||||
| 'trial' | 'personal_app'
|
||||
| 'eval' | 'ledger_issue' | 'reconcile' | 'recall'
|
||||
| 'eval' | 'ledger_issue' | 'reconcile' | 'recall' | 'caliber_trial'
|
||||
|
||||
/**
|
||||
* GET /api/runs/unlisted:同样的筛选下,记录页因为「系统派生的运行默认不列」而没列出的次数。系统派生的是评测、口径对账、
|
||||
|
||||
@@ -58,8 +58,13 @@ export interface LedgerLabel {
|
||||
source?: string
|
||||
as_of?: string
|
||||
stale?: boolean
|
||||
rule?: { inputs?: Record<string, string>; all?: unknown[]; then?: number; else?: number }
|
||||
assessment?: { score?: number | null; level?: string | null; dimensions?: unknown[]; levels?: unknown[] }
|
||||
rule?: { inputs?: string[]; conditions?: number; then?: number; else?: number }
|
||||
assessment?: {
|
||||
score?: number | null
|
||||
level?: string | null
|
||||
dimensions?: { name?: string; weight?: number }[]
|
||||
levels?: { name?: string; min?: number }[]
|
||||
}
|
||||
caliber?: string
|
||||
workflow_id?: string
|
||||
version?: number
|
||||
@@ -385,7 +390,7 @@ export interface CaliberMetric {
|
||||
unit?: string
|
||||
decimals?: number
|
||||
format?: string
|
||||
/** 公式(formula);规则、评估、外部常量还没实现 */
|
||||
/** 公式(formula)、外部常量(external)、单元格条件规则(rule)、评估(assessment) */
|
||||
kind?: string
|
||||
/** 定义卡:文字定义、范围、排除项、负责人、生效日期、引用 */
|
||||
definition?: { text?: string; scope?: string; exclusions?: string[]; owner?: string; effective_from?: string; citations?: unknown[] }
|
||||
|
||||
@@ -147,11 +147,11 @@ await section('导航', '=== 导航 ===', async () => {
|
||||
const labels = await nav(page).locator('li > a[aria-keyshortcuts] > span:not([aria-hidden])').allInnerTexts()
|
||||
// 定时任务是后加的,排在设置前面;设置留在最底下,快捷键还是 ⌥7(定时任务用 ⌥8)。应用(⌥9)排在编排后面。
|
||||
// 评测(next-round 评测-D8,搭建者以上)也排在设置前面,用最后一个空着的 ⌥0
|
||||
check('导航十项:问数据、编排、应用、记录、数据、工具、知识、定时任务、评测、设置',
|
||||
labels.join(',') === '问数据,编排,应用,记录,数据,工具,知识,定时任务,评测,设置', labels.join(','))
|
||||
check('导航十一项:问数据、编排、应用、记录、数据、工具、知识、定时任务、评测、口径、设置',
|
||||
labels.join(',') === '问数据,编排,应用,记录,数据,工具,知识,定时任务,评测,口径,设置', labels.join(','))
|
||||
const keys = await nav(page).locator('li > a[aria-keyshortcuts]').evaluateAll((els) => els.map((el) => el.getAttribute('aria-keyshortcuts')))
|
||||
check('原有快捷键都没动:问数据 ⌥1、编排 ⌥2、记录 ⌥3 … 设置 ⌥7、定时任务 ⌥8;应用是 ⌥9、评测是 ⌥0',
|
||||
keys.join(',') === 'Alt+1,Alt+2,Alt+9,Alt+3,Alt+4,Alt+5,Alt+6,Alt+8,Alt+0,Alt+7', keys.join(','))
|
||||
check('原有快捷键都没动:问数据 ⌥1、编排 ⌥2、记录 ⌥3 … 设置 ⌥7、定时任务 ⌥8;应用是 ⌥9、评测是 ⌥0;口径是 ⌥L',
|
||||
keys.join(',') === 'Alt+1,Alt+2,Alt+9,Alt+3,Alt+4,Alt+5,Alt+6,Alt+8,Alt+0,Alt+L,Alt+7', keys.join(','))
|
||||
const fontSize = await nav(page).locator('li > a[aria-keyshortcuts] > span:not([aria-hidden])').first()
|
||||
.evaluate((el) => getComputedStyle(el).fontSize)
|
||||
check('导航标签 11px', fontSize === '11px', fontSize)
|
||||
|
||||
Reference in New Issue
Block a user