业务本体与标准实体:跨源键值对齐、对账共变算子、健康证据链与实体助手

- 新增标准业务实体(CanonicalEntity)模型、确定性跨源编码翻译、后台候选实体扫描与文档驱动实体智能助手
- 新增指标本体图谱(factor / reconciles_with / upstream_driver)、DAG 防环校验以及 ontology_neighbors / source_health 内置工具
- 新增跨源对账(analysis_reconcile)与滞后共变(analysis_cochange)分析算子,打通 [[h:H1]] 数据源健康证据链、报告导出与本体消融评测
- 数据目录新增「标准业务实体」视图与候选审阅工作台,口径页集成指标本体拓扑面板与端到端浏览器检查
This commit is contained in:
ThearchyHelios
2026-11-12 12:57:53 +08:00
parent 2f90f0eb3a
commit 8d9a712eae
106 changed files with 16654 additions and 295 deletions
+3 -1
View File
@@ -74,6 +74,8 @@ class TurnContext:
analysis: Mapping[str, Any] | None = None
#: 变体:分析工具开不开(variants.analysis_tools)。关的那一臂建图时助手看不到分析工具(TurnDriver 的 analysis_tools)
analysis_tools: bool = True
#: 变体:本体穿透开不开(variants.ontology_tools)。关的那一臂建图时助手看不到本体工具与邻居提示
ontology_tools: bool = True
async def writer_usage(run_ids: list[str]) -> dict[str, Any]:
@@ -163,7 +165,7 @@ class Driver(TurnDriver):
on_phase: Callable[[str], Awaitable[None]] | None = None) -> None:
super().__init__(question, datasource_ids=datasource_ids, on_phase=on_phase,
provider=ctx.assistant.get("provider"), model=ctx.assistant.get("model"),
analysis_tools=ctx.analysis_tools)
analysis_tools=ctx.analysis_tools, ontology_tools=ctx.ontology_tools)
self.ctx = ctx
def catalog_view(self) -> Any:
+14 -3
View File
@@ -64,6 +64,8 @@ GOLD_KINDS = ("sql", "caliber", "from_run")
HUMAN_VALUE_KEYS = ("value", "values", "expected", "answer", "rows", "number", "segment", "segments", "subjects",
"direction", "verdict", "judgement", "driver")
DIAGNOSIS_STAGES = ("health_and_contract", "slice_localization", "ontology_penetration")
#: 每种期望认的键(另有 id、kind、required、label)
_EXPECTATION_KEYS: dict[str, tuple[str, ...]] = {
"scalar": ("column", "metric", "row", "where", "tolerance"),
@@ -74,7 +76,8 @@ _EXPECTATION_KEYS: dict[str, tuple[str, ...]] = {
"text": ("points",),
"contribution": ("op", "mode", "params", "top_n"),
"verdict": ("op", "mode", "params"),
"investigation": ("minimum_depth", "max_queries", "status", "stop_reason"),
"investigation": ("minimum_depth", "max_queries", "status", "stop_reason",
"min_ontology_hops", "require_source_health", "diagnosis_stage"),
}
_COMMON_KEYS = ("id", "kind", "required", "label")
#: 期望答案来源的键(客户端能写的);from_run 只能由服务端建
@@ -292,12 +295,20 @@ def _expectation(raw: Any, i: int, *, gold_kind: str | None, taken: set[str]) ->
if status not in ("complete", "partial"):
raise CaseInvalid("调查期望 status 只能是 complete 或 partial", f"{path}.status")
out["status"] = status
for key, low, high in (("minimum_depth", 0, 5), ("max_queries", 1, 1000)):
if key in raw:
for key, low, high in (("minimum_depth", 0, 5), ("max_queries", 1, 1000), ("min_ontology_hops", 0, 10)):
if key in raw and raw[key] is not None:
value = raw[key]
if isinstance(value, bool) or not isinstance(value, int) or not low <= value <= high:
raise CaseInvalid(f"{key} 需要是 {low} 到 {high} 的整数", f"{path}.{key}")
out[key] = value
if raw.get("require_source_health") is not None:
out["require_source_health"] = _bool(raw["require_source_health"], f"{where}的关联源健康检查要求",
f"{path}.require_source_health")
if raw.get("diagnosis_stage") is not None:
stage = raw["diagnosis_stage"]
if stage not in DIAGNOSIS_STAGES:
raise CaseInvalid(f"诊断阶段只能是 {'、'.join(DIAGNOSIS_STAGES)}", f"{path}.diagnosis_stage")
out["diagnosis_stage"] = stage
if raw.get("stop_reason") is not None:
reason = raw["stop_reason"]
if not isinstance(reason, str) or not reason.strip() or len(reason) > 80:
+10 -8
View File
@@ -31,10 +31,10 @@ CAUSES = ("data", "method", "params", "path", "unknown")
CAUSE_LABEL = {"data": "数据变", "method": "做法变", "params": "只参数变", "path": "路径变", "unknown": "说不清"}
#: 指纹里比的几项和叫法
FINGERPRINT_LABEL = {"assistant_model": "助手模型", "writer_model": "写作模型", "catalog": "目录快照",
"analysis_tools": "分析工具", "prompt": "提示词", "schemas": "表结构", "cases": "题目",
"scorer_version": "评分规则", "analysis": "分析设置"}
"analysis_tools": "分析工具", "ontology_tools": "本体穿透", "prompt": "提示词", "schemas": "表结构",
"cases": "题目", "scorer_version": "评分规则", "analysis": "分析设置"}
#: 指纹里照实写前后两边的几项(别的只说变没变)
_SHOWN = ("assistant_model", "writer_model", "catalog", "analysis_tools", "scorer_version")
_SHOWN = ("assistant_model", "writer_model", "catalog", "analysis_tools", "ontology_tools", "scorer_version")
NO_DIFFERENCE = "这批题上看不出差别"
SIGNIFICANT = 0.05
#: 基准能是哪几种状态
@@ -130,18 +130,20 @@ def transitions(base: Sequence[Mapping[str, Any]], target: Sequence[Mapping[str,
return {"counts": counts, "causes": causes, "items": items}
def _tools(fingerprint: Mapping[str, Any]) -> str:
"""分析工具这一项给人看的样子:开 / 关。加这一项之前的评测指纹里没有它,那时一直开着。"""
def _tools(fingerprint: Mapping[str, Any], key: str = "analysis_tools") -> str:
"""分析工具、本体穿透这一项给人看的样子:开 / 关。加这一项之前的评测指纹里没有它,那时一直开着。"""
from app.eval.variants import ANALYSIS_TOOLS_LABEL
return ANALYSIS_TOOLS_LABEL["off" if fingerprint.get("analysis_tools") == "off" else "on"]
return ANALYSIS_TOOLS_LABEL["off" if fingerprint.get(key) == "off" else "on"]
def fingerprint_diff(base: Mapping[str, Any], target: Mapping[str, Any]) -> list[dict[str, Any]]:
"""两次之间指纹里哪几项变了:[{key, label, before, after}](表结构、题目只说变没变,不展开;分析工具写开、关)。"""
"""两次之间指纹里哪几项变了:[{key, label, before, after}](表结构、题目只说变没变,不展开;分析工具、本体穿透写开、关)。"""
out = []
for key, label in FINGERPRINT_LABEL.items():
a, b = (_tools(base), _tools(target)) if key == "analysis_tools" else (base.get(key), target.get(key))
a, b = ((_tools(base, key), _tools(target, key))
if key in ("analysis_tools", "ontology_tools")
else (base.get(key), target.get(key)))
if a == b:
continue
item: dict[str, Any] = {"key": key, "label": label}
+7 -3
View File
@@ -43,11 +43,11 @@ from app.analysis.contract import OP_LABELS, method_label
#: 两种分析期望
ANALYSIS_KINDS = ("contribution", "verdict")
#: 判定期望能用的分析(贡献恒为变化拆解)
VERDICT_OPS = ("baseline", "trend", "differ", "comparable")
#: 判定类,见模块说明。前六个是贡献的,后九个是判定的
VERDICT_OPS = ("baseline", "trend", "differ", "comparable", "reconcile", "cochange")
#: 判定类,见模块说明。前六个是贡献的,其余是判定的
JUDGEMENTS = ("source", "dispersed", "flat", "mix", "rate", "mixed",
"normal", "anomaly", "rising", "falling", "no_trend", "differs", "no_evidence", "comparable",
"incomparable")
"incomparable", "aligned", "diverged", "co_anomalous", "target_normal")
#: 分析的输入在算子里的名字(报错里照它说:「期望 SQL 里没有『地区』这一列」)
GOLD_REF = "期望 SQL"
#: 贡献期望最多点名前几段
@@ -66,6 +66,8 @@ _JUDGEMENT_OF: dict[tuple[str, str | None], dict[str, str]] = {
("trend", None): {"rising": "rising", "falling": "falling", "no_trend": "no_trend"},
("differ", None): {"differs": "differs", "no_evidence": "no_evidence"},
("comparable", None): {"comparable": "comparable", "conditional": "comparable", "incomparable": "incomparable"},
("reconcile", None): {"aligned": "aligned", "diverged": "diverged", "missing_counterpart": "diverged"},
("cochange", None): {"co_anomalous": "co_anomalous", "target_normal": "target_normal", "primary_normal": "target_normal"},
}
#: 判定类要看方向的:主要来源、结构或自身为主、分散(总变化的涨跌)、超出常规波动(本期变化的涨跌)、差异站得住(A 相对 B)
_DIRECTED = frozenset({"source", "mix", "rate", "mixed", "dispersed", "anomaly", "differs"})
@@ -79,6 +81,8 @@ _AFFIRMS: dict[str, frozenset[str]] = {
"anomaly": frozenset({"anomaly"}), "rising": frozenset({"trend"}), "falling": frozenset({"trend"}),
"no_trend": frozenset({"no_trend"}), "differs": frozenset({"differ"}), "no_evidence": frozenset({"no_diff"}),
"comparable": frozenset(), "incomparable": frozenset(),
"aligned": frozenset(), "diverged": frozenset(),
"co_anomalous": frozenset({"anomaly"}), "target_normal": frozenset({"normal"}),
}
#: 判断词所在的句子最多留多少字给人看
_SNIPPET_MAX = 40
+6 -2
View File
@@ -110,12 +110,14 @@ def prompt_fingerprint() -> str:
from app.engine import analysis_claims, evidence, review
from app.engine.investigation import INVESTIGATION_VERSION, InvestigationPolicy
from app.tools.builtin.analysis import METHOD_GUIDE
from app.tools.ontology import ONTOLOGY_GUIDE
parts = [copilot.NODE_REFERENCE, copilot._STREAM_PROTOCOL,
copilot._user_message(copilot.GenerateIn(instruction="{question}", intent="answer"), patch=False),
turn.RETRY_WITH_BASE, turn.RETRY_FROM_SCRATCH,
str(getattr(review, "REVIEW_SYSTEM", "") or getattr(review, "SYSTEM_PROMPT", "")),
METHOD_GUIDE.id, METHOD_GUIDE.text, evidence.ANALYSIS_PROMPT, evidence.ANALYSIS_SEE_ONLY_PROMPT,
METHOD_GUIDE.id, METHOD_GUIDE.text, ONTOLOGY_GUIDE.id, ONTOLOGY_GUIDE.text,
evidence.ANALYSIS_PROMPT, evidence.ANALYSIS_SEE_ONLY_PROMPT,
analysis_claims.ANALYSIS_RULES_PROMPT, analysis_claims.ANALYSIS_NONE_PROMPT,
analysis_claims.ANALYSIS_REPAIR_PROMPT, analysis_claims.LEXICON_VERSION,
INTENT_VERSION, analysis_gaps_fix(), INVESTIGATION_VERSION, InvestigationPolicy().model_dump()]
@@ -245,6 +247,7 @@ async def start(session: AsyncSession, suite: EvalSuite, variants: Sequence[Mapp
"assistant_model": models.get("assistant"), "writer_model": models.get("writer"),
"catalog_mode": (variant.get("catalog") or {}).get("mode"),
"analysis_tools": variant_rules.analysis_tools(variant),
"ontology_tools": variant_rules.ontology_tools(variant),
"snapshot_artifact": snapshot, "cron": cron},
counts={"total": len(cases) * repeats, "done": 0})
session.add(run)
@@ -554,7 +557,8 @@ class EvalExecution(runtime.JobRun):
arm=arm.name, repeat=unit.repeat,
assistant=arm.variant.get("assistant") or {}, writer=arm.variant.get("writer") or {},
view=arm.view, analysis=self.analysis,
analysis_tools=variant_rules.analysis_tools(arm.variant) == "on")
analysis_tools=variant_rules.analysis_tools(arm.variant) == "on",
ontology_tools=variant_rules.ontology_tools(arm.variant) == "on")
driver = Driver(question, datasource_ids=list(case.get("datasource_ids") or []), ctx=ctx, on_phase=on_phase)
try:
if case.get("kind") == "workflow":
+31 -2
View File
@@ -187,12 +187,14 @@ class Answer:
analyses: list[dict[str, Any]] = field(default_factory=list)
sentences: list[dict[str, Any]] = field(default_factory=list)
investigations: list[dict[str, Any]] = field(default_factory=list)
source_healths: list[dict[str, Any]] = field(default_factory=list)
def to_dict(self) -> dict[str, Any]:
return {"run_id": self.run_id, "text": self.text, "values": self.values, "bare": self.bare,
"queries": self.queries, "cited": self.cited, "stats": self.stats, "tier": self.tier,
"report": self.report, "problem": self.problem, "analyses": self.analyses,
"sentences": self.sentences, "investigations": self.investigations}
"sentences": self.sentences, "investigations": self.investigations,
"source_healths": self.source_healths}
@classmethod
def from_dict(cls, raw: Mapping[str, Any]) -> Answer:
@@ -200,7 +202,8 @@ class Answer:
bare=list(raw.get("bare") or []), queries=list(raw.get("queries") or []),
cited=list(raw.get("cited") or []), stats=dict(raw.get("stats") or {}), tier=raw.get("tier"),
report=raw.get("report"), problem=raw.get("problem"), analyses=list(raw.get("analyses") or []),
sentences=list(raw.get("sentences") or []), investigations=list(raw.get("investigations") or []))
sentences=list(raw.get("sentences") or []), investigations=list(raw.get("investigations") or []),
source_healths=list(raw.get("source_healths") or []))
def _output_text(output: Any) -> str:
@@ -308,6 +311,21 @@ async def read_answer(run_id: str | None, *, reply: str | None = None) -> Answer
"row_count": snap.get("row_count", len(rows)), "truncated": bool(snap.get("truncated")),
"role": snap.get("role")})
found = _ledger_analyses(sealed, cache)
for artifact, info in (getattr(sealed, "source_healths", None) or {}).items():
snap = _snapshot(artifact, cache)
if snap is None:
continue
answer.source_healths.append({
"artifact": artifact,
"node_id": (info or {}).get("node_id"),
"verdict": snap.get("verdict") or (info or {}).get("verdict"),
"sources": list((info or {}).get("sources") or [
s.get("source") or s.get("source_label")
for s in (snap.get("sources") or [])
if isinstance(s, Mapping)
]),
"phrase": snap.get("phrase") or "",
})
from app.db.base import SessionLocal
from app.db.models import Run
@@ -636,6 +654,17 @@ def _expectation_scores(expectations: Sequence[Mapping[str, Any]], values: Mappi
problems.append("未达到要求的下钻深度")
if exp.get("stop_reason") and finding.get("stop_reason") != exp["stop_reason"]:
problems.append("停止原因不符合期望")
if exp.get("min_ontology_hops") is not None:
best_hops = max((int(f.get("ontology_hops") or 0) for f in findings), default=0)
if best_hops < int(exp["min_ontology_hops"]):
problems.append("未达到要求的本体穿透跳数")
if exp.get("require_source_health"):
checked = any(bool(f.get("source_health_checked")) for f in findings) or bool(answer.source_healths)
if not checked:
problems.append("未执行关联数据源健康度检查")
if exp.get("diagnosis_stage") is not None:
if not findings or not any(f.get("diagnosis_stage") == exp["diagnosis_stage"] for f in findings):
problems.append("诊断阶段不符合期望")
if exp.get("max_queries") is not None and sum(int(f.get("query_count") or 0) for f in findings) > exp["max_queries"]:
problems.append("查询次数超出预算")
out.append({**base, "outcome": "wrong" if problems else "traced",
+31 -7
View File
@@ -45,6 +45,9 @@ CATALOG_LABEL = {"live": "当前目录", "reviewed": "只用已核实和已确
#: 分析工具开、关(变体的 analysis.tools)
ANALYSIS_TOOLS = ("on", "off")
ANALYSIS_TOOLS_LABEL = {"on": "开", "off": "关"}
#: 本体穿透开、关(变体的 ontology.tools)
ONTOLOGY_TOOLS = ("on", "off")
ONTOLOGY_TOOLS_LABEL = {"on": "开", "off": "关"}
REPEATS_MAX = 5
#: 定时跑 1 次,A/B 对照缺省 3 次(评测-D10)
SCHEDULE_REPEATS = 1
@@ -52,7 +55,7 @@ PAIR_REPEATS = 3
#: 一次 A/B 最多几臂(目录贡献对照最多三臂)
ARMS_MAX = 3
LABEL_MAX = 40
_KEYS = ("assistant", "writer", "judge", "catalog", "analysis", "repeats", "label")
_KEYS = ("assistant", "writer", "judge", "catalog", "analysis", "ontology", "repeats", "label")
def _invalid(message: str, field_name: str) -> CaseInvalid:
@@ -104,6 +107,10 @@ def normalize_variant(raw: Any, *, judge_default: bool = False, repeats_default:
if not isinstance(analysis, Mapping) or set(analysis) - {"tools"} \
or analysis.get("tools", "on") not in ANALYSIS_TOOLS:
raise _invalid(f"变体的分析工具需要是 {{tools}},tools 只能是 {'、'.join(ANALYSIS_TOOLS)}(开、关)", "variant.analysis")
ontology = {} if raw.get("ontology") is None else raw["ontology"]
if not isinstance(ontology, Mapping) or set(ontology) - {"tools"} \
or ontology.get("tools", "on") not in ONTOLOGY_TOOLS:
raise _invalid(f"变体的本体穿透需要是 {{tools}},tools 只能是 {'、'.join(ONTOLOGY_TOOLS)}(开、关)", "variant.ontology")
repeats = raw.get("repeats", repeats_default)
if isinstance(repeats, bool) or not isinstance(repeats, int) or not 1 <= repeats <= REPEATS_MAX:
raise _invalid(f"每题重复的次数需要是 1 到 {REPEATS_MAX} 的整数", "variant.repeats")
@@ -113,7 +120,8 @@ def normalize_variant(raw: Any, *, judge_default: bool = False, repeats_default:
return {"assistant": _model_pair(raw.get("assistant"), "助手模型", "variant.assistant"),
"writer": _model_pair(raw.get("writer"), "写作模型", "variant.writer"),
"judge": judge, "catalog": {"mode": catalog["mode"], "snapshot": snapshot},
"analysis": {"tools": analysis.get("tools", "on")}, "repeats": repeats,
"analysis": {"tools": analysis.get("tools", "on")},
"ontology": {"tools": ontology.get("tools", "on")}, "repeats": repeats,
"label": (label or "").strip()[:LABEL_MAX] or None}
@@ -122,6 +130,11 @@ def analysis_tools(variant: Mapping[str, Any]) -> str:
return "off" if (variant.get("analysis") or {}).get("tools") == "off" else "on"
def ontology_tools(variant: Mapping[str, Any]) -> str:
"""这一臂的本体穿透开不开:on / off。加这一项之前的变体(存着的老评测)没有它,按开。"""
return "off" if (variant.get("ontology") or {}).get("tools") == "off" else "on"
def catalog_preset(base: Mapping[str, Any], *, reviewed: bool = False) -> list[dict[str, Any]]:
"""「目录的贡献」预设对照:当前目录 vs 关(reviewed 时再加「只用已核实和已确认的项」)。模型、裁判、重复次数照 base。"""
arms = [("live", CATALOG_LABEL["live"]), ("off", CATALOG_LABEL["off"])]
@@ -137,6 +150,12 @@ def analysis_tools_preset(base: Mapping[str, Any]) -> list[dict[str, Any]]:
for tools in ANALYSIS_TOOLS]
def ontology_tools_preset(base: Mapping[str, Any]) -> list[dict[str, Any]]:
"""「本体穿透的贡献」预设对照:本体穿透开 vs 关。模型、目录、分析工具、裁判、重复次数照 base。"""
return [{**copy.deepcopy(dict(base)), "ontology": {"tools": tools}, "label": f"本体穿透{ONTOLOGY_TOOLS_LABEL[tools]}"}
for tools in ONTOLOGY_TOOLS]
def arm_name(variant: Mapping[str, Any], index: int) -> str:
"""这一臂的名字:变体里写了就用,否则按目录(只有目录不同时一看就懂),再不然 A、B、C。"""
return str(variant.get("label") or "") or "ABC"[index]
@@ -221,6 +240,8 @@ def variant_key(variant: Mapping[str, Any], models: Mapping[str, Any]) -> str:
"snapshot": catalog.get("snapshot") if catalog.get("mode") == "snapshot" else None}
if analysis_tools(variant) == "off":
body["analysis_tools"] = "off"
if ontology_tools(variant) == "off":
body["ontology_tools"] = "off"
return content_hash(canonical_json(body))[:16]
@@ -231,12 +252,15 @@ def describe(variant: Mapping[str, Any], models: Mapping[str, Any] | None = None
parts = [f"助手 {models.get('assistant') or (variant.get('assistant') or {}).get('model') or '缺省'}",
f"写作 {models.get('writer') or (variant.get('writer') or {}).get('model') or '各节点自己的'}",
CATALOG_LABEL.get(catalog.get("mode") or "live", "当前目录"),
f"分析工具{ANALYSIS_TOOLS_LABEL[analysis_tools(variant)]}",
f"结论裁判{'开' if variant.get('judge') else '关'}", f"每题 {variant.get('repeats') or 1} 次"]
f"分析工具{ANALYSIS_TOOLS_LABEL[analysis_tools(variant)]}"]
if ontology_tools(variant) == "off":
parts.append(f"本体穿透{ONTOLOGY_TOOLS_LABEL['off']}")
parts += [f"结论裁判{'开' if variant.get('judge') else '关'}", f"每题 {variant.get('repeats') or 1} 次"]
return " · ".join(parts)
__all__ = ["ANALYSIS_TOOLS", "ANALYSIS_TOOLS_LABEL", "ARMS_MAX", "CATALOG_LABEL", "CATALOG_MODES", "PAIR_REPEATS",
__all__ = ["ANALYSIS_TOOLS", "ANALYSIS_TOOLS_LABEL", "ARMS_MAX", "CATALOG_LABEL", "CATALOG_MODES",
"ONTOLOGY_TOOLS", "ONTOLOGY_TOOLS_LABEL", "PAIR_REPEATS",
"REPEATS_MAX", "Resolved", "SCHEDULE_REPEATS", "VARIANT_INVALID", "VARIANT_UNAVAILABLE", "analysis_tools",
"analysis_tools_preset", "arm_name", "catalog_preset", "describe", "normalize_variant", "resolve_models",
"snapshot_artifact", "variant_key", "view_of"]
"analysis_tools_preset", "arm_name", "catalog_preset", "describe", "normalize_variant", "ontology_tools",
"ontology_tools_preset", "resolve_models", "snapshot_artifact", "variant_key", "view_of"]