业务本体与标准实体:跨源键值对齐、对账共变算子、健康证据链与实体助手

- 新增标准业务实体(CanonicalEntity)模型、确定性跨源编码翻译、后台候选实体扫描与文档驱动实体智能助手
- 新增指标本体图谱(factor / reconciles_with / upstream_driver)、DAG 防环校验以及 ontology_neighbors / source_health 内置工具
- 新增跨源对账(analysis_reconcile)与滞后共变(analysis_cochange)分析算子,打通 [[h:H1]] 数据源健康证据链、报告导出与本体消融评测
- 数据目录新增「标准业务实体」视图与候选审阅工作台,口径页集成指标本体拓扑面板与端到端浏览器检查
This commit is contained in:
ThearchyHelios
2026-11-12 12:57:53 +08:00
parent 2f90f0eb3a
commit 8d9a712eae
106 changed files with 16654 additions and 295 deletions
+1 -1
View File
@@ -29,7 +29,7 @@ from typing import Any
#: time_role / refund_rule 是 K04 的业务定义字段(统计时间口径、退款处理):改了定义身份就变,绑定同步状态跟着重判。
SEMANTIC_FIELDS = ("id", "name", "expression", "unit", "decimals", "format", "kind", "definition", "params", "external",
"rule", "assessment", "grain", "time_field", "numerator", "denominator", "dimensions", "filters",
"contract", "time_role", "refund_rule")
"contract", "time_role", "refund_rule", "semantic", "links")
def _blank(value: Any) -> bool:
+566
View File
@@ -0,0 +1,566 @@
"""标准业务实体(CanonicalEntity)的校验、持久化与跨源键值对齐。
核心铁律:
1. 物理表列绑定(bindings)与口径参数映射(caliber_params)分设 proposed / verified / confirmed / rejected 四态;
运行时自动穿透、跨源实体值翻译、参数传递**只认 verified / confirmed**,proposed 候选绝不自动参与穿透。
2. 跨源实体值翻译(translate_entity_value)坚持确定性映射:有标准成员表(members)时只按显式 source_codes /
标准值 / 别名精确匹配,映射不上返回 None,严禁靠子串猜测。
3. 受限视角(restricted=True,如普通 user 角色)隐去物理表名、物理列名与跨源底层编码,只暴露业务实体名与标准值。
"""
from __future__ import annotations
import re
from collections.abc import Iterable, Mapping
from typing import Any
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.base import new_id, utcnow
from app.db.models_ontology import CanonicalEntity
ENTITY_STATUSES = ("proposed", "verified", "confirmed", "rejected")
ACTIVE_ENTITY_STATUSES = ("verified", "confirmed")
ENTITY_SOURCES = ("human", "rule", "ai", "profile")
_KEY_RE = re.compile(r"^[a-z][a-z0-9_]{0,63}$")
class EntityInvalid(ValueError):
"""标准实体请求格式或取值不合法。"""
def __init__(self, message: str, *, field: str = "") -> None:
super().__init__(message)
self.message = message
self.field = field
class EntityConflict(ValueError):
"""标准实体版本冲突或唯一键冲突。"""
def __init__(self, message: str, *, field: str = "") -> None:
super().__init__(message)
self.message = message
self.field = field
class EntityNotFound(LookupError):
"""标准实体或子项不存在。"""
def _norm_token(value: Any) -> str:
return str(value or "").strip().lower()
def normalize_entity_key(raw: Any, *, field: str = "key") -> str:
if not isinstance(raw, str) or not _KEY_RE.match(raw.strip()):
raise EntityInvalid("实体标识(key)须以小写字母开头,仅含小写字母、数字和下划线(最长 64 字符)", field=field)
return raw.strip()
def _clean_str_list(raw: Any, *, field: str) -> list[str]:
if raw in (None, ""):
return []
if not isinstance(raw, list):
raise EntityInvalid(f"字段「{field}」需要是字符串列表", field=field)
seen: set[str] = set()
out: list[str] = []
for idx, item in enumerate(raw):
if not isinstance(item, str):
raise EntityInvalid(f"字段「{field}[{idx}]」需要是字符串", field=f"{field}[{idx}]")
val = item.strip()
if val and val.lower() not in seen:
seen.add(val.lower())
out.append(val)
return out
def normalize_member(raw: Any, *, index: int = 0) -> dict[str, Any]:
at = f"members[{index}]"
if not isinstance(raw, Mapping):
raise EntityInvalid("实体成员需要是对象", field=at)
value = str(raw.get("value") or "").strip()
if not value:
raise EntityInvalid("实体成员缺少标准值(value)", field=f"{at}.value")
label = str(raw.get("label") or value).strip() or value
aliases = _clean_str_list(raw.get("aliases"), field=f"{at}.aliases")
raw_codes = raw.get("source_codes")
source_codes: dict[str, str] = {}
if raw_codes not in (None, {}):
if not isinstance(raw_codes, Mapping):
raise EntityInvalid("跨源编码(source_codes)需要是「数据源名 → 编码」对象", field=f"{at}.source_codes")
for src, code in raw_codes.items():
src_key = str(src or "").strip()
code_val = str(code or "").strip()
if src_key and code_val:
source_codes[src_key] = code_val
return {"value": value, "label": label, "aliases": aliases, "source_codes": source_codes}
def normalize_binding(raw: Any, *, index: int = 0, actor: str | None = None,
force_proposed: bool = False) -> dict[str, Any]:
at = f"bindings[{index}]"
if not isinstance(raw, Mapping):
raise EntityInvalid("物理表列绑定需要是对象", field=at)
source_name = str(raw.get("source_name") or "").strip()
source_id = str(raw.get("source_id") or "").strip() or None
table_name = str(raw.get("table_name") or "").strip()
column_name = str(raw.get("column_name") or "").strip()
if not source_name:
raise EntityInvalid("物理绑定缺少数据源名称(source_name)", field=f"{at}.source_name")
if not table_name:
raise EntityInvalid("物理绑定缺少表名(table_name)", field=f"{at}.table_name")
if not column_name:
raise EntityInvalid("物理绑定缺少列名(column_name)", field=f"{at}.column_name")
source = str(raw.get("source") or "human").strip()
if source not in ENTITY_SOURCES:
raise EntityInvalid(f"绑定来源「{source}」无效,可选:{', '.join(ENTITY_SOURCES)}", field=f"{at}.source")
status = "proposed" if (force_proposed or source in ("ai", "profile") and "status" not in raw) else str(
raw.get("status") or "confirmed"
).strip()
if force_proposed:
status = "proposed"
if status not in ENTITY_STATUSES:
raise EntityInvalid(f"绑定状态「{status}」无效,可选:{', '.join(ENTITY_STATUSES)}", field=f"{at}.status")
return {
"id": str(raw.get("id") or "").strip() or new_id()[:12],
"source_id": source_id,
"source_name": source_name,
"table_name": table_name,
"column_name": column_name,
"status": status,
"source": source,
"note": str(raw.get("note") or "").strip(),
"reason": str(raw.get("reason") or "").strip(),
"updated_at": str(raw.get("updated_at") or utcnow().isoformat()),
"updated_by": str(raw.get("updated_by") or actor or "").strip() or None,
}
def normalize_caliber_param(raw: Any, *, index: int = 0, force_proposed: bool = False) -> dict[str, Any]:
at = f"caliber_params[{index}]"
if not isinstance(raw, Mapping):
raise EntityInvalid("口径参数映射需要是对象", field=at)
workflow_id = str(raw.get("workflow_id") or "").strip()
card_id = str(raw.get("card_id") or "").strip()
param_name = str(raw.get("param_name") or "").strip()
if not workflow_id or not param_name:
raise EntityInvalid("口径参数映射须写明工作流 ID(workflow_id)与参数名(param_name)", field=at)
source = str(raw.get("source") or "human").strip()
if source not in ENTITY_SOURCES:
raise EntityInvalid(f"口径参数映射来源「{source}」无效", field=f"{at}.source")
status = "proposed" if force_proposed else str(raw.get("status") or "confirmed").strip()
if status not in ENTITY_STATUSES:
raise EntityInvalid(f"口径参数映射状态「{status}」无效", field=f"{at}.status")
return {
"id": str(raw.get("id") or "").strip() or new_id()[:12],
"workflow_id": workflow_id,
"card_id": card_id,
"param_name": param_name,
"status": status,
"source": source,
"note": str(raw.get("note") or "").strip(),
}
def normalize_entity_payload(payload: Mapping[str, Any], *, partial: bool = False,
actor: str | None = None) -> dict[str, Any]:
if not isinstance(payload, Mapping):
raise EntityInvalid("请求内容需要是对象")
out: dict[str, Any] = {}
if not partial or "key" in payload:
out["key"] = normalize_entity_key(payload.get("key"))
if not partial or "label" in payload:
label = str(payload.get("label") or "").strip()
if not label:
raise EntityInvalid("实体名称(label)不能为空", field="label")
if len(label) > 100:
raise EntityInvalid("实体名称不能超过 100 个字符", field="label")
out["label"] = label
if not partial or "description" in payload:
out["description"] = str(payload.get("description") or "").strip()
if not partial or "aliases" in payload:
out["aliases"] = _clean_str_list(payload.get("aliases"), field="aliases")
if not partial or "members" in payload:
raw_members = payload.get("members") or []
if not isinstance(raw_members, list):
raise EntityInvalid("成员列表(members)需要是数组", field="members")
seen_vals: set[str] = set()
members: list[dict[str, Any]] = []
for idx, m in enumerate(raw_members):
norm = normalize_member(m, index=idx)
vkey = norm["value"].lower()
if vkey in seen_vals:
raise EntityInvalid(f"实体成员值「{norm['value']}」重复", field=f"members[{idx}].value")
seen_vals.add(vkey)
members.append(norm)
out["members"] = members
if not partial or "bindings" in payload:
raw_bindings = payload.get("bindings") or []
if not isinstance(raw_bindings, list):
raise EntityInvalid("物理绑定列表(bindings)需要是数组", field="bindings")
seen_cols: set[tuple[str, str, str]] = set()
bindings: list[dict[str, Any]] = []
for idx, b in enumerate(raw_bindings):
norm = normalize_binding(b, index=idx, actor=actor)
col_key = (norm["source_name"].lower(), norm["table_name"].lower(), norm["column_name"].lower())
if col_key in seen_cols:
raise EntityInvalid(
f"数据源「{norm['source_name']}」表「{norm['table_name']}.{norm['column_name']}」重复绑定",
field=f"bindings[{idx}]",
)
seen_cols.add(col_key)
bindings.append(norm)
out["bindings"] = bindings
if not partial or "caliber_params" in payload:
raw_params = payload.get("caliber_params") or []
if not isinstance(raw_params, list):
raise EntityInvalid("口径参数映射(caliber_params)需要是数组", field="caliber_params")
out["caliber_params"] = [normalize_caliber_param(p, index=idx) for idx, p in enumerate(raw_params)]
return out
# --------------------------------------------------------------------------
# 运行时对齐与状态过滤(只认 verified / confirmed)
# --------------------------------------------------------------------------
def _as_dict(entity: CanonicalEntity | Mapping[str, Any]) -> dict[str, Any]:
if isinstance(entity, Mapping):
return dict(entity)
return {
"id": entity.id,
"key": entity.key,
"label": entity.label,
"description": entity.description or "",
"aliases": list(entity.aliases or []),
"members": list(entity.members or []),
"bindings": list(entity.bindings or []),
"caliber_params": list(entity.caliber_params or []),
"version": int(entity.version or 1),
"created_by": entity.created_by,
"updated_by": entity.updated_by,
"created_at": entity.created_at.isoformat() if getattr(entity, "created_at", None) else None,
"updated_at": entity.updated_at.isoformat() if getattr(entity, "updated_at", None) else None,
}
def entity_to_dict(entity: CanonicalEntity | Mapping[str, Any], *, restricted: bool = False) -> dict[str, Any]:
"""序列化标准实体;受限视角(user 角色)隐去物理表名、列名、数据源内部 ID 与底层编码。"""
data = _as_dict(entity)
if not restricted:
return data
return {
"id": data["id"],
"key": data["key"],
"label": data["label"],
"description": data["description"],
"aliases": list(data.get("aliases") or []),
"members": [
{"value": m.get("value"), "label": m.get("label"), "aliases": list(m.get("aliases") or [])}
for m in (data.get("members") or []) if isinstance(m, Mapping)
],
"version": data["version"],
}
def confirmed_bindings(
entity: CanonicalEntity | Mapping[str, Any],
*,
source_id: str | None = None,
source_name: str | None = None,
table_name: str | None = None,
) -> list[dict[str, Any]]:
"""返回已生效(verified / confirmed)的物理列绑定;proposed 与 rejected 一律排除。"""
data = _as_dict(entity)
out: list[dict[str, Any]] = []
for item in data.get("bindings") or []:
if not isinstance(item, Mapping) or item.get("status") not in ACTIVE_ENTITY_STATUSES:
continue
if source_id and str(item.get("source_id") or "") != source_id:
continue
if source_name and _norm_token(item.get("source_name")) != _norm_token(source_name):
continue
if table_name and _norm_token(item.get("table_name")) != _norm_token(table_name):
continue
out.append(dict(item))
return out
def confirmed_caliber_params(
entity: CanonicalEntity | Mapping[str, Any],
*,
workflow_id: str | None = None,
card_id: str | None = None,
) -> list[dict[str, Any]]:
"""返回已生效(verified / confirmed)的口径输入参数映射。"""
data = _as_dict(entity)
out: list[dict[str, Any]] = []
for item in data.get("caliber_params") or []:
if not isinstance(item, Mapping) or item.get("status") not in ACTIVE_ENTITY_STATUSES:
continue
if workflow_id and str(item.get("workflow_id") or "") != workflow_id:
continue
if card_id and item.get("card_id") and str(item.get("card_id") or "") != card_id:
continue
out.append(dict(item))
return out
def resolve_caliber_param(
entity: CanonicalEntity | Mapping[str, Any],
*,
workflow_id: str,
card_id: str | None = None,
) -> str | None:
"""把标准业务实体解析为目标口径的入口参数名(优先匹配指定 card_id,其次匹配同工作流通用参数)。"""
matches = confirmed_caliber_params(entity, workflow_id=workflow_id)
if card_id:
for item in matches:
if str(item.get("card_id") or "") == card_id:
return str(item["param_name"])
for item in matches:
if not item.get("card_id") or card_id is None:
return str(item["param_name"])
return str(matches[0]["param_name"]) if matches else None
def translate_entity_value(
entity: CanonicalEntity | Mapping[str, Any],
raw_value: Any,
*,
from_source: str | None = None,
to_source: str | None = None,
) -> dict[str, str] | None:
"""把源侧实体值确定性地翻译成标准值与目标源编码。
- 有标准成员表(members 非空)时:严格按 from_source 的 source_codes、标准 value、label、aliases 精确匹配,
再用 to_source 的 source_codes(缺省回退标准 value)给出 target_value;找不到返回 None,绝不模糊猜测。
- 成员表为空(开放式编码域,如千万级订单号/批次号)时:仅当两侧数据源在该实体上均有已确认绑定时原样透传。
"""
probe = str(raw_value or "").strip()
if not probe:
return None
data = _as_dict(entity)
members = [m for m in (data.get("members") or []) if isinstance(m, Mapping)]
probe_norm = probe.lower()
if members:
matched: Mapping[str, Any] | None = None
if from_source:
src_norm = _norm_token(from_source)
for member in members:
codes = member.get("source_codes")
if isinstance(codes, Mapping):
for k, v in codes.items():
if _norm_token(k) == src_norm and _norm_token(v) == probe_norm:
matched = member
break
if matched is not None:
break
if matched is None:
for member in members:
candidates = [member.get("value"), member.get("label"), *(member.get("aliases") or [])]
codes = member.get("source_codes")
if not from_source and isinstance(codes, Mapping):
candidates.extend(codes.values())
if any(_norm_token(c) == probe_norm for c in candidates if c is not None):
matched = member
break
if matched is None:
return None
canonical = str(matched.get("value") or "").strip()
label = str(matched.get("label") or canonical).strip()
target_value = canonical
if to_source and isinstance(matched.get("source_codes"), Mapping):
dst_norm = _norm_token(to_source)
for k, v in matched["source_codes"].items():
if _norm_token(k) == dst_norm and str(v or "").strip():
target_value = str(v).strip()
break
return {"canonical": canonical, "label": label, "target_value": target_value}
if from_source and not confirmed_bindings(data, source_name=from_source):
return None
if to_source and not confirmed_bindings(data, source_name=to_source):
return None
return {"canonical": probe, "label": probe, "target_value": probe}
def entities_bridging_sources(
entities: Iterable[CanonicalEntity | Mapping[str, Any]],
*,
source_name: str,
tables: Iterable[str] | None = None,
) -> list[dict[str, Any]]:
"""找出在当前数据源(及可选表集合)上有已确认绑定、并且同时桥接到其他数据源的标准实体。"""
table_keys = {_norm_token(t) for t in tables} if tables is not None else None
out: list[dict[str, Any]] = []
for entity in entities:
data = _as_dict(entity)
active = confirmed_bindings(data)
here = [
b for b in active
if _norm_token(b.get("source_name")) == _norm_token(source_name)
and (table_keys is None or _norm_token(b.get("table_name")) in table_keys)
]
if not here:
continue
peers = [b for b in active if _norm_token(b.get("source_name")) != _norm_token(source_name)]
out.append({
"entity_id": data["id"],
"key": data["key"],
"label": data["label"],
"local_bindings": here,
"peer_bindings": peers,
"peer_sources": sorted({str(b["source_name"]) for b in peers}),
})
return out
# --------------------------------------------------------------------------
# 数据库读写与乐观锁审阅
# --------------------------------------------------------------------------
async def list_entities(
session: AsyncSession,
*,
source_name: str | None = None,
only_active_bindings: bool = False,
) -> list[CanonicalEntity]:
rows = list((await session.execute(select(CanonicalEntity).order_by(CanonicalEntity.key.asc()))).scalars())
if not source_name:
return rows
src_norm = _norm_token(source_name)
matched: list[CanonicalEntity] = []
for row in rows:
bindings = confirmed_bindings(row) if only_active_bindings else (row.bindings or [])
if any(isinstance(b, Mapping) and _norm_token(b.get("source_name")) == src_norm for b in bindings):
matched.append(row)
return matched
async def get_entity(session: AsyncSession, entity_id: str) -> CanonicalEntity | None:
return await session.get(CanonicalEntity, str(entity_id or "").strip())
async def get_entity_by_key(session: AsyncSession, key: str) -> CanonicalEntity | None:
clean = str(key or "").strip().lower()
if not clean:
return None
return (
await session.execute(select(CanonicalEntity).where(CanonicalEntity.key == clean).limit(1))
).scalar_one_or_none()
async def create_entity(
session: AsyncSession,
payload: Mapping[str, Any],
*,
actor: str | None = None,
) -> CanonicalEntity:
fields = normalize_entity_payload(payload, partial=False, actor=actor)
existing = await get_entity_by_key(session, fields["key"])
if existing is not None:
raise EntityConflict(f"标准实体标识「{fields['key']}」已存在", field="key")
row = CanonicalEntity(
key=fields["key"],
label=fields["label"],
description=fields["description"],
aliases=fields["aliases"],
members=fields["members"],
bindings=fields["bindings"],
caliber_params=fields["caliber_params"],
version=1,
created_by=actor,
updated_by=actor,
)
session.add(row)
await session.flush()
return row
async def update_entity(
session: AsyncSession,
entity_id: str,
payload: Mapping[str, Any],
*,
actor: str | None = None,
) -> CanonicalEntity:
row = await get_entity(session, entity_id)
if row is None:
raise EntityNotFound(f"标准实体「{entity_id}」不存在")
expected_version = payload.get("version")
if expected_version is not None:
if isinstance(expected_version, bool) or not isinstance(expected_version, int) or expected_version != row.version:
raise EntityConflict(
f"标准实体「{row.label}」已被其他人修改(当前版本 v{row.version}),请刷新后再保存",
field="version",
)
fields = normalize_entity_payload(payload, partial=True, actor=actor)
if "key" in fields and fields["key"] != row.key:
clash = await get_entity_by_key(session, fields["key"])
if clash is not None and clash.id != row.id:
raise EntityConflict(f"标准实体标识「{fields['key']}」已存在", field="key")
row.key = fields["key"]
for attr in ("label", "description", "aliases", "members", "bindings", "caliber_params"):
if attr in fields:
setattr(row, attr, fields[attr])
row.version = int(row.version or 1) + 1
row.updated_by = actor
await session.flush()
return row
async def review_entity_item(
session: AsyncSession,
entity_id: str,
*,
target_kind: str,
item_id: str,
status: str,
reason: str = "",
actor: str | None = None,
expected_version: int | None = None,
) -> CanonicalEntity:
"""审阅单条物理表列绑定或口径参数映射(确认 confirmed / 驳回 rejected / 验证通过 verified)。"""
if target_kind not in ("binding", "caliber_param"):
raise EntityInvalid("审阅对象类型须为 binding 或 caliber_param", field="target_kind")
if status not in ENTITY_STATUSES:
raise EntityInvalid(f"状态「{status}」无效,可选:{', '.join(ENTITY_STATUSES)}", field="status")
row = await get_entity(session, entity_id)
if row is None:
raise EntityNotFound(f"标准实体「{entity_id}」不存在")
if expected_version is not None and int(expected_version) != int(row.version or 1):
raise EntityConflict(f"标准实体「{row.label}」版本已变更(当前 v{row.version}),请刷新后重试", field="version")
attr = "bindings" if target_kind == "binding" else "caliber_params"
items = [dict(x) for x in (getattr(row, attr) or []) if isinstance(x, Mapping)]
found = False
now = utcnow().isoformat()
for item in items:
if str(item.get("id") or "") == str(item_id or "").strip():
item["status"] = status
if reason.strip():
item["reason"] = reason.strip()
item["updated_at"] = now
item["updated_by"] = actor
found = True
break
if not found:
raise EntityNotFound(f"标准实体「{row.label}」中不存在子项「{item_id}」")
setattr(row, attr, items)
row.version = int(row.version or 1) + 1
row.updated_by = actor
await session.flush()
return row
async def delete_entity(session: AsyncSession, entity_id: str) -> bool:
row = await get_entity(session, entity_id)
if row is None:
return False
await session.delete(row)
await session.flush()
return True
+942
View File
@@ -0,0 +1,942 @@
"""实体助手(E2):在标准业务实体页与使用者流式对话,跨数据源与业务文档对齐维护 CanonicalEntity。
核心规矩:
1. 只提结构化提案(propose_entity_changes),绝不静默直接写库;使用者在界面看过后点击「保存修改」才经
apply_entity_patch(mode="confirmed") 落库。
2. 跨源只读探查(run_query)仅对已开启数据剖析的数据源放行,严格经 guard.check(readonly=True) 与遮罩列脱敏
(mask_query_result);遮罩列只给列名,不给注释或码值。
3. 支持直接读取本次对话挂载的上传附件或知识库文档片段(read_document),并要求在提案 reason 中区分「使用者说:……」、
「文档《…》写着:……」与「数据探查显示:……」,同时主动向使用者汇报文档码表与实表取值的差异。
"""
from __future__ import annotations
import asyncio
import contextlib
import json
import logging
from collections.abc import AsyncIterator, Iterable, Mapping, Sequence
from dataclasses import dataclass, field
from datetime import date, datetime
from decimal import Decimal
from typing import Any
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.data import catalog, catalog_assistant, catalog_profile, entities, entity_docs, entity_patch, introspect, table_versions
from app.data.draft_model import DRAFT_MODEL_PLACE
from app.data.engine import SnapshotTampered, masked_columns
from app.db.base import SessionLocal
from app.db.models import DataSource, Workflow, WorkflowVersion
logger = logging.getLogger(__name__)
MAX_MESSAGES = 40
MESSAGE_MAX = 4000
MAX_ROUNDS = 8
MAX_QUERIES = 6
QUERY_ROWS = 50
SHOWN_ROWS = 10
CELL_MAX = 300
TOOL_CHARS = 12_000
RETRY_DELAYS_S = (3.0, 10.0)
HEARTBEAT_S = 10.0
INSPECT_TOOL = "inspect_sources"
READ_DOC_TOOL = "read_document"
QUERY_TOOL = "run_query"
PROPOSE_TOOL = "propose_entity_changes"
CODE_MODEL = "assistant_model_error"
CODE_TRUNCATED = "assistant_truncated"
CODE_TIMEOUT = "assistant_timeout"
CODE_ROUNDS = "assistant_rounds"
CODE_FAILED = "assistant_failed"
# ---- 面向界面展示的错误文案
ERROR_MODEL = (
"调用模型「{model}」失败:{reason}。请稍后重试;如持续失败,请在{place}中换一个模型,"
"或到「设置 → 模型接入」测试该模型"
)
ERROR_TRUNCATED = "{cut},回答没有写完。可以把问题拆小一点再问,或在{place}中换一个模型"
ERROR_TIMEOUT = "模型 {seconds} 秒内没有回答完,已停止这一轮。可以把问题拆小一点再问,或在{place}中换一个模型"
ERROR_ROUNDS = "这一轮已经调用了 {n} 次模型仍没有答完,已停止。可以换个问法,或者分几次问"
ERROR_FAILED = "实体助手出错:{line}"
LAST_NOT_USER = "对话的最后一条应当是使用者的话,请重新发送"
QUERY_TAMPERED = "数据源「{name}」的{error}。请联系管理员检查数据目录"
# ---- 面向模型的提示词(命名含 _SYSTEM / _PROMPT)
ENTITY_ASSISTANT_SYSTEM = """你是 AgentLab 的「实体助手」,协助搭建者围绕当前选定范围的数据源与业务文档建立、完善「标准业务实体」(CanonicalEntity):
包括实体的标准标识(key)、中文名称(label)、常用叫法(aliases)、物理表列绑定(bindings)、标准成员编码映射(members.source_codes)与口径入口参数映射(caliber_params)。
核心规矩:
1. 你不能修改业务数据库数据,也不直接写入标准业务实体:一切新增或修改都通过 propose_entity_changes 工具整理成结构化提案,由使用者在界面审阅后点击保存才生效。
2. 严格限定在「当前分析范围的数据源」内工作:只探查、查询和绑定范围内的数据源,绝不提及、猜测或扩展到未纳入范围的其他数据源。若当前范围只有 1 个数据源,聚焦盘点该数据源内的核心业务实体、多表共享主外键、码值字典与口径入参;若选了多个数据源,则重点对齐这些选定数据源之间的同义实体与编码映射。
3. 表名、列名、数据源名、工作流 ID、口径参数名必须先通过 inspect_sources 核实,严禁猜测不存在的表名或列名。
4. 若挂载了业务文档或编码对照表,优先用 read_document 阅读原文定义或码值映射,并结合 inspect_sources / run_query 去真实数据源交叉核验:
- 如果文档里提到的编码与数据库实表抽样不一致(例如实表多出文档未登记的新编码,或文档编码在实表未出现),必须在回答正文中明确告知使用者,询问是否一并纳入。
- 如果拿不准两套编码如何对应,先简短具体地向使用者提问,不要硬猜。
5. 提案中每条操作的 reason 字段必须如实写明依据来源:
- 来自本次对话使用者亲口说明的,写「使用者说:……」;
- 来自挂载文档或知识库文档的,写「文档《文档名》写着:……」;
- 来自表目录或只读 SQL 核查的,写「数据探查显示:……」。
6. 标着「已确认」或「已验证」的既有绑定,除非使用者明确要求调整,否则不要擅自改动或删除;标着「已驳回」的绑定使用者曾明确拒绝过,不要重复提议。
7. 标着「遮罩列」的敏感字段禁止查询明细取值,也禁止为其生成成员编码映射(members.source_codes)。
8. 回答用简短直接的中文。
propose_entity_changes 的 operations 数组支持以下 5 种增量操作:
- upsert_entity: {"op": "upsert_entity", "entity_key": "小写英文下划线标识", "label": "中文名称", "description": "业务粒度说明", "aliases": ["常用叫法"], "reason": "依据"}
- upsert_binding: {"op": "upsert_binding", "entity_key": "实体标识", "entity_label": "中文名(新建时必填)", "source_name": "数据源名", "table_name": "物理表名", "column_name": "物理列名", "note": "备注", "reason": "依据"}
- remove_binding: {"op": "remove_binding", "entity_key": "实体标识", "source_name": "数据源名", "table_name": "物理表名", "column_name": "物理列名", "reason": "依据"}
- upsert_member: {"op": "upsert_member", "entity_key": "实体标识", "entity_label": "中文名", "value": "标准值", "label": "标准展示名", "aliases": ["成员别名"], "source_codes": {"数据源名": "该源底层编码"}, "reason": "依据"}
- upsert_caliber_param: {"op": "upsert_caliber_param", "entity_key": "实体标识", "entity_label": "中文名", "workflow_id": "工作流ID", "card_id": "可选口径卡节点ID", "param_name": "入口参数名", "note": "备注", "reason": "依据"}
"""
ENTITY_QUERY_PROMPT = (
"跨源只读查询:可对开启了数据剖析的数据源调用 run_query 执行只读 SQL(如 SELECT DISTINCT 核对编码取值与行数分布),"
"单次最多返回 {rows} 行,本轮最多执行 {limit} 次。可查询的数据源:{sources}。"
)
ENTITY_NO_QUERY_PROMPT = "当前范围内没有开启数据剖析的数据源,不能执行 run_query 查实表数据,请根据表结构、数据目录与业务文档推断对齐。"
ENTITY_OPENING_PROMPT = (
"(使用者刚打开实体助手,尚未发言。)请结合当前分析范围的数据源、既有实体与挂载文档概况,用两三句话指出当前范围内最值得优先建立或补齐的业务实体,并提出一两个具体问题;先不要调用 propose_entity_changes。"
)
ENTITY_OPENED_PROMPT = "(使用者打开了实体助手。)"
ENTITY_TOOL_PROMPT = {
"query_disabled_source": "数据源「{source}」未开启数据剖析或不存在,无法执行只读查询。可查询的数据源:{allowed}",
"query_capped": "这一轮的只读查询次数已达上限({n} 次),请根据已有信息回答或提出提案。",
"empty_sql": "请填写要执行的只读 SQL 语句。",
"query_failed": "查询未成功:{error}",
"no_ops": "operations 为空,请提供至少一条修改操作。",
"proposed": "已通过核验并向界面提交 {n} 项实体变更提案:{items}",
"nothing": "没有产生新的有效变更(可能与当前状态完全相同,或全部未通过核验)。",
"dropped": "以下项未通过平台核验,请核对表列名或参数后修正:{reasons}",
"no_docs": "当前对话未挂载任何业务文档或对照表附件。",
"unknown_tool": "未知工具「{name}」,可用工具:{tools}",
"last_round": "注意:下一轮是最后一次模型调用,请直接用中文向使用者总结结论,不要再调用工具。",
}
INSPECT_TOOL_PROMPT: dict[str, Any] = {
"type": "function",
"function": {
"name": INSPECT_TOOL,
"description": "在已启用的数据源表结构、数据目录(列中文名、含义、码值)及已发布口径入口参数中检索与业务实体相关的表列和入参。",
"parameters": {
"type": "object",
"properties": {
"keyword": {
"type": "string",
"description": "检索关键词(如「产线」「line」「仓库」「region」);留空则返回各数据源表概览与口径入参清单。",
},
"source_name": {
"type": "string",
"description": "可选:限定只查看某一个数据源名称。",
},
},
},
},
}
READ_DOC_TOOL_PROMPT: dict[str, Any] = {
"type": "function",
"function": {
"name": READ_DOC_TOOL,
"description": "读取本次对话挂载的业务文档、数据字典或编码对照表片段。",
"parameters": {
"type": "object",
"properties": {
"query": {
"type": "string",
"description": "按关键词检索相关段落或表格行;留空则按顺序读取分段。",
},
"doc_ref": {
"type": "string",
"description": "可选:限定某一份文档的 ID 或标题关键词。",
},
"start_chunk": {
"type": "integer",
"description": "顺序读取时的起始段号(从 0 开始)。",
},
"limit": {
"type": "integer",
"description": "最多返回几段(默认 6,最多 15)。",
},
},
},
},
}
QUERY_TOOL_PROMPT: dict[str, Any] = {
"type": "function",
"function": {
"name": QUERY_TOOL,
"description": "在指定数据源上执行一条只读 SQL(如 SELECT DISTINCT code, name FROM ... LIMIT 50),用于核对跨源成员编码与分布。",
"parameters": {
"type": "object",
"properties": {
"source_name": {"type": "string", "description": "要查询的数据源名称"},
"sql": {"type": "string", "description": "只读 SELECT 查询语句"},
},
"required": ["source_name", "sql"],
},
},
}
PROPOSE_TOOL_PROMPT: dict[str, Any] = {
"type": "function",
"function": {
"name": PROPOSE_TOOL,
"description": "提交一批标准业务实体的新建或修改提案(经平台核验后展示在界面待保存区,不直接写库)。",
"parameters": {
"type": "object",
"properties": {
"operations": {
"type": "array",
"description": "增量操作数组(upsert_entity / upsert_binding / remove_binding / upsert_member / upsert_caliber_param)",
"items": {"type": "object"},
},
},
"required": ["operations"],
},
},
}
@dataclass(slots=True)
class _SourceEntry:
row: DataSource
resolved: Any
kind: str
masked: set[str]
settings: catalog_profile.ProfileSettings
can_query: bool
catalog_notes: dict[str, Any]
@dataclass(slots=True)
class EntityAssistantContext:
sources: dict[str, _SourceEntry]
workflows: list[dict[str, Any]]
doc_bundles: list[entity_docs.EntityDocBundle]
pending_ops: list[dict[str, Any]] = field(default_factory=list)
system: str = ""
@property
def queryable_sources(self) -> list[str]:
return [entry.row.name for entry in self.sources.values() if entry.can_query]
def tools(self) -> list[dict[str, Any]]:
items = [INSPECT_TOOL_PROMPT]
if self.doc_bundles:
items.append(READ_DOC_TOOL_PROMPT)
if self.queryable_sources:
items.append(QUERY_TOOL_PROMPT)
items.append(PROPOSE_TOOL_PROMPT)
return items
resolve_model = catalog_assistant.resolve_model
def conversation_problem(messages: Iterable[Mapping[str, Any]]) -> str | None:
said = [m for m in messages if str(m.get("content") or "").strip()]
if any(m.get("role") == "user" for m in said) and said[-1].get("role") != "user":
return LAST_NOT_USER
return None
def _history(messages: Iterable[Mapping[str, Any]]) -> list[Any]:
from langchain_core.messages import AIMessage, HumanMessage
merged: list[tuple[str, str]] = []
for m in messages:
text = str(m.get("content") or "").strip()
if not text:
continue
role = "user" if m.get("role") == "user" else "assistant"
if merged and merged[-1][0] == role:
merged[-1] = (role, f"{merged[-1][1]}\n\n{text}")
else:
merged.append((role, text))
out: list[Any] = []
if merged and merged[0][0] == "assistant":
out.append(HumanMessage(content=ENTITY_OPENED_PROMPT))
for role, text in merged:
out.append(HumanMessage(content=text) if role == "user" else AIMessage(content=text))
if not any(role == "user" for role, _ in merged):
out.append(HumanMessage(content=ENTITY_OPENING_PROMPT))
return out
def _extract_wf_summary(
wf_rows: Sequence[Workflow],
ver_rows: Sequence[WorkflowVersion],
*,
allowed_sources: set[str] | None = None,
) -> list[dict[str, Any]]:
from app.tools.datasource import source_of_tool
published_by_wf: dict[str, WorkflowVersion] = {}
for ver in ver_rows:
cur = published_by_wf.get(ver.workflow_id)
if cur is None or int(ver.version or 0) > int(cur.version or 0):
published_by_wf[ver.workflow_id] = ver
summaries: list[dict[str, Any]] = []
for wf in wf_rows:
ver = published_by_wf.get(wf.id)
graph = (ver.graph if ver and isinstance(ver.graph, Mapping) else wf.graph) or {}
if not isinstance(graph, Mapping):
continue
cards: list[dict[str, Any]] = []
inputs: list[dict[str, Any]] = []
wf_sources: set[str] = set()
for node in graph.get("nodes") or []:
if not isinstance(node, Mapping):
continue
ntype = str(node.get("type") or "")
nid = str(node.get("id") or "")
data = node.get("data") if isinstance(node.get("data"), Mapping) else {}
cfg = data.get("config") if isinstance(data.get("config"), Mapping) else (
node.get("config") if isinstance(node.get("config"), Mapping) else {}
)
t_single = str(cfg.get("tool") or "").strip()
if s := source_of_tool(t_single):
if "{{" not in s:
wf_sources.add(s.lower())
for t_item in cfg.get("tools") or []:
if isinstance(t_item, str) and (s := source_of_tool(t_item.strip())) and "{{" not in s:
wf_sources.add(s.lower())
for skey in ("source", "source_name", "datasource", "source_id"):
sv = str(cfg.get(skey) or "").strip()
if sv and "{{" not in sv:
wf_sources.add(sv.lower())
if ntype == "metrics" and nid:
m_names = [
str(m.get("name") or m.get("id") or "")
for m in (cfg.get("metrics") or [])
if isinstance(m, Mapping)
]
cards.append({"card_id": nid, "metrics": [x for x in m_names if x][:8]})
elif ntype == "input":
for f in cfg.get("fields") or []:
if isinstance(f, Mapping) and str(f.get("name") or "").strip():
inputs.append({
"name": str(f["name"]).strip(),
"label": str(f.get("label") or f.get("description") or "").strip(),
"type": str(f.get("type") or "string"),
})
if allowed_sources is not None:
if wf_sources:
if not (wf_sources & allowed_sources):
continue
elif not allowed_sources:
continue
if cards and inputs:
summaries.append({
"workflow_id": wf.id,
"name": wf.name,
"published": ver is not None,
"inputs": inputs,
"cards": cards,
"sources": sorted(wf_sources),
})
return summaries
async def prepare(
session: AsyncSession,
*,
source_ids: Sequence[str] | None = None,
focus_entity_key: str | None = None,
attachment_ids: Sequence[str] = (),
document_ids: Sequence[str] = (),
pending_ops: Sequence[Mapping[str, Any]] = (),
) -> EntityAssistantContext:
"""组装实体助手上下文:按选定范围过滤的数据源表目录、既有标准实体、口径入参、挂载文档摘要与待保存提案。"""
all_enabled = list(
(await session.execute(select(DataSource).where(DataSource.enabled.is_(True)).order_by(DataSource.name))).scalars()
)
if source_ids is not None:
wanted = {str(s or "").strip().lower() for s in source_ids if str(s or "").strip()}
src_rows = [r for r in all_enabled if r.id.lower() in wanted or r.name.lower() in wanted]
else:
src_rows = all_enabled
allowed_sources: set[str] | None = (
{x for r in src_rows for x in (r.id.lower(), r.name.lower())}
if source_ids is not None
else None
)
sources: dict[str, _SourceEntry] = {}
source_lines: list[str] = ["当前分析范围的数据源:"]
for row in src_rows:
try:
resolved = await table_versions.resolve_source(session, row)
except table_versions.SnapshotMissing:
resolved = row
opts = dict(row.options or {}) if isinstance(row.options, Mapping) else {}
masked = {c.lower() for c in masked_columns(opts)}
p_settings = catalog_profile.profile_settings(opts)
can_query = bool(p_settings.enabled and row.enabled)
cat_entries = await catalog.read_catalog(session, resolved.id)
notes_map = {tname: entry.notes for tname, entry in cat_entries.items()}
sources[row.name.lower()] = _SourceEntry(
row=row,
resolved=resolved,
kind=str(row.kind or ""),
masked=masked,
settings=p_settings,
can_query=can_query,
catalog_notes=notes_map,
)
tables = ((resolved.schema_cache or {}).get("tables") or {}) if isinstance(resolved.schema_cache, Mapping) else {}
q_flag = "可只读查询" if can_query else "未开剖析"
source_lines.append(f"- {row.name}({row.kind},共 {len(tables)} 张表,{q_flag}){(':' + row.description) if row.description else ''}")
if len(source_lines) == 1:
source_lines.append("- (当前未选定任何数据源,仅基于挂载文档与既有实体分析)")
clean_pending = [dict(op) for op in (pending_ops or ()) if isinstance(op, Mapping)]
pending_keys = {
str(op.get("entity_key") or "").strip().lower()
for op in clean_pending
if str(op.get("entity_key") or "").strip()
}
ent_rows = await entities.list_entities(session)
focus_norm = (focus_entity_key or "").strip().lower()
in_scope_entities: list[Any] = []
other_domain_keys: list[str] = []
for erow in ent_rows:
ekey_low = erow.key.lower()
b_list = [b for b in (erow.bindings or []) if isinstance(b, Mapping)]
if allowed_sources is None or ekey_low == focus_norm or ekey_low in pending_keys or not b_list:
in_scope_entities.append(erow)
continue
touches_scope = any(
str(b.get("source_name") or "").strip().lower() in allowed_sources
or str(b.get("source_id") or "").strip().lower() in allowed_sources
for b in b_list
)
if touches_scope:
in_scope_entities.append(erow)
else:
other_domain_keys.append(f"{erow.key}({erow.label})")
entity_lines: list[str] = ["", f"当前范围相关的标准业务实体(共 {len(in_scope_entities)} 个):"]
for erow in in_scope_entities[:60]:
b_list = [b for b in (erow.bindings or []) if isinstance(b, Mapping)]
if allowed_sources is not None and erow.key.lower() != focus_norm:
b_list = [
b for b in b_list
if str(b.get("source_name") or "").strip().lower() in allowed_sources
or str(b.get("source_id") or "").strip().lower() in allowed_sources
]
active_b = [f"{b.get('source_name')}.{b.get('table_name')}.{b.get('column_name')}" for b in b_list if b.get("status") in ("confirmed", "verified")]
prop_b = [f"{b.get('source_name')}.{b.get('table_name')}.{b.get('column_name')}" for b in b_list if b.get("status") == "proposed"]
rej_b = [f"{b.get('source_name')}.{b.get('table_name')}.{b.get('column_name')}" for b in b_list if b.get("status") == "rejected"]
m_count = len(erow.members or [])
head = f"- {erow.label}(key={erow.key},v{erow.version},成员 {m_count} 项)"
if active_b:
head += f"|已确认绑定:{', '.join(active_b[:8])}"
if prop_b:
head += f"|待确认:{', '.join(prop_b[:6])}"
if rej_b:
head += f"|已驳回(勿重提):{', '.join(rej_b[:6])}"
entity_lines.append(head)
if focus_norm and erow.key.lower() == focus_norm:
entity_lines.append(f" 【当前使用者正在查看该实体】说明:{erow.description or '无'};别名:{', '.join(erow.aliases or []) or '无'}")
for m in (erow.members or [])[:25]:
if isinstance(m, Mapping):
codes_txt = ", ".join(f"{k}={v}" for k, v in (m.get("source_codes") or {}).items())
entity_lines.append(f" · 成员 {m.get('value')}({m.get('label')}):{codes_txt or '无跨源编码'}")
if other_domain_keys:
entity_lines.append(
f"(其他未选定数据源已占用的全局实体 key,避免重名冲突,无需分析未选定源:{', '.join(other_domain_keys[:30])})"
)
wf_rows = list((await session.execute(select(Workflow))).scalars())
ver_rows = list((await session.execute(select(WorkflowVersion).where(WorkflowVersion.published_at.is_not(None)))).scalars())
workflows = _extract_wf_summary(wf_rows, ver_rows, allowed_sources=allowed_sources)
doc_bundles = await entity_docs.load_doc_bundles(session, attachment_ids=attachment_ids, document_ids=document_ids)
doc_lines: list[str] = []
if doc_bundles:
doc_lines.append("")
doc_lines.append(f"本次对话挂载的业务文档/对照表(共 {len(doc_bundles)} 份,可用 read_document 检索详情):")
for b in doc_bundles:
preview = b.chunks[0][:300].replace("\n", " ") if b.chunks else ""
doc_lines.append(f"- 《{b.title}》(ref={b.ref_id},共 {len(b.chunks)} 段,{b.char_count} 字)首段预览:{preview}…")
pending_lines: list[str] = []
if clean_pending:
pending_lines.append("")
pending_lines.append(f"界面待保存区已有的修改提案(共 {len(clean_pending)} 项,避免重复提议):")
for pop in clean_pending[-40:]:
pending_lines.append(f"- {json.dumps(pop, ensure_ascii=False, default=str)}")
queryable = [e.row.name for e in sources.values() if e.can_query]
query_prompt = (
ENTITY_QUERY_PROMPT.format(rows=QUERY_ROWS, limit=MAX_QUERIES, sources="、".join(queryable))
if queryable
else ENTITY_NO_QUERY_PROMPT
)
system = "\n\n".join(
part
for part in [
ENTITY_ASSISTANT_SYSTEM,
query_prompt,
"\n".join([*source_lines, *entity_lines, *doc_lines, *pending_lines]),
]
if part
)
return EntityAssistantContext(
sources=sources,
workflows=workflows,
doc_bundles=doc_bundles,
pending_ops=clean_pending,
system=system,
)
def _cell(value: Any, limit: int | None = None) -> Any:
if value is None or isinstance(value, (bool, int, float)):
return value
if isinstance(value, (datetime, date)):
text = value.isoformat()
elif isinstance(value, Decimal):
text = str(value)
elif isinstance(value, (bytes, bytearray)):
text = f"<{len(value)} 字节>"
else:
text = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False, default=str)
if limit is not None and len(text) > limit:
return text[:limit] + "…"
return text
def _rows(rows: Any, n: int, limit: int | None) -> list[list[Any]]:
out: list[list[Any]] = []
for row in (rows or [])[:n]:
cells = row if isinstance(row, list) else list(row.values()) if isinstance(row, dict) else [row]
out.append([_cell(v, limit) for v in cells])
return out
class _EntityTools:
def __init__(self, ctx: EntityAssistantContext) -> None:
from app.engine.masking import SeenValues
self.ctx = ctx
self.queries = 0
self.seen_by_source: dict[str, Any] = {k: SeenValues() for k in ctx.sources}
def inspect(self, keyword: Any, source_name: Any) -> str:
kw = str(keyword or "").strip().lower()
src_filter = str(source_name or "").strip().lower()
lines: list[str] = []
for skey, entry in self.ctx.sources.items():
if src_filter and skey != src_filter and entry.row.id.lower() != src_filter:
continue
tables = ((entry.resolved.schema_cache or {}).get("tables") or {}) if isinstance(entry.resolved.schema_cache, Mapping) else {}
if not tables:
lines.append(f"数据源 {entry.row.name}:尚未探查表结构")
continue
matched_tables: list[str] = []
for tname, tmeta in tables.items():
if not isinstance(tmeta, Mapping):
continue
notes = entry.catalog_notes.get(tname) if isinstance(entry.catalog_notes.get(tname), Mapping) else {}
t_label_item = notes.get("label") if isinstance(notes.get("label"), Mapping) else {}
t_label = str(t_label_item.get("value") or "") if t_label_item.get("status") != "rejected" else ""
col_notes = notes.get("columns") if isinstance(notes.get("columns"), Mapping) else {}
cols = introspect.table_columns(dict(tmeta))
col_hits: list[str] = []
for col in cols:
cname = str(col.get("name") or "")
is_masked = (
cname.lower() in entry.masked
or f"{tname.lower()}.{cname.lower()}" in entry.masked
)
cnote = col_notes.get(cname) if isinstance(col_notes.get(cname), Mapping) else {}
clabel = ""
cmeaning = ""
ccodes = ""
if not is_masked and isinstance(cnote, Mapping):
li = cnote.get("label")
if isinstance(li, Mapping) and li.get("status") != "rejected":
clabel = str(li.get("value") or "")
mi = cnote.get("meaning")
if isinstance(mi, Mapping) and mi.get("status") != "rejected":
cmeaning = str(mi.get("value") or "")
ci = cnote.get("codes")
if isinstance(ci, Mapping) and ci.get("status") != "rejected" and isinstance(ci.get("value"), Mapping):
pairs = [f"{k}={v}" for k, v in list(ci["value"].items())[:12]]
ccodes = f" 码值[{', '.join(pairs)}]" if pairs else ""
comment = "" if is_masked else str(col.get("comment") or "")
haystack = " ".join([tname, t_label, cname, clabel, cmeaning, comment, ccodes]).lower()
if not kw or kw in haystack:
desc_parts = [p for p in [clabel, cmeaning, comment] if p]
mask_tag = " [遮罩列]" if is_masked else ""
col_hits.append(
f" · {cname} ({col.get('type') or 'unknown'}){mask_tag}"
+ (f":{' / '.join(desc_parts)}" if desc_parts else "")
+ ccodes
)
if col_hits:
head = f" 表 {tname}" + (f"({t_label})" if t_label else "")
matched_tables.append(head + "\n" + "\n".join(col_hits[:25]))
if matched_tables:
lines.append(f"【数据源 {entry.row.name}】\n" + "\n".join(matched_tables[:15]))
wf_hits: list[str] = []
for wf in self.ctx.workflows:
hay = json.dumps(wf, ensure_ascii=False).lower()
if not kw or kw in hay:
in_txt = ", ".join(f"{i['name']}({i['label'] or i['type']})" for i in wf["inputs"])
cd_txt = ", ".join(f"{c['card_id']}[{'/'.join(c['metrics'])}]" for c in wf["cards"])
pub_tag = "已发布" if wf["published"] else "草稿"
wf_hits.append(f"- 工作流 {wf['name']}(workflow_id={wf['workflow_id']},{pub_tag})|入口参数:{in_txt}|口径卡:{cd_txt}")
if wf_hits:
lines.append("【口径工作流与入口参数】\n" + "\n".join(wf_hits[:20]))
if not lines:
return f"未找到与「{keyword}」匹配的表列或口径参数,请尝试更短的词根或留空 keyword 查看全景概览。"
text = "\n\n".join(lines)
return text[:TOOL_CHARS] + ("…(已截断)" if len(text) > TOOL_CHARS else "")
def read_doc(self, query: Any, doc_ref: Any, start_chunk: Any, limit: Any) -> str:
if not self.ctx.doc_bundles:
return ENTITY_TOOL_PROMPT["no_docs"]
hits = entity_docs.search_doc_bundles(
self.ctx.doc_bundles,
query=str(query or ""),
doc_ref=str(doc_ref or ""),
start_chunk=int(start_chunk) if isinstance(start_chunk, int) else 0,
limit=int(limit) if isinstance(limit, int) else 6,
)
if not hits:
return "未检索到匹配的文档段落,可清空 query 按 start_chunk=0 顺序浏览。"
blocks = [
f"【{h['citation']}(共 {h['total_chunks']} 段,ref={h['doc_ref']})】\n{h['content']}"
for h in hits
]
text = "\n\n".join(blocks)
return text[:TOOL_CHARS] + ("…(已截断)" if len(text) > TOOL_CHARS else "")
async def query(self, source_name: Any, sql: Any) -> tuple[dict[str, Any] | None, str]:
prompts = ENTITY_TOOL_PROMPT
src_key = str(source_name or "").strip().lower()
entry = self.ctx.sources.get(src_key)
if entry is None or not entry.can_query:
allowed = "、".join(self.ctx.queryable_sources) or "无"
return None, prompts["query_disabled_source"].format(source=source_name or "", allowed=allowed)
if self.queries >= MAX_QUERIES:
return None, prompts["query_capped"].format(n=MAX_QUERIES)
clean_sql = sql.strip() if isinstance(sql, str) else ""
if not clean_sql:
return None, prompts["empty_sql"]
self.queries += 1
seen = self.seen_by_source[src_key]
mask = masked_columns(dict(entry.row.options or {}) if isinstance(entry.row.options, Mapping) else None)
known = await asyncio.to_thread(seen.pooled)
known = known if known.strong else None
error: str | None = None
result: Any = None
try:
result = await catalog_profile.run_single_query(
entry.resolved,
clean_sql,
settings=entry.settings,
max_rows=QUERY_ROWS,
kind=entry.kind,
)
except catalog_profile.QueryFailed as exc:
error = exc.detail
except SnapshotTampered as exc:
error = QUERY_TAMPERED.format(name=entry.row.name, error=exc)
except ValueError as exc:
error = str(exc)
if error is not None:
if mask or known is not None:
from app.engine.masking import mask_error_text
error = mask_error_text(error, clean_sql, known)
event = {
"type": "query",
"source": entry.row.name,
"sql": clean_sql,
"columns": [],
"rows": [],
"row_count": 0,
"truncated": False,
"error": error,
}
return event, prompts["query_failed"].format(error=error)
payload = result.to_payload()
if mask or known is not None:
from app.engine.masking import mask_query_result
schema = getattr(entry.resolved, "schema_cache", None)
done = await asyncio.to_thread(mask_query_result, payload, mask, entry.kind, known, schema=schema)
payload = done.payload
await asyncio.to_thread(seen.add, done.secrets)
columns = [str(c) for c in payload.get("columns") or []]
rows = payload.get("rows") or []
event = {
"type": "query",
"source": entry.row.name,
"sql": str(payload.get("sql") or clean_sql),
"columns": columns,
"rows": _rows(rows, SHOWN_ROWS, None),
"row_count": int(payload.get("row_count") or len(rows)),
"truncated": bool(payload.get("truncated")),
"error": None,
}
given = {
"source": entry.row.name,
"columns": columns,
"rows": _rows(rows, QUERY_ROWS, CELL_MAX),
"row_count": event["row_count"],
"truncated": event["truncated"],
}
text = json.dumps(given, ensure_ascii=False, default=str)
while len(text) > TOOL_CHARS and given["rows"]:
given["rows"] = given["rows"][: max(1, len(given["rows"]) // 2)] if len(given["rows"]) > 1 else []
text = json.dumps(given, ensure_ascii=False, default=str)
return event, text
async def propose(self, operations: Any) -> tuple[dict[str, Any] | None, str]:
prompts = ENTITY_TOOL_PROMPT
if not isinstance(operations, list) or not operations:
return None, prompts["no_ops"]
async with SessionLocal() as session:
plan = await entity_patch.plan_entity_patch(
session,
operations,
pending_ops=self.ctx.pending_ops,
mode="confirmed",
)
said: list[str] = []
event: dict[str, Any] | None = None
if plan.changes:
for ch in plan.changes:
self.ctx.pending_ops.append(ch.op_payload)
event = {
"type": "proposal",
"changes": [ch.as_dict() for ch in plan.changes],
"expected_versions": plan.expected_versions,
}
said.append(
prompts["proposed"].format(
n=len(plan.changes),
items=";".join(ch.summary for ch in plan.changes),
)
)
else:
said.append(prompts["nothing"])
if plan.problems:
said.append(prompts["dropped"].format(reasons=";".join(plan.problems)))
return event, "\n".join(said)
_SENTENCE_END = ("。", "!", "?", ";", ":", ".", "!", "?", ";", ":")
def _sse(obj: Mapping[str, Any]) -> str:
return f"data: {json.dumps(obj, ensure_ascii=False, default=str)}\n\n"
class _CallFailed(Exception):
def __init__(self, code: str, message: str) -> None:
super().__init__(message)
self.code = code
self.message = message
def _seconds(value: Any) -> str:
number = float(value)
return str(int(number)) if number.is_integer() else f"{number:g}"
def _transient(exc: BaseException) -> bool:
from app.core.errors import NETWORK, RATE_LIMIT, SERVER, classify
return classify(exc) in (RATE_LIMIT, SERVER, NETWORK)
async def _pump(runner: Any, history: list[Any], queue: asyncio.Queue[tuple[str, Any]]) -> None:
try:
async for chunk in runner.astream(history):
await queue.put(("chunk", chunk))
await queue.put(("end", None))
except asyncio.CancelledError:
raise
except Exception as exc: # noqa: BLE001
await queue.put(("error", exc))
async def _round(runner: Any, history: list[Any], model_id: str) -> AsyncIterator[tuple[str, Any]]:
from app.core.errors import TIMEOUT, classify, describe_exception
from app.engine.state import message_text
from app.providers.budget import cut_off
loop = asyncio.get_running_loop()
attempt = 0
while True:
queue: asyncio.Queue[tuple[str, Any]] = asyncio.Queue()
task = asyncio.ensure_future(_pump(runner, history, queue))
deadline = loop.time() + float(catalog.MODEL_TIMEOUT_S)
message: Any = None
cut = False
started = False
failure: BaseException | None = None
try:
while True:
remaining = deadline - loop.time()
if remaining <= 0:
raise _CallFailed(
CODE_TIMEOUT,
ERROR_TIMEOUT.format(seconds=_seconds(catalog.MODEL_TIMEOUT_S), place=DRAFT_MODEL_PLACE),
)
try:
kind, value = await asyncio.wait_for(queue.get(), timeout=min(remaining, HEARTBEAT_S))
except asyncio.TimeoutError:
yield "beat", None
continue
if kind == "end":
break
if kind == "error":
failure = value
break
message = value if message is None else message + value
cut = cut or bool(cut_off(value))
text = message_text(value)
if text:
started = True
yield "text", text
finally:
if not task.done():
task.cancel()
with contextlib.suppress(asyncio.CancelledError, Exception):
await task
if failure is None:
yield "message", (message, cut)
return
if not started and attempt < len(RETRY_DELAYS_S) and _transient(failure):
logger.info("实体助手:调用模型出错(%s),%s 秒后重试", type(failure).__name__, RETRY_DELAYS_S[attempt])
await asyncio.sleep(RETRY_DELAYS_S[attempt])
attempt += 1
continue
if isinstance(failure, TimeoutError) or classify(failure) == TIMEOUT:
raise _CallFailed(
CODE_TIMEOUT,
ERROR_TIMEOUT.format(seconds=_seconds(catalog.MODEL_TIMEOUT_S), place=DRAFT_MODEL_PLACE),
) from failure
raise _CallFailed(
CODE_MODEL,
ERROR_MODEL.format(model=model_id or "", reason=describe_exception(failure), place=DRAFT_MODEL_PLACE),
) from failure
async def stream(
model: Any,
model_id: str,
ctx: EntityAssistantContext,
messages: Iterable[Mapping[str, Any]],
) -> AsyncIterator[str]:
"""实体助手流式生成器:逐行产出 SSE data 事件与心跳注释。"""
from langchain_core.messages import SystemMessage, ToolMessage, message_chunk_to_message
from app.providers.budget import cut_words
history: list[Any] = [SystemMessage(content=ctx.system), *_history(messages)]
tools = _EntityTools(ctx)
usage = {"input_tokens": 0, "output_tokens": 0}
wrote = ""
tool_defs = ctx.tools()
names = [t["function"]["name"] for t in tool_defs]
try:
runner = model.bind_tools(tool_defs)
for round_no in range(MAX_ROUNDS):
fresh = True
message: Any = None
cut = False
async with contextlib.aclosing(_round(runner, history, model_id)) as rounds:
async for kind, value in rounds:
if kind == "beat":
yield ": keep-alive\n\n"
elif kind == "text":
if fresh and wrote.endswith(_SENTENCE_END):
yield _sse({"type": "text", "delta": "\n\n"})
fresh = False
wrote = value
yield _sse({"type": "text", "delta": value})
else:
message, cut = value
meta = getattr(message, "usage_metadata", None) or {}
usage["input_tokens"] += int(meta.get("input_tokens") or 0)
usage["output_tokens"] += int(meta.get("output_tokens") or 0)
if cut:
limit = getattr(model, "max_tokens", None)
yield _sse({
"type": "error",
"code": CODE_TRUNCATED,
"message": ERROR_TRUNCATED.format(cut=cut_words("实体助手", limit), place=DRAFT_MODEL_PLACE),
})
return
calls = list(getattr(message, "tool_calls", None) or []) if message is not None else []
if not calls:
yield _sse({"type": "done", "usage": usage, "queries": tools.queries})
return
if round_no == MAX_ROUNDS - 1:
yield _sse({"type": "error", "code": CODE_ROUNDS, "message": ERROR_ROUNDS.format(n=MAX_ROUNDS)})
return
history.append(message_chunk_to_message(message))
for call in calls:
name = str(call.get("name") or "")
args = call.get("args") if isinstance(call.get("args"), Mapping) else {}
call_id = str(call.get("id") or f"{name}-{round_no}-{len(history)}")
event: dict[str, Any] | None = None
if name == INSPECT_TOOL:
content = tools.inspect(args.get("keyword"), args.get("source_name"))
elif name == READ_DOC_TOOL:
content = tools.read_doc(
args.get("query"),
args.get("doc_ref"),
args.get("start_chunk"),
args.get("limit"),
)
elif name == QUERY_TOOL:
event, content = await tools.query(args.get("source_name"), args.get("sql"))
elif name == PROPOSE_TOOL:
event, content = await tools.propose(args.get("operations"))
else:
content = ENTITY_TOOL_PROMPT["unknown_tool"].format(name=name, tools="、".join(names))
if event is not None:
yield _sse(event)
if round_no == MAX_ROUNDS - 2:
content = f"{content}\n{ENTITY_TOOL_PROMPT['last_round']}"
history.append(ToolMessage(content=content, tool_call_id=call_id))
except _CallFailed as exc:
yield _sse({"type": "error", "code": exc.code, "message": exc.message})
except asyncio.CancelledError:
raise
except Exception as exc: # noqa: BLE001
from app.core.errors import describe_exception, first_line
logger.exception("实体助手执行异常")
yield _sse({
"type": "error",
"code": CODE_FAILED,
"message": ERROR_FAILED.format(line=first_line(exc) or describe_exception(exc)),
})
+687
View File
@@ -0,0 +1,687 @@
"""后台主动发现候选标准业务实体任务(E2:扫描发现候选实体)。
扫描范围:
1. 所选已启用数据源的表结构(主键、外键、标识/属性/状态列)与数据目录(中文名、含义、非遮罩码值);
2. 已发布与草稿口径工作流的入口参数(entry_fields / input 节点)及口径卡;
3. 可选挂载的业务文档或跨系统编码对照表(知识库 Document 或临时上传附件)。
铁律:
- 发现结果一律经 entity_patch.apply_entity_patch(..., mode="proposed") 写入,所有表列绑定与口径入参映射强制标记为
status="proposed", source="ai",绝不直接参与线上跨源穿透,也绝不覆盖人工已确认、已验证或已驳回的项。
"""
from __future__ import annotations
import asyncio
import contextlib
import json
import logging
import re
import time
from collections.abc import Mapping, Sequence
from datetime import timezone
from typing import Any
from sqlalchemy import select, update
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.errors import describe_exception, first_line
from app.data import catalog, catalog_assistant, entities, entity_assistant, entity_docs, entity_patch, introspect, table_versions
from app.data.engine import masked_columns
from app.db.base import SessionLocal, utcnow
from app.db.models import DataSource, Workflow, WorkflowVersion
from app.db.models_ontology import EntityDiscoveryJob
logger = logging.getLogger(__name__)
OPEN_STATUSES = ("queued", "running")
NOTE_MAX = 1000
MAX_DOC_CHARS_IN_DISCOVERY = 28_000
MAX_SCHEMA_CHARS_IN_DISCOVERY = 36_000
DISCOVERY_BUSY_CODE = "discovery_job_busy"
DISCOVERY_FINISHED_CODE = "discovery_job_finished"
INTERRUPTED_ERROR = "服务重启,候选实体扫描中断"
JOB_FAILED_TEXT = "扫描候选实体失败:{reason}"
_FENCE_RE = re.compile(r"```(?:json)?\s*([\s\S]*?)```", re.IGNORECASE)
_ACTIVE_TASKS: dict[str, asyncio.Task[Any]] = {}
_CREATE_LOCK = asyncio.Lock()
class DiscoveryBusy(Exception):
"""已有正在排队或执行的候选实体扫描任务。"""
class DiscoveryFinished(Exception):
"""要取消的候选实体扫描任务已经结束。"""
class DiscoveryEmptyScope(ValueError):
"""扫描范围内没有任何可用的数据源表结构或业务文档。"""
DISCOVERY_SYSTEM_PROMPT = """你是 AgentLab 的「业务实体发现专家」。请根据给定的数据源表结构与数据目录、口径工作流入口参数以及业务文档/编码对照表,识别共用的「标准业务实体」(如产线、工厂、车间、仓库、门店、销售大区、渠道、客户等级、不良代码等),并输出增量操作 JSON。
硬性纪律:
1. 只输出一个合法 JSON 对象,格式为 {"operations": [...]},不要输出任何额外解释文字。
2. 若输入包含多个数据源,优先发现在多源之间形成桥接的核心业务对象;若仅包含单个数据源,则发现在该源多张业务表、口径入口参数或业务文档之间共用的核心业务维度实体。纯技术自增流水主键不要提议为业务实体。
3. 表名、列名、数据源名、工作流 ID、参数名必须逐字取自输入材料,严禁编造不存在的字段。
4. 标着「[遮罩列]」的敏感字段严禁为其生成 upsert_member 编码映射;标着「已驳回」的历史绑定严禁重复提议。
5. 每条操作必须填写简明的中文 reason:
- 来自文档或对照表的写「文档《文档名》写着:……」;
- 来自表目录或字段语义对齐的写「目录语义对齐:……」。
operations 支持以下 4 种结构:
- {"op": "upsert_entity", "entity_key": "小写英文下划线标识(如 production_line)", "label": "中文名称", "description": "粒度与业务对齐说明", "aliases": ["同义词"], "reason": "依据"}
- {"op": "upsert_binding", "entity_key": "实体标识", "entity_label": "中文名称", "source_name": "数据源名称", "table_name": "物理表名", "column_name": "物理列名", "note": "字段角色备注", "reason": "依据"}
- {"op": "upsert_member", "entity_key": "实体标识", "entity_label": "中文名称", "value": "标准编码值", "label": "标准展示名", "aliases": ["别名"], "source_codes": {"数据源名称": "该源编码值"}, "reason": "依据"}
- {"op": "upsert_caliber_param", "entity_key": "实体标识", "entity_label": "中文名称", "workflow_id": "工作流ID", "card_id": "口径卡节点ID(可空串)", "param_name": "入口参数名", "note": "参数对齐说明", "reason": "依据"}
"""
DISCOVERY_USER_PROMPT = """请扫描以下数据源结构、口径参数与业务文档,提取候选标准业务实体及其跨源绑定与编码映射,输出 {{"operations": [...]}} JSON:
{body}
"""
def _iso(val: Any) -> str | None:
if val is None or not hasattr(val, "isoformat"):
return None
if getattr(val, "tzinfo", None) is None:
val = val.replace(tzinfo=timezone.utc)
return val.isoformat()
def job_to_dict(row: EntityDiscoveryJob) -> dict[str, Any]:
return {
"id": row.id,
"status": row.status,
"source_ids": list(row.source_ids or []),
"document_ids": list(row.document_ids or []),
"attachment_ids": list(row.attachment_ids or []),
"note": row.note or "",
"phase": row.phase or "",
"summary": dict(row.summary or {}),
"error": row.error,
"started_by": row.started_by,
"started_at": _iso(row.started_at),
"finished_at": _iso(row.finished_at),
"created_at": _iso(row.created_at),
"updated_at": _iso(row.updated_at),
}
def parse_discovery_operations(raw_text: str) -> list[dict[str, Any]]:
"""从模型回复中提取 operations 列表。"""
text = (raw_text or "").strip()
if not text:
return []
m = _FENCE_RE.search(text)
candidate = m.group(1).strip() if m else text
if not candidate.startswith(("{", "[")):
brace = candidate.find("{")
bracket = candidate.find("[")
starts = [i for i in (brace, bracket) if i >= 0]
if starts:
candidate = candidate[min(starts) :]
try:
parsed = json.loads(candidate)
except ValueError:
# 尝试截取最后一个匹配括号
end_brace = candidate.rfind("}")
if end_brace > 0:
try:
parsed = json.loads(candidate[: end_brace + 1])
except ValueError:
return []
else:
return []
if isinstance(parsed, list):
return [dict(x) for x in parsed if isinstance(x, Mapping)]
if isinstance(parsed, Mapping):
ops = parsed.get("operations")
if isinstance(ops, list):
return [dict(x) for x in ops if isinstance(x, Mapping)]
return []
async def get_active_or_latest_job(session: AsyncSession) -> dict[str, Any]:
active = (
await session.execute(
select(EntityDiscoveryJob)
.where(EntityDiscoveryJob.status.in_(OPEN_STATUSES))
.order_by(EntityDiscoveryJob.created_at.desc())
.limit(1)
)
).scalar_one_or_none()
latest = (
await session.execute(
select(EntityDiscoveryJob)
.order_by(EntityDiscoveryJob.created_at.desc())
.limit(1)
)
).scalar_one_or_none()
return {
"job": job_to_dict(active) if active is not None else None,
"latest": job_to_dict(latest) if latest is not None else None,
}
async def get_job(session: AsyncSession, job_id: str) -> EntityDiscoveryJob | None:
return await session.get(EntityDiscoveryJob, str(job_id or "").strip())
async def _build_discovery_prompt_body(
session: AsyncSession,
*,
source_ids: Sequence[str],
document_ids: Sequence[str],
attachment_ids: Sequence[str],
note: str,
) -> tuple[str, dict[str, Any]]:
"""返回 (prompt_body, 扫描范围量化统计)。"""
stmt = select(DataSource).where(DataSource.enabled.is_(True)).order_by(DataSource.name)
if source_ids:
stmt = stmt.where(DataSource.id.in_(list(source_ids)))
src_rows = list((await session.execute(stmt)).scalars())
schema_blocks: list[str] = []
valid_sources = 0
source_names: list[str] = []
tables_scanned = 0
columns_scanned = 0
for row in src_rows:
try:
resolved = await table_versions.resolve_source(session, row)
except table_versions.SnapshotMissing:
resolved = row
tables = ((resolved.schema_cache or {}).get("tables") or {}) if isinstance(resolved.schema_cache, Mapping) else {}
if not tables:
continue
valid_sources += 1
source_names.append(row.name)
opts = dict(row.options or {}) if isinstance(row.options, Mapping) else {}
masked = {c.lower() for c in masked_columns(opts)}
cat_entries = await catalog.read_catalog(session, resolved.id)
t_lines = [f"### 数据源:{row.name}({row.kind})"]
for tname, tmeta in list(tables.items())[:40]:
if not isinstance(tmeta, Mapping):
continue
entry = cat_entries.get(tname)
notes = entry.notes if entry else {}
t_label_item = notes.get("label") if isinstance(notes.get("label"), Mapping) else {}
t_label = str(t_label_item.get("value") or "") if t_label_item.get("status") != "rejected" else ""
col_notes = notes.get("columns") if isinstance(notes.get("columns"), Mapping) else {}
cols = introspect.table_columns(dict(tmeta))
c_parts: list[str] = []
for col in cols[:40]:
cname = str(col.get("name") or "")
is_masked = cname.lower() in masked or f"{tname.lower()}.{cname.lower()}" in masked
cnote = col_notes.get(cname) if isinstance(cnote := col_notes.get(cname), Mapping) else {}
measure = ""
clabel = ""
cmeaning = ""
ccodes = ""
if not is_masked and isinstance(cnote, Mapping):
mi = cnote.get("measure")
if isinstance(mi, Mapping) and mi.get("status") != "rejected":
measure = str(mi.get("value") or "")
# 跳过明显的纯度量浮点流量列,聚焦维度/标识/状态/外键或未分类列
if measure in ("flow", "stock", "ratio"):
continue
li = cnote.get("label")
if isinstance(li, Mapping) and li.get("status") != "rejected":
clabel = str(li.get("value") or "")
mgi = cnote.get("meaning")
if isinstance(mgi, Mapping) and mgi.get("status") != "rejected":
cmeaning = str(mgi.get("value") or "")
ci = cnote.get("codes")
if isinstance(ci, Mapping) and ci.get("status") != "rejected" and isinstance(ci.get("value"), Mapping):
pairs = [f"{k}={v}" for k, v in list(ci["value"].items())[:15]]
ccodes = f" 码值[{', '.join(pairs)}]" if pairs else ""
comment = "" if is_masked else str(col.get("comment") or "")
tags = [p for p in [clabel, cmeaning, comment] if p]
pk_tag = "[PK]" if col.get("pk") else ""
mask_tag = "[遮罩列]" if is_masked else ""
c_parts.append(
f" - {cname} ({col.get('type') or ''}){pk_tag}{mask_tag}"
+ (f":{' / '.join(tags)}" if tags else "")
+ ccodes
)
if c_parts:
tables_scanned += 1
columns_scanned += len(c_parts)
t_lines.append(f"- 表 {tname}" + (f"({t_label})" if t_label else "") + "\n" + "\n".join(c_parts))
block = "\n".join(t_lines)
schema_blocks.append(block)
schema_text = "\n\n".join(schema_blocks)
if len(schema_text) > MAX_SCHEMA_CHARS_IN_DISCOVERY:
schema_text = schema_text[:MAX_SCHEMA_CHARS_IN_DISCOVERY] + "\n…(表结构摘要已截断)"
allowed_sources: set[str] | None = (
{x for r in src_rows for x in (r.id.lower(), r.name.lower())}
if source_ids
else None
)
wf_rows = list((await session.execute(select(Workflow))).scalars())
ver_rows = list((await session.execute(select(WorkflowVersion).where(WorkflowVersion.published_at.is_not(None)))).scalars())
workflows = entity_assistant._extract_wf_summary(wf_rows, ver_rows, allowed_sources=allowed_sources)
wf_lines: list[str] = []
if workflows:
wf_lines.append("## 口径工作流与入口参数")
for wf in workflows[:30]:
ins = ", ".join(f"{i['name']}({i['label'] or i['type']})" for i in wf["inputs"])
cds = ", ".join(f"{c['card_id']}[{'/'.join(c['metrics'])}]" for c in wf["cards"])
wf_lines.append(f"- {wf['name']}(workflow_id={wf['workflow_id']})|入口参数:{ins}|口径卡:{cds}")
ent_rows = await entities.list_entities(session)
ent_lines: list[str] = []
if ent_rows:
ent_lines.append("## 既有标准业务实体(请复用同域已有 entity_key 增补绑定,勿重提已驳回项)")
other_keys: list[str] = []
for erow in ent_rows[:60]:
b_list = [b for b in (erow.bindings or []) if isinstance(b, Mapping)]
if allowed_sources is not None and b_list:
b_scoped = [
b for b in b_list
if str(b.get("source_name") or "").strip().lower() in allowed_sources
or str(b.get("source_id") or "").strip().lower() in allowed_sources
]
if not b_scoped:
other_keys.append(f"{erow.key}({erow.label})")
continue
b_list = b_scoped
active_b = [f"{b.get('source_name')}.{b.get('table_name')}.{b.get('column_name')}" for b in b_list if b.get("status") != "rejected"]
rej_b = [f"{b.get('source_name')}.{b.get('table_name')}.{b.get('column_name')}" for b in b_list if b.get("status") == "rejected"]
line = f"- {erow.label}(entity_key={erow.key})|已有绑定:{', '.join(active_b) or '无'}"
if rej_b:
line += f"|已驳回:{', '.join(rej_b)}"
ent_lines.append(line)
if other_keys:
ent_lines.append(f"(其他未选定数据源已占用的实体标识:{', '.join(other_keys[:30])})")
bundles = await entity_docs.load_doc_bundles(session, attachment_ids=attachment_ids, document_ids=document_ids)
doc_blocks: list[str] = []
used_doc_chars = 0
for bundle in bundles:
piece_text = "\n\n".join(bundle.chunks)
remain = MAX_DOC_CHARS_IN_DISCOVERY - used_doc_chars
if remain <= 0:
break
clipped = piece_text[:remain]
used_doc_chars += len(clipped)
doc_blocks.append(f"### 文档《{bundle.title}》\n{clipped}")
sections: list[str] = []
if note.strip():
sections.append(f"## 使用者补充的扫描重点\n{note.strip()}")
if schema_text:
sections.append(f"## 数据源与表目录\n{schema_text}")
if wf_lines:
sections.append("\n".join(wf_lines))
if ent_lines:
sections.append("\n".join(ent_lines))
if doc_blocks:
sections.append("## 业务文档与主数据对照表\n" + "\n\n".join(doc_blocks))
scope_stats: dict[str, Any] = {
"sources_scanned": valid_sources,
"source_names": source_names,
"tables_scanned": tables_scanned,
"columns_scanned": columns_scanned,
"workflows_scanned": len(workflows),
"documents_scanned": len(bundles),
"doc_titles": [b.title for b in bundles[:15]],
"chunks_scanned": sum(len(b.chunks) for b in bundles),
}
return "\n\n".join(sections), scope_stats
async def create_job(
session: AsyncSession,
*,
source_ids: Sequence[str] | None = None,
document_ids: Sequence[str] | None = None,
attachment_ids: Sequence[str] | None = None,
note: str | None = None,
actor: str | None = None,
) -> EntityDiscoveryJob:
clean_note = (note or "").strip()
if len(clean_note) > NOTE_MAX:
raise ValueError(f"扫描补充说明最多 {NOTE_MAX} 字,当前 {len(clean_note)} 字")
# 建任务前先校验起草模型可用性
await catalog_assistant.resolve_model(session)
async with _CREATE_LOCK:
existing = (
await session.execute(
select(EntityDiscoveryJob).where(EntityDiscoveryJob.status.in_(OPEN_STATUSES)).limit(1)
)
).scalar_one_or_none()
if existing is not None:
raise DiscoveryBusy("已有一个正在执行的候选实体扫描任务,请等待完成后再发起")
clean_sources = list(dict.fromkeys(str(s).strip() for s in (source_ids or []) if str(s or "").strip()))
clean_docs = list(dict.fromkeys(str(d).strip() for d in (document_ids or []) if str(d or "").strip()))
clean_atts = list(dict.fromkeys(str(a).strip() for a in (attachment_ids or []) if str(a or "").strip()))
prompt_body, scope_stats = await _build_discovery_prompt_body(
session,
source_ids=clean_sources,
document_ids=clean_docs,
attachment_ids=clean_atts,
note=clean_note,
)
if scope_stats["sources_scanned"] == 0 and scope_stats["documents_scanned"] == 0 or not prompt_body.strip():
raise DiscoveryEmptyScope("所选范围内没有已探查表结构的数据源或有效业务文档,请先探查表结构或添加文档")
job = EntityDiscoveryJob(
status="queued",
source_ids=clean_sources,
document_ids=clean_docs,
attachment_ids=clean_atts,
note=clean_note,
phase="已加入扫描队列…",
summary={
**scope_stats,
"step_index": 1,
"step_total": 4,
"progress_pct": 8,
},
started_by=actor,
)
session.add(job)
await session.commit()
await session.refresh(job)
task = asyncio.create_task(_execute_job(job.id))
_ACTIVE_TASKS[job.id] = task
task.add_done_callback(lambda _t, jid=job.id: _ACTIVE_TASKS.pop(jid, None))
return job
async def _flush_job_progress(job_id: str, *, phase: str, summary: Mapping[str, Any]) -> None:
async with SessionLocal() as session:
await session.execute(
update(EntityDiscoveryJob)
.where(EntityDiscoveryJob.id == job_id, EntityDiscoveryJob.status.in_(OPEN_STATUSES))
.values(phase=phase, summary=dict(summary))
)
await session.commit()
async def _execute_job(job_id: str) -> None:
from langchain_core.messages import HumanMessage, SystemMessage
from app.engine.state import message_text
try:
async with SessionLocal() as session:
job = await session.get(EntityDiscoveryJob, job_id)
if job is None or job.status not in OPEN_STATUSES:
return
job.status = "running"
job.started_at = utcnow()
base_summary = dict(job.summary or {})
base_summary.update({"step_index": 1, "step_total": 4, "progress_pct": 18})
job.summary = base_summary
job.phase = "步骤 1/4:正在读取数据源表目录与口径参数…"
await session.commit()
prompt_body, scope_stats = await _build_discovery_prompt_body(
session,
source_ids=job.source_ids or [],
document_ids=job.document_ids or [],
attachment_ids=job.attachment_ids or [],
note=job.note or "",
)
model, model_id = await catalog_assistant.resolve_model(session)
doc_note = (
f"、{scope_stats['documents_scanned']} 份文档({scope_stats['chunks_scanned']} 段)"
if scope_stats["documents_scanned"] > 0
else ""
)
live_summary: dict[str, Any] = {
**scope_stats,
"model_id": model_id,
"step_index": 3,
"step_total": 4,
"progress_pct": 38,
"model_chars": 0,
"ops_detected": 0,
"candidate_keys": [],
}
job.summary = live_summary
job.phase = (
f"步骤 3/4:已汇总 {scope_stats['sources_scanned']} 个源({scope_stats['tables_scanned']} 张表){doc_note},"
"正在调用模型识别候选实体…"
)
await session.commit()
msgs = [
SystemMessage(content=DISCOVERY_SYSTEM_PROMPT),
HumanMessage(content=DISCOVERY_USER_PROMPT.format(body=prompt_body)),
]
first_token_seen = False
call_started = time.monotonic()
async def _thinking_heartbeat() -> None:
nonlocal live_summary
while not first_token_seen:
await asyncio.sleep(2.0)
if first_token_seen:
break
waited = int(time.monotonic() - call_started)
pct = min(52, 38 + waited // 2)
live_summary = {
**live_summary,
"step_index": 3,
"step_total": 4,
"progress_pct": pct,
}
await _flush_job_progress(
job_id,
phase="步骤 3/4:模型正在深度分析表目录与业务关系,等待生成结果…",
summary=live_summary,
)
hb_task = asyncio.create_task(_thinking_heartbeat())
try:
if hasattr(model, "astream") and not getattr(model, "invoke_reply", ""):
pieces: list[str] = []
last_flush = time.monotonic()
async for chunk in model.astream(msgs):
piece = message_text(chunk)
if piece:
first_token_seen = True
pieces.append(piece)
now_mono = time.monotonic()
if now_mono - last_flush >= 0.8 and pieces:
last_flush = now_mono
so_far = "".join(pieces)
op_count = len(re.findall(r'"op"\s*:', so_far))
found_keys = list(dict.fromkeys(re.findall(r'"entity_key"\s*:\s*"([a-z][a-z0-9_]{0,63})"', so_far)))
pct = min(86, 53 + min(33, len(so_far) // 80))
live_summary = {
**live_summary,
"step_index": 3,
"step_total": 4,
"progress_pct": pct,
"model_chars": len(so_far),
"ops_detected": op_count,
"candidate_keys": found_keys[:15],
}
phase_txt = (
f"步骤 3/4:模型正在识别候选实体(已识别 {op_count} 项操作,输出 {len(so_far)} 字)…"
if op_count > 0
else f"步骤 3/4:模型正在流式输出候选结果(已输出 {len(so_far)} 字)…"
)
await _flush_job_progress(job_id, phase=phase_txt, summary=live_summary)
raw_text = "".join(pieces)
else:
reply = await model.ainvoke(msgs)
first_token_seen = True
raw_text = message_text(reply)
finally:
first_token_seen = True
hb_task.cancel()
with contextlib.suppress(asyncio.CancelledError, Exception):
await hb_task
ops = parse_discovery_operations(raw_text)
found_keys_final = list(
dict.fromkeys(
str(op.get("entity_key") or "").strip().lower()
for op in ops
if isinstance(op, Mapping) and str(op.get("entity_key") or "").strip()
)
)
async with SessionLocal() as session:
job = await session.get(EntityDiscoveryJob, job_id)
if job is None or job.status == "cancelled":
return
job.phase = f"步骤 4/4:正在核验 {len(ops)} 项候选的表列真实性并写入待确认列表…"
job.summary = {
**live_summary,
"step_index": 4,
"step_total": 4,
"progress_pct": 90,
"model_chars": len(raw_text),
"ops_detected": len(ops),
"candidate_keys": found_keys_final[:15],
}
await session.commit()
if ops:
applied = await entity_patch.apply_entity_patch(
session,
ops,
mode="proposed",
actor=job.started_by,
)
else:
applied = {
"applied_count": 0,
"created_entities": 0,
"updated_entities": 0,
"bindings_changed": 0,
"caliber_params_changed": 0,
"members_changed": 0,
"entities": [],
"problems": [],
}
touched_entities = [
{"id": e["id"], "key": e["key"], "label": e["label"]}
for e in (applied.get("entities") or [])[:30]
if isinstance(e, Mapping) and e.get("id") and e.get("key")
]
problems = [str(p) for p in (applied.get("problems") or [])]
job.status = "succeeded"
job.phase = "扫描完成"
job.finished_at = utcnow()
job.summary = {
**scope_stats,
"model_id": model_id,
"step_index": 4,
"step_total": 4,
"progress_pct": 100,
"model_chars": len(raw_text),
"ops_detected": len(ops),
"candidate_keys": found_keys_final[:15],
"applied_count": applied["applied_count"],
"created_entities": applied["created_entities"],
"updated_entities": applied["updated_entities"],
"proposed_bindings": applied["bindings_changed"],
"proposed_params": applied["caliber_params_changed"],
"added_members": applied["members_changed"],
"skipped_count": len(problems),
"skipped_reasons": problems[:15],
"touched_entities": touched_entities,
}
await session.commit()
except asyncio.CancelledError:
async with SessionLocal() as session:
job = await session.get(EntityDiscoveryJob, job_id)
if job is not None and job.status in OPEN_STATUSES:
job.status = "cancelled"
job.phase = "已停止扫描"
job.finished_at = utcnow()
await session.commit()
raise
except Exception as exc: # noqa: BLE001
logger.exception("候选实体扫描任务 %s 失败", job_id)
async with SessionLocal() as session:
job = await session.get(EntityDiscoveryJob, job_id)
if job is not None and job.status in OPEN_STATUSES:
job.status = "failed"
job.phase = "扫描失败"
job.error = JOB_FAILED_TEXT.format(reason=first_line(exc) or describe_exception(exc))
job.finished_at = utcnow()
await session.commit()
async def cancel_job(session: AsyncSession, job_id: str) -> EntityDiscoveryJob:
job = await session.get(EntityDiscoveryJob, str(job_id or "").strip())
if job is None:
raise LookupError("候选实体扫描任务不存在")
if job.status not in OPEN_STATUSES:
raise DiscoveryFinished("该候选实体扫描任务已经结束")
task = _ACTIVE_TASKS.get(job.id)
if task is not None and not task.done():
task.cancel()
with contextlib.suppress(asyncio.CancelledError, Exception):
await asyncio.wait_for(asyncio.shield(task), timeout=2.0)
await session.refresh(job)
if job.status in OPEN_STATUSES:
job.status = "cancelled"
job.phase = "已停止扫描"
job.finished_at = utcnow()
await session.commit()
await session.refresh(job)
return job
async def startup(*, live: Any = None) -> int:
"""服务启动或清理孤儿任务时,将上次残留的未结束扫描任务标为 interrupted。"""
_ = live
running_ids = set(_ACTIVE_TASKS.keys())
async with SessionLocal() as session:
rows = list(
(
await session.execute(
select(EntityDiscoveryJob.id).where(EntityDiscoveryJob.status.in_(OPEN_STATUSES))
)
).scalars()
)
stale_ids = [jid for jid in rows if jid not in running_ids]
if stale_ids:
await session.execute(
update(EntityDiscoveryJob)
.where(EntityDiscoveryJob.id.in_(stale_ids), EntityDiscoveryJob.status.in_(OPEN_STATUSES))
.values(
status="interrupted",
phase="扫描中断",
error=INTERRUPTED_ERROR,
finished_at=utcnow(),
)
)
await session.commit()
return len(stale_ids)
async def shutdown() -> None:
"""服务关停时取消进程内正在执行的扫描任务并标为 interrupted。"""
tasks = list(_ACTIVE_TASKS.values())
for task in tasks:
if not task.done():
task.cancel()
for task in tasks:
with contextlib.suppress(asyncio.CancelledError, Exception):
await task
await startup()
+334
View File
@@ -0,0 +1,334 @@
"""标准业务实体维护的业务文档与主数据对照表附件服务(E1)。
支持两类文档来源:
1. 临时上传附件(POST /api/ontology/entities/attachments):复用 memory/parsing.extract(..., tables=True)
提取 .md / .txt / .docx / .pdf / .pptx / .html 以及 .xlsx / .csv / .tsv 编码对照表正文,
按 kb.chunk_text 切块(跨块自动保留 Markdown 表头)并存入内容寻址工件(entity_doc_attachment);
若勾选 save_to_kb=True,同步创建知识库 Document 并在后台完成切块向量索引,返回 document_id 供后续直接复用。
2. 已有知识库文档(Document + Chunk):若文档仍在后台切块中,自动回退按 doc.content 即时切块,无需等待向量计算。
"""
from __future__ import annotations
import asyncio
import logging
import re
from collections.abc import Sequence
from dataclasses import dataclass
from typing import Any
from fastapi import BackgroundTasks, UploadFile
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.core import artifact_store
from app.core.config import settings
from app.core.errors import explain, raw as raw_err
from app.db.base import SessionLocal
from app.db.models import Chunk, Document
from app.memory import kb
from app.memory.parsing import UnsupportedDocument, extract
logger = logging.getLogger(__name__)
ATTACHMENT_KIND = "entity_doc_attachment"
MAX_DOC_SEARCH_LIMIT = 15
#: 持有后台切块任务引用,防止事件循环提前回收
_KB_TASKS: set[asyncio.Task[Any]] = set()
_TOKEN_RE = re.compile(r"[A-Za-z0-9_]+|[\u4e00-\u9fff]{2,}")
class AttachmentTooLarge(ValueError):
"""上传附件超过系统大小上限。"""
class AttachmentInvalid(ValueError):
"""上传附件格式不支持或无有效文本。"""
@dataclass(slots=True)
class EntityDocBundle:
ref_id: str
kind: str # "attachment" | "kb"
title: str
mime: str
char_count: int
chunks: list[str]
document_id: str | None = None
async def _index_kb_document(doc_id: str) -> None:
"""后台对新建的知识库文档切块并建立向量与倒排索引。"""
async with SessionLocal() as session:
doc = await session.get(Document, doc_id)
if not doc:
return
try:
await kb.process_document(session, doc)
except Exception as exc: # noqa: BLE001
logger.warning("实体文档存入知识库索引失败(%s):%s", doc_id, raw_err(exc))
await session.rollback()
doc = await session.get(Document, doc_id)
if doc:
reason, hint = explain(exc)
doc.status = "failed"
doc.error = (f"文档分块或生成向量时出错:{reason}" + (f"。{hint}" if hint else ""))[:500]
await session.commit()
async def read_upload_limited(file: UploadFile) -> bytes:
limit = settings.max_upload_mb * 1024 * 1024
pieces: list[bytes] = []
total = 0
while chunk := await file.read(1024 * 1024):
total += len(chunk)
if total > limit:
raise AttachmentTooLarge(f"文件超过大小上限({settings.max_upload_mb} MB)")
pieces.append(chunk)
return b"".join(pieces)
async def ingest_entity_attachment(
session: AsyncSession,
*,
raw_bytes: bytes,
filename: str,
mime: str = "",
save_to_kb: bool = False,
collection: str = "default",
background: BackgroundTasks | None = None,
) -> dict[str, Any]:
"""解析上传的业务文档或主数据表格,生成附件工件并按需存入知识库。"""
clean_name = (filename or "上传文档").strip() or "上传文档"
clean_mime = (mime or "text/plain").strip() or "text/plain"
if not raw_bytes:
raise AttachmentInvalid("上传文件为空,请选择包含有效内容的文件")
try:
content = extract(raw_bytes, clean_name, clean_mime, tables=True)
except UnsupportedDocument as exc:
raise AttachmentInvalid(str(exc)) from None
content = (content or "").strip()
if not content:
raise AttachmentInvalid("未能从文件中提取出有效文字或表格内容")
chunks = kb.chunk_text(content)
if not chunks:
chunks = [content[:4000]]
doc_id: str | None = None
clean_collection = (collection or "default").strip() or "default"
if save_to_kb:
doc = await kb.create_document(
session,
collection=clean_collection,
title=clean_name,
content=content,
source=clean_name,
mime=clean_mime,
meta={"origin": "entity_agent"},
status="processing",
)
doc_id = doc.id
if background is not None:
background.add_task(_index_kb_document, doc.id)
else:
task = asyncio.create_task(_index_kb_document(doc.id))
_KB_TASKS.add(task)
task.add_done_callback(_KB_TASKS.discard)
payload = {
"title": clean_name,
"filename": clean_name,
"mime": clean_mime,
"char_count": len(content),
"chunks": chunks,
"document_id": doc_id,
}
attachment_id = await artifact_store.put_json(
payload,
kind=ATTACHMENT_KIND,
meta={"title": clean_name, "document_id": doc_id},
)
return {
"attachment_id": attachment_id,
"title": clean_name,
"mime": clean_mime,
"char_count": len(content),
"chunk_count": len(chunks),
"saved_to_kb": bool(doc_id),
"document_id": doc_id,
"collection": clean_collection if doc_id else None,
}
async def load_doc_bundles(
session: AsyncSession,
*,
attachment_ids: Sequence[str] = (),
document_ids: Sequence[str] = (),
) -> list[EntityDocBundle]:
"""统一加载临时附件与知识库文档的切块,按引用去重。"""
bundles: list[EntityDocBundle] = []
seen_doc_ids: set[str] = set()
seen_att_ids: set[str] = set()
for raw_att in attachment_ids or ():
att_id = str(raw_att or "").strip()
if not att_id or att_id in seen_att_ids:
continue
seen_att_ids.add(att_id)
try:
loaded = artifact_store.load(att_id)
except ValueError:
loaded = None
if not isinstance(loaded, dict) or not isinstance(loaded.get("chunks"), list):
continue
chunks = [str(c) for c in loaded["chunks"] if isinstance(c, str) and c.strip()]
if not chunks:
continue
linked_doc_id = str(loaded.get("document_id") or "").strip() or None
if linked_doc_id:
seen_doc_ids.add(linked_doc_id)
bundles.append(
EntityDocBundle(
ref_id=att_id,
kind="attachment",
title=str(loaded.get("title") or "上传附件"),
mime=str(loaded.get("mime") or "text/plain"),
char_count=int(loaded.get("char_count") or sum(len(c) for c in chunks)),
chunks=chunks,
document_id=linked_doc_id,
)
)
clean_doc_ids = [
str(d).strip()
for d in (document_ids or ())
if str(d or "").strip() and str(d).strip() not in seen_doc_ids
]
if clean_doc_ids:
seen_doc_ids.update(clean_doc_ids)
docs = {
row.id: row
for row in (await session.execute(select(Document).where(Document.id.in_(clean_doc_ids)))).scalars()
}
chunk_rows = list(
(
await session.execute(
select(Chunk)
.where(Chunk.document_id.in_(clean_doc_ids))
.order_by(Chunk.document_id.asc(), Chunk.ordinal.asc())
)
).scalars()
)
by_doc: dict[str, list[str]] = {}
for crow in chunk_rows:
if crow.content and crow.content.strip():
by_doc.setdefault(crow.document_id, []).append(crow.content)
for doc_id in clean_doc_ids:
doc = docs.get(doc_id)
if doc is None:
continue
pieces = by_doc.get(doc_id) or kb.chunk_text(doc.content or "")
if not pieces and (doc.content or "").strip():
pieces = [(doc.content or "").strip()[:4000]]
if not pieces:
continue
bundles.append(
EntityDocBundle(
ref_id=doc.id,
kind="kb",
title=doc.title or doc.source or "知识库文档",
mime=doc.mime or "text/plain",
char_count=len(doc.content or ""),
chunks=pieces,
document_id=doc.id,
)
)
return bundles
def _keywords(query: str) -> list[str]:
q = (query or "").strip().lower()
if not q:
return []
tokens = [m.group(0) for m in _TOKEN_RE.finditer(q)]
out: list[str] = []
seen: set[str] = set()
for tok in [q, *tokens]:
if tok and tok not in seen:
seen.add(tok)
out.append(tok)
return out
def search_doc_bundles(
bundles: Sequence[EntityDocBundle],
*,
query: str = "",
doc_ref: str = "",
start_chunk: int = 0,
limit: int = 6,
) -> list[dict[str, Any]]:
"""按关键词或分段序号检索已挂载文档片段,返回带文档标题与段号的原文片段。"""
cap = max(1, min(int(limit or 6), MAX_DOC_SEARCH_LIMIT))
offset = max(0, int(start_chunk or 0))
ref_norm = (doc_ref or "").strip().lower()
scoped = [
b
for b in bundles
if not ref_norm
or b.ref_id.lower() == ref_norm
or (b.document_id and b.document_id.lower() == ref_norm)
or ref_norm in b.title.lower()
]
if not scoped:
scoped = list(bundles)
terms = _keywords(query)
if not terms:
out: list[dict[str, Any]] = []
for bundle in scoped:
for idx in range(offset, len(bundle.chunks)):
out.append({
"doc_ref": bundle.ref_id,
"title": bundle.title,
"chunk_index": idx,
"total_chunks": len(bundle.chunks),
"citation": f"文档《{bundle.title}》第 {idx + 1} 段",
"content": bundle.chunks[idx],
})
if len(out) >= cap:
return out
return out
scored: list[tuple[int, int, int, EntityDocBundle, str]] = []
for b_idx, bundle in enumerate(scoped):
for c_idx, chunk in enumerate(bundle.chunks):
if c_idx < offset:
continue
low = chunk.lower()
score = 0
for t_idx, term in enumerate(terms):
hits = low.count(term)
if hits:
weight = 5 if t_idx == 0 else (3 if len(term) >= 3 else 1)
score += hits * weight
if score > 0:
scored.append((-score, b_idx, c_idx, bundle, chunk))
scored.sort(key=lambda item: (item[0], item[1], item[2]))
return [
{
"doc_ref": bundle.ref_id,
"title": bundle.title,
"chunk_index": c_idx,
"total_chunks": len(bundle.chunks),
"citation": f"文档《{bundle.title}》第 {c_idx + 1} 段",
"content": chunk,
}
for _neg, _b, c_idx, bundle, chunk in scored[:cap]
]
+841
View File
@@ -0,0 +1,841 @@
"""标准业务实体的受控增量补丁规划器与落库器(E1)。
供交互式「实体助手」(Entity Copilot)与后台「扫描发现候选实体」共用同一套核验与合并内核:
1. 物理真实性硬校验:表列绑定必须在已启用数据源的 schema_cache 中真实存在;口径参数映射必须在对应工作流入口节点中真实存在。
2. 敏感遮罩隔离:遮罩列(masked_columns)严禁写入或更新 members.source_codes 枚举字典。
3. 剥离自封审批态:模型传入的 status / approved / verified / actor / version 一律忽略重算。
4. 双模式落库:
- mode="confirmed"(人工在实体助手点「保存修改」):严格校验 expected_versions CAS 乐观锁,绑定与参数落为 confirmed / human。
- mode="proposed"(后台主动发现):新增绑定与参数一律落为 proposed / ai,绝不覆盖人工已确认、已验证或已驳回的项。
"""
from __future__ import annotations
import copy
from collections.abc import Mapping, Sequence
from dataclasses import dataclass, field
from typing import Any, Literal
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.artifact_store import canonical_json, content_hash
from app.data import entities, introspect
from app.data.engine import masked_columns
from app.db.base import new_id, utcnow
from app.db.models import DataSource, Workflow, WorkflowVersion
from app.db.models_ontology import CanonicalEntity
OP_TYPES = (
"upsert_entity",
"upsert_binding",
"remove_binding",
"upsert_member",
"upsert_caliber_param",
)
_FORBIDDEN_KEYS = frozenset({
"status", "approve", "approved", "approval", "approver", "verified", "verify",
"actor", "role", "review", "reviewed", "reviewed_by", "updated_by", "created_by", "version",
})
@dataclass(slots=True)
class EntityPatchChange:
id: str
op: str
entity_key: str
entity_label: str
summary: str
before: Any
after: Any
reason: str
op_payload: dict[str, Any]
def as_dict(self) -> dict[str, Any]:
return {
"id": self.id,
"op": self.op,
"entity_key": self.entity_key,
"entity_label": self.entity_label,
"summary": self.summary,
"before": self.before,
"after": self.after,
"reason": self.reason,
"op_payload": self.op_payload,
}
@dataclass(slots=True)
class EntityPatchPlan:
changes: list[EntityPatchChange] = field(default_factory=list)
problems: list[str] = field(default_factory=list)
expected_versions: dict[str, int] = field(default_factory=dict)
@dataclass(slots=True)
class _ValidationEnv:
sources_by_norm: dict[str, DataSource]
masked_by_source: dict[str, set[str]]
workflows_by_id: dict[str, Workflow]
workflows_by_name: dict[str, Workflow]
workflow_inputs: dict[str, set[str]]
workflow_cards: dict[str, set[str]]
entities_by_key: dict[str, CanonicalEntity]
def _strip_forbidden(raw: Mapping[str, Any]) -> dict[str, Any]:
return {k: v for k, v in raw.items() if k not in _FORBIDDEN_KEYS}
def _extract_graph_inputs_and_cards(graph: Mapping[str, Any] | None) -> tuple[set[str], set[str]]:
inputs: set[str] = set()
cards: set[str] = set()
if not isinstance(graph, Mapping):
return inputs, cards
for node in graph.get("nodes") or []:
if not isinstance(node, Mapping):
continue
ntype = str(node.get("type") or "").strip()
nid = str(node.get("id") or "").strip()
if ntype == "metrics" and nid:
cards.add(nid)
if ntype == "input":
data = node.get("data") if isinstance(node.get("data"), Mapping) else {}
cfg = data.get("config") if isinstance(data.get("config"), Mapping) else (
node.get("config") if isinstance(node.get("config"), Mapping) else {}
)
for f in cfg.get("fields") or []:
if isinstance(f, Mapping):
fname = str(f.get("name") or "").strip()
if fname:
inputs.add(fname)
return inputs, cards
async def _load_env(session: AsyncSession) -> _ValidationEnv:
from app.data import table_versions
src_rows = list(
(await session.execute(select(DataSource).where(DataSource.enabled.is_(True)).order_by(DataSource.name))).scalars()
)
sources_by_norm: dict[str, Any] = {}
masked_by_source: dict[str, set[str]] = {}
for row in src_rows:
try:
resolved = await table_versions.resolve_source(session, row)
except table_versions.SnapshotMissing:
resolved = row
key = row.name.strip().lower()
sources_by_norm[key] = resolved
sources_by_norm[row.id.strip().lower()] = resolved
masked_by_source[row.name.strip().lower()] = {
c.strip().lower() for c in masked_columns(row.options if isinstance(row.options, dict) else None)
}
wf_rows = list((await session.execute(select(Workflow))).scalars())
ver_rows = list(
(
await session.execute(
select(WorkflowVersion).where(WorkflowVersion.published_at.is_not(None))
)
).scalars()
)
workflows_by_id: dict[str, Workflow] = {}
name_counts: dict[str, int] = {}
for wf in wf_rows:
workflows_by_id[wf.id] = wf
if wf.name:
nkey = wf.name.strip().lower()
name_counts[nkey] = name_counts.get(nkey, 0) + 1
workflows_by_name = {
wf.name.strip().lower(): wf
for wf in wf_rows
if wf.name and name_counts.get(wf.name.strip().lower()) == 1
}
workflow_inputs: dict[str, set[str]] = {}
workflow_cards: dict[str, set[str]] = {}
for wf in wf_rows:
ins, cds = _extract_graph_inputs_and_cards(wf.graph if isinstance(wf.graph, dict) else None)
workflow_inputs.setdefault(wf.id, set()).update(ins)
workflow_cards.setdefault(wf.id, set()).update(cds)
for ver in ver_rows:
ins, cds = _extract_graph_inputs_and_cards(ver.graph if isinstance(ver.graph, dict) else None)
workflow_inputs.setdefault(ver.workflow_id, set()).update(ins)
workflow_cards.setdefault(ver.workflow_id, set()).update(cds)
ent_rows = await entities.list_entities(session)
entities_by_key = {row.key.strip().lower(): row for row in ent_rows}
return _ValidationEnv(
sources_by_norm=sources_by_norm,
masked_by_source=masked_by_source,
workflows_by_id=workflows_by_id,
workflows_by_name=workflows_by_name,
workflow_inputs=workflow_inputs,
workflow_cards=workflow_cards,
entities_by_key=entities_by_key,
)
def _is_column_masked(masked_set: set[str], table_name: str, column_name: str) -> bool:
col_low = column_name.strip().lower()
tbl_low = table_name.strip().lower()
short_tbl = tbl_low.rsplit(".", 1)[-1]
return (
col_low in masked_set
or f"{tbl_low}.{col_low}" in masked_set
or f"{short_tbl}.{col_low}" in masked_set
)
def _resolve_table_and_column(
source: DataSource,
raw_table: str,
raw_column: str,
) -> tuple[str | None, str | None, str | None]:
"""返回 (规范表名, 规范列名, 错误说明)。"""
cache = source.schema_cache if isinstance(source.schema_cache, Mapping) else {}
tables = cache.get("tables") if isinstance(cache.get("tables"), Mapping) else {}
if not tables:
return None, None, f"数据源「{source.name}」尚未探查表结构,无法核实表「{raw_table}」与列「{raw_column}」"
t_probe = raw_table.strip()
matched_table: str | None = None
meta: Mapping[str, Any] | None = None
if t_probe in tables:
matched_table, meta = t_probe, tables[t_probe]
else:
t_low = t_probe.lower()
t_short = t_low.rsplit(".", 1)[-1]
for tname, tmeta in tables.items():
if not isinstance(tmeta, Mapping):
continue
qual = str(tmeta.get("qualified") or tname).strip()
if tname.lower() == t_low or qual.lower() == t_low or tname.lower() == t_short:
matched_table, meta = tname, tmeta
break
if meta is None or matched_table is None:
sample = "、".join(list(tables.keys())[:15])
return None, None, f"数据源「{source.name}」中不存在表「{raw_table}」(已探查表如:{sample})"
cols = introspect.table_columns(dict(meta))
c_low = raw_column.strip().lower()
for col in cols:
cname = str(col.get("name") or "").strip()
if cname.lower() == c_low:
return matched_table, cname, None
col_sample = "、".join(str(c.get("name")) for c in cols[:20])
return None, None, f"数据源「{source.name}」表「{matched_table}」中不存在列「{raw_column}」(可用列:{col_sample})"
def _working_from_row(row: CanonicalEntity) -> dict[str, Any]:
return {
"id": row.id,
"key": row.key,
"label": row.label,
"description": str(row.description or ""),
"aliases": list(row.aliases or []),
"members": [copy.deepcopy(dict(m)) for m in (row.members or []) if isinstance(m, Mapping)],
"bindings": [copy.deepcopy(dict(b)) for b in (row.bindings or []) if isinstance(b, Mapping)],
"caliber_params": [copy.deepcopy(dict(p)) for p in (row.caliber_params or []) if isinstance(p, Mapping)],
"version": int(row.version or 1),
"is_new": False,
}
def _change_id(op: str, entity_key: str, identity: str) -> str:
return content_hash(canonical_json({"op": op, "entity_key": entity_key, "id": identity}))[:16]
def _normalize_single_op(
raw_op: Any,
*,
index: int,
env: _ValidationEnv,
working: dict[str, dict[str, Any]],
mode: Literal["confirmed", "proposed"],
) -> tuple[EntityPatchChange | None, str | None]:
if not isinstance(raw_op, Mapping):
return None, f"第 {index + 1} 项操作需要是对象"
cleaned = _strip_forbidden(raw_op)
op = str(cleaned.get("op") or "").strip()
if op not in OP_TYPES:
return None, f"第 {index + 1} 项操作类型「{op}」无效,支持:{', '.join(OP_TYPES)}"
if mode == "proposed" and op == "remove_binding":
return None, "后台发现模式不允许删除既有表列绑定(remove_binding)"
raw_key = cleaned.get("entity_key") or cleaned.get("key")
try:
entity_key = entities.normalize_entity_key(raw_key, field="entity_key")
except entities.EntityInvalid as exc:
return None, f"第 {index + 1} 项:{exc.message}"
reason = str(cleaned.get("reason") or "").strip()[:500]
note = str(cleaned.get("note") or "").strip()[:500]
# 初始化当前实体工作副本
state = working.get(entity_key)
if state is None and entity_key in env.entities_by_key:
state = _working_from_row(env.entities_by_key[entity_key])
working[entity_key] = state
if op == "upsert_entity":
raw_label = str(cleaned.get("label") or cleaned.get("entity_label") or "").strip()
if state is None and not raw_label:
return None, f"新建标准实体「{entity_key}」缺少中文名称(label)"
label = raw_label or (state["label"] if state else entity_key)
if len(label) > 100:
return None, f"标准实体「{entity_key}」名称不能超过 100 个字符"
desc = str(cleaned.get("description") if "description" in cleaned else (state["description"] if state else "") or "").strip()
raw_aliases = cleaned.get("aliases")
if raw_aliases is None:
new_aliases = list(state["aliases"]) if state else []
elif isinstance(raw_aliases, list):
merged = list(state["aliases"]) if (state and mode == "proposed") else []
seen_a = {a.lower() for a in merged}
for item in raw_aliases:
if isinstance(item, str) and item.strip() and item.strip().lower() not in seen_a:
seen_a.add(item.strip().lower())
merged.append(item.strip())
new_aliases = merged
else:
return None, f"标准实体「{entity_key}」的 aliases 需要是字符串数组"
if state is None:
working[entity_key] = {
"id": None,
"key": entity_key,
"label": label,
"description": desc,
"aliases": new_aliases,
"members": [],
"bindings": [],
"caliber_params": [],
"version": 0,
"is_new": True,
}
payload = {
"op": "upsert_entity",
"entity_key": entity_key,
"label": label,
"description": desc,
"aliases": new_aliases,
"reason": reason,
}
return EntityPatchChange(
id=_change_id(op, entity_key, "meta"),
op=op,
entity_key=entity_key,
entity_label=label,
summary=f"新建标准业务实体「{label}({entity_key})」",
before=None,
after={"label": label, "description": desc, "aliases": new_aliases},
reason=reason,
op_payload=payload,
), None
# 更新已有实体元信息(后台发现模式下不覆盖已有中文名与非空说明)
target_label = state["label"] if mode == "proposed" else label
target_desc = (state["description"] or desc) if mode == "proposed" else desc
before_meta = {
"label": state["label"],
"description": state["description"],
"aliases": list(state["aliases"]),
}
after_meta = {
"label": target_label,
"description": target_desc,
"aliases": new_aliases,
}
if before_meta == after_meta:
return None, None
state["label"] = target_label
state["description"] = target_desc
state["aliases"] = new_aliases
payload = {
"op": "upsert_entity",
"entity_key": entity_key,
"label": target_label,
"description": target_desc,
"aliases": new_aliases,
"reason": reason,
}
return EntityPatchChange(
id=_change_id(op, entity_key, "meta"),
op=op,
entity_key=entity_key,
entity_label=target_label,
summary=f"更新标准实体「{target_label}({entity_key})」基础信息",
before=before_meta,
after=after_meta,
reason=reason,
op_payload=payload,
), None
# 其余子项操作:若实体尚不存在,允许随操作附带 entity_label 自动初始化,否则明确报错
if state is None:
fallback_label = str(cleaned.get("entity_label") or "").strip()
if not fallback_label:
return (
None,
f"标准实体「{entity_key}」尚不存在,请先提供一条 upsert_entity(含 label 中文名称)或在操作中提供 entity_label",
)
state = {
"id": None,
"key": entity_key,
"label": fallback_label[:100],
"description": "",
"aliases": [],
"members": [],
"bindings": [],
"caliber_params": [],
"version": 0,
"is_new": True,
}
working[entity_key] = state
if op == "upsert_binding":
raw_src = str(cleaned.get("source_name") or "").strip()
raw_tbl = str(cleaned.get("table_name") or "").strip()
raw_col = str(cleaned.get("column_name") or "").strip()
if not raw_src or not raw_tbl or not raw_col:
return None, f"实体「{entity_key}」的表列绑定须完整填写 source_name、table_name、column_name"
src_row = env.sources_by_norm.get(raw_src.lower())
if src_row is None:
avail_src = "、".join(sorted({r.name for r in env.sources_by_norm.values()})) or "无"
return None, f"未找到已启用的数据源「{raw_src}」(当前可用数据源:{avail_src})"
canon_tbl, canon_col, problem = _resolve_table_and_column(src_row, raw_tbl, raw_col)
if problem or canon_tbl is None or canon_col is None:
return None, problem or "表列核验未通过"
target_status = "confirmed" if mode == "confirmed" else "proposed"
target_source = "human" if mode == "confirmed" else "ai"
bkey = (src_row.name.lower(), canon_tbl.lower(), canon_col.lower())
existing_idx = next(
(
i
for i, b in enumerate(state["bindings"])
if (
str(b.get("source_name") or "").lower(),
str(b.get("table_name") or "").lower(),
str(b.get("column_name") or "").lower(),
)
== bkey
),
None,
)
if existing_idx is not None:
cur = state["bindings"][existing_idx]
cur_status = str(cur.get("status") or "confirmed")
if mode == "proposed" and cur_status in ("confirmed", "verified", "rejected"):
return None, None
if (
cur_status == target_status
and str(cur.get("note") or "") == (note or str(cur.get("note") or ""))
and str(cur.get("reason") or "") == (reason or str(cur.get("reason") or ""))
):
return None, None
before_val = copy.deepcopy(cur)
updated = {
**cur,
"source_id": src_row.id,
"source_name": src_row.name,
"table_name": canon_tbl,
"column_name": canon_col,
"status": target_status,
"source": target_source,
"note": note or str(cur.get("note") or ""),
"reason": reason or str(cur.get("reason") or ""),
"updated_at": utcnow().isoformat(),
}
state["bindings"][existing_idx] = updated
else:
before_val = None
updated = {
"id": new_id()[:12],
"source_id": src_row.id,
"source_name": src_row.name,
"table_name": canon_tbl,
"column_name": canon_col,
"status": target_status,
"source": target_source,
"note": note,
"reason": reason,
"updated_at": utcnow().isoformat(),
}
state["bindings"].append(updated)
payload = {
"op": "upsert_binding",
"entity_key": entity_key,
"entity_label": state["label"],
"source_name": src_row.name,
"table_name": canon_tbl,
"column_name": canon_col,
"note": note,
"reason": reason,
}
action_word = "新增" if before_val is None else "更新"
return EntityPatchChange(
id=_change_id(op, entity_key, f"{src_row.name}.{canon_tbl}.{canon_col}"),
op=op,
entity_key=entity_key,
entity_label=state["label"],
summary=f"{action_word}实体「{state['label']}」表列绑定:{src_row.name}.{canon_tbl}.{canon_col}",
before=before_val,
after=updated,
reason=reason,
op_payload=payload,
), None
if op == "remove_binding":
raw_src = str(cleaned.get("source_name") or "").strip()
raw_tbl = str(cleaned.get("table_name") or "").strip()
raw_col = str(cleaned.get("column_name") or "").strip()
bkey = (raw_src.lower(), raw_tbl.lower(), raw_col.lower())
existing_idx = next(
(
i
for i, b in enumerate(state["bindings"])
if (
str(b.get("source_name") or "").lower(),
str(b.get("table_name") or "").lower(),
str(b.get("column_name") or "").lower(),
)
== bkey
),
None,
)
if existing_idx is None:
return None, f"实体「{entity_key}」中不存在待移除的绑定「{raw_src}.{raw_tbl}.{raw_col}」"
removed = state["bindings"].pop(existing_idx)
payload = {
"op": "remove_binding",
"entity_key": entity_key,
"source_name": removed.get("source_name"),
"table_name": removed.get("table_name"),
"column_name": removed.get("column_name"),
"reason": reason,
}
return EntityPatchChange(
id=_change_id(op, entity_key, f"{raw_src}.{raw_tbl}.{raw_col}"),
op=op,
entity_key=entity_key,
entity_label=state["label"],
summary=f"移除实体「{state['label']}」表列绑定:{removed.get('source_name')}.{removed.get('table_name')}.{removed.get('column_name')}",
before=removed,
after=None,
reason=reason,
op_payload=payload,
), None
if op == "upsert_member":
val = str(cleaned.get("value") or "").strip()
if not val:
return None, f"实体「{entity_key}」成员映射缺少标准值(value)"
label = str(cleaned.get("label") or val).strip() or val
raw_aliases = cleaned.get("aliases")
if raw_aliases is not None and not isinstance(raw_aliases, list):
return None, f"实体「{entity_key}」成员「{val}」的 aliases 需要是字符串数组"
raw_codes = cleaned.get("source_codes")
if raw_codes is not None and not isinstance(raw_codes, Mapping):
return None, f"实体「{entity_key}」成员「{val}」的 source_codes 需要是「数据源名 → 编码」对象"
canon_codes: dict[str, str] = {}
for src_k, code_v in (raw_codes or {}).items():
s_str = str(src_k or "").strip()
c_str = str(code_v or "").strip()
if not s_str or not c_str:
continue
src_row = env.sources_by_norm.get(s_str.lower())
canon_src = src_row.name if src_row is not None else s_str
# 检查该实体在此数据源上的绑定是否涉及敏感遮罩列
masked_set = env.masked_by_source.get(canon_src.lower(), set())
if masked_set:
for b in state["bindings"]:
if str(b.get("source_name") or "").lower() == canon_src.lower():
if _is_column_masked(masked_set, str(b.get("table_name") or ""), str(b.get("column_name") or "")):
return (
None,
f"数据源「{canon_src}」表列「{b.get('table_name')}.{b.get('column_name')}」为敏感遮罩列,禁止为其配置成员编码映射",
)
canon_codes[canon_src] = c_str
m_idx = next(
(i for i, m in enumerate(state["members"]) if str(m.get("value") or "").strip().lower() == val.lower()),
None,
)
if m_idx is not None:
cur_m = state["members"][m_idx]
before_m = copy.deepcopy(cur_m)
merged_aliases = list(cur_m.get("aliases") or [])
seen_ma = {a.lower() for a in merged_aliases if isinstance(a, str)}
for item in raw_aliases or []:
if isinstance(item, str) and item.strip() and item.strip().lower() not in seen_ma:
seen_ma.add(item.strip().lower())
merged_aliases.append(item.strip())
merged_codes = dict(cur_m.get("source_codes") or {})
for cs, cv in canon_codes.items():
if mode == "proposed" and cs in merged_codes and str(merged_codes[cs]).strip():
continue
merged_codes[cs] = cv
target_mlabel = str(cur_m.get("label") or val) if mode == "proposed" else label
after_m = {
"value": str(cur_m.get("value") or val),
"label": target_mlabel,
"aliases": merged_aliases,
"source_codes": merged_codes,
}
if before_m == after_m:
return None, None
state["members"][m_idx] = after_m
else:
before_m = None
clean_aliases = []
seen_ma = set()
for item in raw_aliases or []:
if isinstance(item, str) and item.strip() and item.strip().lower() not in seen_ma:
seen_ma.add(item.strip().lower())
clean_aliases.append(item.strip())
after_m = {
"value": val,
"label": label,
"aliases": clean_aliases,
"source_codes": canon_codes,
}
state["members"].append(after_m)
payload = {
"op": "upsert_member",
"entity_key": entity_key,
"entity_label": state["label"],
"value": after_m["value"],
"label": after_m["label"],
"aliases": after_m["aliases"],
"source_codes": after_m["source_codes"],
"reason": reason,
}
code_desc = ",".join(f"{k}={v}" for k, v in after_m["source_codes"].items())
suffix = f"({code_desc})" if code_desc else ""
return EntityPatchChange(
id=_change_id(op, entity_key, after_m["value"].lower()),
op=op,
entity_key=entity_key,
entity_label=state["label"],
summary=f"{'新增' if before_m is None else '更新'}实体「{state['label']}」成员「{after_m['label']}({after_m['value']})」{suffix}",
before=before_m,
after=after_m,
reason=reason,
op_payload=payload,
), None
# upsert_caliber_param
raw_wf = str(cleaned.get("workflow_id") or "").strip()
raw_card = str(cleaned.get("card_id") or "").strip()
raw_param = str(cleaned.get("param_name") or "").strip()
if not raw_wf or not raw_param:
return None, f"实体「{entity_key}」口径参数映射须提供 workflow_id 与 param_name"
wf_row = env.workflows_by_id.get(raw_wf) or env.workflows_by_name.get(raw_wf.lower())
if wf_row is None:
return None, f"不存在工作流「{raw_wf}」,请核对已发布口径的工作流 ID"
valid_inputs = env.workflow_inputs.get(wf_row.id, set())
canon_param = next((p for p in valid_inputs if p.lower() == raw_param.lower()), None)
if canon_param is None:
sample_in = "、".join(sorted(valid_inputs)) or "无入口参数"
return None, f"工作流「{wf_row.name}({wf_row.id})」入口节点不存在参数「{raw_param}」(可用参数:{sample_in})"
valid_cards = env.workflow_cards.get(wf_row.id, set())
canon_card = ""
if raw_card:
matched_card = next((c for c in valid_cards if c.lower() == raw_card.lower()), None)
if matched_card is None:
sample_cd = "、".join(sorted(valid_cards)) or "无口径卡节点"
return None, f"工作流「{wf_row.name}({wf_row.id})」中不存在口径卡「{raw_card}」(可用口径卡:{sample_cd})"
canon_card = matched_card
target_status = "confirmed" if mode == "confirmed" else "proposed"
target_source = "human" if mode == "confirmed" else "ai"
pkey = (wf_row.id, canon_card.lower(), canon_param.lower())
p_idx = next(
(
i
for i, p in enumerate(state["caliber_params"])
if (
str(p.get("workflow_id") or ""),
str(p.get("card_id") or "").lower(),
str(p.get("param_name") or "").lower(),
)
== pkey
),
None,
)
if p_idx is not None:
cur_p = state["caliber_params"][p_idx]
cur_status = str(cur_p.get("status") or "confirmed")
if mode == "proposed" and cur_status in ("confirmed", "verified", "rejected"):
return None, None
if cur_status == target_status and str(cur_p.get("note") or "") == (note or str(cur_p.get("note") or "")):
return None, None
before_p = copy.deepcopy(cur_p)
after_p = {
**cur_p,
"workflow_id": wf_row.id,
"card_id": canon_card,
"param_name": canon_param,
"status": target_status,
"source": target_source,
"note": note or reason or str(cur_p.get("note") or ""),
}
state["caliber_params"][p_idx] = after_p
else:
before_p = None
after_p = {
"id": new_id()[:12],
"workflow_id": wf_row.id,
"card_id": canon_card,
"param_name": canon_param,
"status": target_status,
"source": target_source,
"note": note or reason,
}
state["caliber_params"].append(after_p)
payload = {
"op": "upsert_caliber_param",
"entity_key": entity_key,
"entity_label": state["label"],
"workflow_id": wf_row.id,
"card_id": canon_card,
"param_name": canon_param,
"note": note or reason,
"reason": reason,
}
card_part = f" · {canon_card}" if canon_card else ""
return EntityPatchChange(
id=_change_id(op, entity_key, f"{wf_row.id}:{canon_card}:{canon_param}"),
op=op,
entity_key=entity_key,
entity_label=state["label"],
summary=f"{'新增' if before_p is None else '更新'}实体「{state['label']}」口径入参映射:{wf_row.name}{card_part} → {canon_param}",
before=before_p,
after=after_p,
reason=reason,
op_payload=payload,
), None
async def plan_entity_patch(
session: AsyncSession,
raw_ops: Sequence[Any],
*,
pending_ops: Sequence[Any] = (),
mode: Literal["confirmed", "proposed"] = "confirmed",
) -> EntityPatchPlan:
"""校验一批增量修改操作并生成结构化变更预览,不写入数据库。"""
env = await _load_env(session)
working: dict[str, dict[str, Any]] = {}
# 先回放界面待保存的操作,保证后续新增实体/成员在此基础上叠加
for p_idx, pop in enumerate(pending_ops or ()):
_normalize_single_op(pop, index=p_idx, env=env, working=working, mode=mode)
plan = EntityPatchPlan()
for idx, item in enumerate(raw_ops or ()):
change, problem = _normalize_single_op(item, index=idx, env=env, working=working, mode=mode)
if problem:
plan.problems.append(problem)
continue
if change is not None:
plan.changes.append(change)
touched_keys = {c.entity_key for c in plan.changes}
for ekey in sorted(touched_keys):
existing = env.entities_by_key.get(ekey)
plan.expected_versions[ekey] = int(existing.version or 1) if existing is not None else 0
return plan
async def apply_entity_patch(
session: AsyncSession,
raw_ops: Sequence[Any],
*,
mode: Literal["confirmed", "proposed"],
actor: str | None = None,
expected_versions: Mapping[str, int] | None = None,
) -> dict[str, Any]:
"""将一批增量操作合并并经 entities.create_entity / update_entity 统一落库。"""
if not raw_ops:
raise entities.EntityInvalid("没有可保存的实体修改项", field="operations")
env = await _load_env(session)
working: dict[str, dict[str, Any]] = {}
changes: list[EntityPatchChange] = []
problems: list[str] = []
for idx, item in enumerate(raw_ops):
change, problem = _normalize_single_op(item, index=idx, env=env, working=working, mode=mode)
if problem:
problems.append(problem)
continue
if change is not None:
changes.append(change)
if mode == "confirmed" and problems:
raise entities.EntityInvalid(";".join(problems), field="operations")
touched_keys = list(dict.fromkeys(c.entity_key for c in changes))
if mode == "confirmed" and expected_versions is not None:
for ekey in touched_keys:
if ekey not in expected_versions:
continue
exp_ver = int(expected_versions[ekey])
cur_row = env.entities_by_key.get(ekey)
cur_ver = int(cur_row.version or 1) if cur_row is not None else 0
if exp_ver != cur_ver:
label = cur_row.label if cur_row is not None else ekey
raise entities.EntityConflict(
f"标准实体「{label}」在对话期间已被修改(基准 v{exp_ver},当前 v{cur_ver}),请刷新后再应用修改",
field="expected_versions",
)
created_count = 0
updated_count = 0
saved_entities: list[dict[str, Any]] = []
for ekey in touched_keys:
state = working.get(ekey)
if state is None:
continue
payload: dict[str, Any] = {
"key": state["key"],
"label": state["label"],
"description": state["description"],
"aliases": state["aliases"],
"members": state["members"],
"bindings": state["bindings"],
"caliber_params": state["caliber_params"],
}
cur_row = env.entities_by_key.get(ekey)
if cur_row is None:
row = await entities.create_entity(session, payload, actor=actor)
created_count += 1
else:
payload["version"] = int(cur_row.version or 1)
row = await entities.update_entity(session, cur_row.id, payload, actor=actor)
updated_count += 1
saved_entities.append(entities.entity_to_dict(row))
proposed_bindings = sum(1 for c in changes if c.op == "upsert_binding")
proposed_params = sum(1 for c in changes if c.op == "upsert_caliber_param")
added_members = sum(1 for c in changes if c.op == "upsert_member")
return {
"applied_count": len(changes),
"created_entities": created_count,
"updated_entities": updated_count,
"bindings_changed": proposed_bindings,
"caliber_params_changed": proposed_params,
"members_changed": added_members,
"entities": saved_entities,
"problems": problems,
}
+463
View File
@@ -0,0 +1,463 @@
"""跨数据源健康与漂移快速探针(source_health,W1-b)。
用于指标异动与跨源穿透时的第 1 顺位核查(先排数,后归因):
1. 读取 `SourceChange` 待处理变化记录(`date_stalled`、`date_gap`、`volume_drop`、`rows_drop`、`codes_new`、
`codes_shift`、`coverage_drop`、`table_removed`、`column_removed`、`column_type` 等);
2. 对目标表非遮罩的时间列做有界只读鲜度与时间窗覆盖探查(`PROBE_TIMEOUT_S = 10.0s`,`guard.check + run_query`),
超时或失败优雅降级为 `unverified`,绝不拖垮主运行;
3. 严守遮罩铁律(铁律 3):绝不探查或返回任何 `mask_columns` 列的值;`restricted=True`(普通 user 角色)时自动隐去
物理表名、物理列名与底层码值细节,只输出业务数据源说明与高层健康判定。
"""
from __future__ import annotations
import asyncio
from collections.abc import Iterable, Mapping
from datetime import date, datetime
from typing import Any
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.core import artifact_store
from app.data import schema_diff as sd
from app.data import source_changes as sc
from app.data.coverage import _as_day
from app.data.engine import QueryLimits, masked_columns, run_query
from app.data.guard import check, is_write
from app.data.names import name_key
from app.db.models import DataSource, Workflow, WorkflowVersion
from app.db.models_sources import SourceChange
ARTIFACT_KIND = "source_health"
LEDGER_KIND = "source_health"
CATALOG_PREFIX = "H"
PROBE_TIMEOUT_S = 10.0
MAX_TABLES_PROBED = 6
HEALTHY = "healthy"
DEGRADED = "degraded"
ANOMALOUS = "anomalous"
UNVERIFIED = "unverified"
VERDICTS = (HEALTHY, DEGRADED, ANOMALOUS, UNVERIFIED)
_SEVERE_CHANGE_KINDS = frozenset({
sd.TABLE_REMOVED,
sd.COLUMN_REMOVED,
sd.COLUMN_TYPE,
sc.DATE_STALLED,
sc.DATE_GAP,
sc.VOLUME_DROP,
sc.ROWS_DROP,
sc.COVERAGE_DROP,
})
_WARN_CHANGE_KINDS = frozenset({
sd.TABLE_REFLECT_FAILED,
sd.TABLE_LEFT_SCOPE,
sc.CODES_NEW,
sc.CODES_SHIFT,
sc.ROWS_JUMP,
})
def _quote_ident(name: str, dialect: str) -> str:
clean = str(name)
if dialect in ("mysql", "mariadb"):
return "`" + clean.replace("`", "``") + "`"
return '"' + clean.replace('"', '""') + '"'
def _date_columns_of(table_meta: Mapping[str, Any], masked: set[str]) -> list[str]:
cols = table_meta.get("columns")
if not isinstance(cols, list):
return []
out: list[str] = []
for col in cols:
if not isinstance(col, Mapping):
continue
cname = str(col.get("name") or "").strip()
if not cname or cname.lower() in masked:
continue
ctype = str(col.get("type") or "").lower()
low = cname.lower()
if any(tok in ctype for tok in ("date", "time", "timestamp")) or any(
low == k or low.endswith(f"_{k}") or low.startswith(f"{k}_")
for k in ("date", "day", "dt", "shift_date", "production_day", "biz_date", "created_at")
):
out.append(cname)
return out[:2]
async def _resolve_caliber_targets(
session: AsyncSession,
*,
workflow_id: str,
version: int | None,
card_id: str | None,
metric_id: str | None,
) -> list[tuple[str, list[str]]]:
"""根据已发布(或指定版本)口径自动反查 (source_name, [table_names])。"""
from app.ledger.ontology import workflow_ontology_index
wf = await session.get(Workflow, workflow_id)
if wf is None:
return []
ver = version if version is not None else wf.published_version or wf.version
snap = (
await session.execute(
select(WorkflowVersion).where(
WorkflowVersion.workflow_id == workflow_id,
WorkflowVersion.version == int(ver),
)
)
).scalar_one_or_none()
graph = (snap.graph if snap is not None else wf.graph) or {}
idx = workflow_ontology_index(workflow_id=workflow.id if (workflow := wf) else workflow_id, version=int(ver), graph=graph)
by_src: dict[str, set[str]] = {}
for card in idx["cards"]:
if card_id and card["card_id"] != card_id:
continue
if metric_id:
for m in card["metrics"]:
if m["metric_id"] == metric_id:
for item in m["lineage"]:
by_src.setdefault(item["source"], set()).update(item["tables"])
else:
for src, tbls in card["tables_by_source"].items():
by_src.setdefault(src, set()).update(tbls)
return [(src, sorted(tbls)) for src, tbls in sorted(by_src.items())]
def _parse_window_day(raw: str | date | None) -> date | None:
if raw is None:
return None
if isinstance(raw, datetime):
return raw.date()
if isinstance(raw, date):
return raw
d = _as_day(str(raw).strip())
return d.day if d is not None else None
async def _probe_table_coverage(
source: DataSource,
table_name: str,
table_meta: Mapping[str, Any],
*,
masked: set[str],
start_day: date | None,
end_day: date | None,
) -> dict[str, Any]:
"""对单张表做只读行数与非遮罩日期边界探查,超时或报错返回 status='unverified'。"""
dialect = (source.kind or "sqlite").lower()
q_table = _quote_ident(table_name, dialect)
date_cols = _date_columns_of(table_meta, masked)
select_parts = ["COUNT(*) AS row_cnt"]
for idx, col in enumerate(date_cols):
qc = _quote_ident(col, dialect)
select_parts.append(f"MIN({qc}) AS min_d{idx}")
select_parts.append(f"MAX({qc}) AS max_d{idx}")
sql = f"SELECT {', '.join(select_parts)} FROM {q_table}"
if is_write(sql, dialect=dialect):
return {"table": table_name, "status": UNVERIFIED, "reason": "探查语句未通过只读校验"}
check(sql, readonly=True, source_name=source.name, dialect=dialect)
try:
res = await asyncio.wait_for(
run_query(source, sql, limits=QueryLimits(max_rows=5, timeout_seconds=PROBE_TIMEOUT_S)),
timeout=PROBE_TIMEOUT_S + 1.0,
)
except Exception: # noqa: BLE001 - 探查失败优雅降级,绝不中断主流程
return {"table": table_name, "status": UNVERIFIED, "reason": "实时探查超时或暂时不可达"}
if not res.rows:
return {"table": table_name, "status": ANOMALOUS, "issue": "empty_table", "row_count": 0}
first = res.rows[0]
row_cnt = int(first[0] or 0) if first and first[0] is not None else 0
if row_cnt == 0:
return {"table": table_name, "status": ANOMALOUS, "issue": "empty_table", "row_count": 0}
ranges: list[dict[str, Any]] = []
window_issues: list[dict[str, Any]] = []
for idx, col in enumerate(date_cols):
pos = 1 + idx * 2
raw_min = first[pos] if pos < len(first) else None
raw_max = first[pos + 1] if (pos + 1) < len(first) else None
d_min = _as_day(raw_min)
d_max = _as_day(raw_max)
ranges.append({
"column": col,
"min": d_min.day.isoformat() if d_min else (str(raw_min) if raw_min is not None else None),
"max": d_max.day.isoformat() if d_max else (str(raw_max) if raw_max is not None else None),
})
if start_day and d_max and d_max.day < start_day:
window_issues.append({
"kind": sc.DATE_STALLED,
"severity": "action",
"column": col,
"max_date": d_max.day.isoformat(),
"requested_start": start_day.isoformat(),
})
elif end_day and d_min and d_min.day >= end_day:
window_issues.append({
"kind": sc.DATE_GAP,
"severity": "action",
"column": col,
"min_date": d_min.day.isoformat(),
"requested_end": end_day.isoformat(),
})
return {
"table": table_name,
"status": ANOMALOUS if window_issues else HEALTHY,
"row_count": row_cnt,
"ranges": ranges,
"window_issues": window_issues,
}
def _render_phrase(source_title: str, verdict: str, issues: list[dict[str, Any]], *, probe_unverified: bool) -> str:
if verdict == ANOMALOUS:
labels = list(dict.fromkeys(str(i.get("label") or i.get("kind") or "数据异常") for i in issues))
shown = "、".join(labels[:3])
return f"数据源「{source_title}」检测到数据健康异常({shown}),当前异动优先归因于上游同步或主数据质量问题,不宜直接归结为业务实质变化。"
if verdict == DEGRADED:
labels = list(dict.fromkeys(str(i.get("label") or i.get("kind") or "取值变化") for i in issues))
shown = "、".join(labels[:3])
return f"数据源「{source_title}」存在待审阅的数据变化提醒({shown}),跨源对比与归因时需留意码值或分布漂移。"
if verdict == UNVERIFIED or probe_unverified:
return f"数据源「{source_title}」未发现已登记的变化告警,但实时范围探查未完成(状态记为待核实)。"
return f"数据源「{source_title}」同步状态与查询区间覆盖正常,未发现断更、缺口或未处理的结构漂移。"
async def check_source_health(
session: AsyncSession,
*,
source_name: str | None = None,
tables: Iterable[str] | None = None,
workflow_id: str | None = None,
version: int | None = None,
card_id: str | None = None,
metric_id: str | None = None,
start: str | date | None = None,
end: str | date | None = None,
restricted: bool = False,
live_probe: bool = True,
) -> dict[str, Any]:
"""检查目标数据源(或目标口径底层依赖源)的健康度与数据漂移,返回确定性结构化结论。"""
targets: list[tuple[str, list[str]]] = []
if source_name and str(source_name).strip():
explicit_tables = [str(t).strip() for t in (tables or ()) if str(t).strip()]
targets.append((str(source_name).strip(), explicit_tables))
elif workflow_id and str(workflow_id).strip():
targets = await _resolve_caliber_targets(
session,
workflow_id=str(workflow_id).strip(),
version=version,
card_id=card_id,
metric_id=metric_id,
)
if not targets:
return {
"kind": ARTIFACT_KIND,
"verdict": UNVERIFIED,
"sources": [],
"issues": [],
"phrase": "未能定位目标数据源,健康状态记为未核实。",
}
start_day = _parse_window_day(start)
end_day = _parse_window_day(end)
source_reports: list[dict[str, Any]] = []
all_issues: list[dict[str, Any]] = []
overall_rank = 0 # 0 healthy, 1 unverified, 2 degraded, 3 anomalous
rank_to_verdict = {0: HEALTHY, 1: UNVERIFIED, 2: DEGRADED, 3: ANOMALOUS}
for src_name, target_tables in targets:
src = (
await session.execute(select(DataSource).where(DataSource.name == src_name).limit(1))
).scalar_one_or_none()
if src is None or not src.enabled:
overall_rank = max(overall_rank, 3 if src is not None and not src.enabled else 1)
status = ANOMALOUS if (src is not None and not src.enabled) else UNVERIFIED
issue_label = "数据源已停用" if status == ANOMALOUS else "数据源未找到"
issue = {"kind": "source_unavailable", "severity": "action", "label": issue_label}
if not restricted:
issue["source"] = src_name
all_issues.append(issue)
source_reports.append({
"source_label": (src.description or src.name) if src else ("关联数据源" if restricted else src_name),
**({} if restricted else {"source": src_name}),
"verdict": status,
"issues": [issue],
})
continue
masked = {c.lower() for c in masked_columns(src.options)}
schema_tables = (src.schema_cache or {}).get("tables") if isinstance(src.schema_cache, Mapping) else {}
if not isinstance(schema_tables, Mapping):
schema_tables = {}
wanted_keys = {name_key(t) for t in target_tables} if target_tables else None
change_rows = list(
(
await session.execute(
select(SourceChange)
.where(SourceChange.source_id == src.id, SourceChange.status.in_(sc.PENDING))
.order_by(SourceChange.seq.desc())
)
).scalars()
)
src_issues: list[dict[str, Any]] = []
src_rank = 0
for ch in change_rows:
if wanted_keys is not None and ch.table_name and name_key(ch.table_name) not in wanted_keys:
continue
# 遮罩列上的变化不暴露列名与取值细节(铁律 3)
if ch.column_name and ch.column_name.lower() in masked:
continue
label = sc.KIND_LABEL.get(ch.kind, ch.kind)
is_severe = ch.kind in _SEVERE_CHANGE_KINDS or ch.severity == "action"
src_rank = max(src_rank, 3 if is_severe else 2)
entry: dict[str, Any] = {
"kind": ch.kind,
"severity": ch.severity,
"status": ch.status,
"label": label,
}
if not restricted:
entry["source"] = src.name
if ch.table_name:
entry["table"] = ch.table_name
if ch.column_name:
entry["column"] = ch.column_name
entry["summary"] = sc.summary(ch)
src_issues.append(entry)
probe_unverified = False
probed_tables: list[dict[str, Any]] = []
if live_probe:
candidate_names = target_tables or list(schema_tables.keys())[:MAX_TABLES_PROBED]
for tbl_name in candidate_names[:MAX_TABLES_PROBED]:
# 找到 schema_cache 里精确的表名键
matched_name = next((k for k in schema_tables if name_key(k) == name_key(tbl_name)), tbl_name)
meta = schema_tables.get(matched_name) if isinstance(schema_tables.get(matched_name), Mapping) else {}
probe = await _probe_table_coverage(
src,
matched_name,
meta,
masked=masked,
start_day=start_day,
end_day=end_day,
)
if probe["status"] == UNVERIFIED:
probe_unverified = True
if src_rank == 0:
src_rank = 1
elif probe["status"] == ANOMALOUS:
src_rank = max(src_rank, 3)
if probe.get("issue") == "empty_table":
item: dict[str, Any] = {
"kind": sc.ROWS_DROP,
"severity": "action",
"label": "目标表当前无数据(0 行)",
}
if not restricted:
item["source"] = src.name
item["table"] = matched_name
src_issues.append(item)
for wi in probe.get("window_issues") or []:
item = {
"kind": wi["kind"],
"severity": wi["severity"],
"label": sc.KIND_LABEL.get(wi["kind"], wi["kind"]),
}
if not restricted:
item["source"] = src.name
item["table"] = matched_name
item["column"] = wi["column"]
if "max_date" in wi:
item["max_date"] = wi["max_date"]
if "min_date" in wi:
item["min_date"] = wi["min_date"]
src_issues.append(item)
if not restricted:
probed_tables.append(probe)
src_verdict = rank_to_verdict[src_rank]
overall_rank = max(overall_rank, src_rank)
all_issues.extend(src_issues)
display_title = (src.description or "关联业务源") if restricted else src.name
rep: dict[str, Any] = {
"source_label": display_title,
"verdict": src_verdict,
"issue_count": len(src_issues),
"issues": src_issues,
"phrase": _render_phrase(display_title, src_verdict, src_issues, probe_unverified=probe_unverified),
}
if not restricted:
rep["source"] = src.name
rep["tables_probed"] = probed_tables
source_reports.append(rep)
overall_verdict = rank_to_verdict[overall_rank]
lead_phrase = source_reports[0]["phrase"] if len(source_reports) == 1 else (
";".join(r["phrase"] for r in source_reports)
)
out: dict[str, Any] = {
"kind": ARTIFACT_KIND,
"verdict": overall_verdict,
"sources": source_reports,
"issues": all_issues,
"phrase": lead_phrase,
}
if workflow_id:
out["caliber_ref"] = {
"workflow_id": workflow_id,
**({"version": version} if version is not None else {}),
**({"card_id": card_id} if card_id else {}),
**({"metric_id": metric_id} if metric_id else {}),
}
return out
def is_health_result(content: Any) -> bool:
return (
isinstance(content, Mapping)
and content.get("kind") == ARTIFACT_KIND
and content.get("verdict") in VERDICTS
and isinstance(content.get("sources"), list)
)
async def store_source_health(result: Mapping[str, Any], *, run_id: str | None, node_id: str | None = None) -> str:
return await artifact_store.put_json(dict(result), kind=ARTIFACT_KIND, run_id=run_id, meta={
"verdict": result.get("verdict"),
"node_id": node_id or "",
})
def ledger_entry(
result: Mapping[str, Any],
*,
node_id: str,
exec_no: int,
artifact: str,
call_id: str | None = None,
) -> dict[str, Any]:
sources = [
str(s.get("source") or s.get("source_label") or "")
for s in (result.get("sources") or [])
if isinstance(s, Mapping)
]
entry: dict[str, Any] = {
"kind": LEDGER_KIND,
"node_id": node_id,
"exec": exec_no,
"artifact": artifact,
"verdict": result.get("verdict"),
"sources": [s for s in sources if s],
"issue_count": len(result.get("issues") or []),
}
if call_id is not None:
entry["call_id"] = call_id
return entry