mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-20 12:31:57 +00:00
业务本体与标准实体:跨源键值对齐、对账共变算子、健康证据链与实体助手
- 新增标准业务实体(CanonicalEntity)模型、确定性跨源编码翻译、后台候选实体扫描与文档驱动实体智能助手 - 新增指标本体图谱(factor / reconciles_with / upstream_driver)、DAG 防环校验以及 ontology_neighbors / source_health 内置工具 - 新增跨源对账(analysis_reconcile)与滞后共变(analysis_cochange)分析算子,打通 [[h:H1]] 数据源健康证据链、报告导出与本体消融评测 - 数据目录新增「标准业务实体」视图与候选审阅工作台,口径页集成指标本体拓扑面板与端到端浏览器检查
This commit is contained in:
@@ -29,7 +29,7 @@ from typing import Any
|
||||
#: time_role / refund_rule 是 K04 的业务定义字段(统计时间口径、退款处理):改了定义身份就变,绑定同步状态跟着重判。
|
||||
SEMANTIC_FIELDS = ("id", "name", "expression", "unit", "decimals", "format", "kind", "definition", "params", "external",
|
||||
"rule", "assessment", "grain", "time_field", "numerator", "denominator", "dimensions", "filters",
|
||||
"contract", "time_role", "refund_rule")
|
||||
"contract", "time_role", "refund_rule", "semantic", "links")
|
||||
|
||||
|
||||
def _blank(value: Any) -> bool:
|
||||
|
||||
@@ -0,0 +1,566 @@
|
||||
"""标准业务实体(CanonicalEntity)的校验、持久化与跨源键值对齐。
|
||||
|
||||
核心铁律:
|
||||
1. 物理表列绑定(bindings)与口径参数映射(caliber_params)分设 proposed / verified / confirmed / rejected 四态;
|
||||
运行时自动穿透、跨源实体值翻译、参数传递**只认 verified / confirmed**,proposed 候选绝不自动参与穿透。
|
||||
2. 跨源实体值翻译(translate_entity_value)坚持确定性映射:有标准成员表(members)时只按显式 source_codes /
|
||||
标准值 / 别名精确匹配,映射不上返回 None,严禁靠子串猜测。
|
||||
3. 受限视角(restricted=True,如普通 user 角色)隐去物理表名、物理列名与跨源底层编码,只暴露业务实体名与标准值。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Iterable, Mapping
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.base import new_id, utcnow
|
||||
from app.db.models_ontology import CanonicalEntity
|
||||
|
||||
ENTITY_STATUSES = ("proposed", "verified", "confirmed", "rejected")
|
||||
ACTIVE_ENTITY_STATUSES = ("verified", "confirmed")
|
||||
ENTITY_SOURCES = ("human", "rule", "ai", "profile")
|
||||
|
||||
_KEY_RE = re.compile(r"^[a-z][a-z0-9_]{0,63}$")
|
||||
|
||||
|
||||
class EntityInvalid(ValueError):
|
||||
"""标准实体请求格式或取值不合法。"""
|
||||
|
||||
def __init__(self, message: str, *, field: str = "") -> None:
|
||||
super().__init__(message)
|
||||
self.message = message
|
||||
self.field = field
|
||||
|
||||
|
||||
class EntityConflict(ValueError):
|
||||
"""标准实体版本冲突或唯一键冲突。"""
|
||||
|
||||
def __init__(self, message: str, *, field: str = "") -> None:
|
||||
super().__init__(message)
|
||||
self.message = message
|
||||
self.field = field
|
||||
|
||||
|
||||
class EntityNotFound(LookupError):
|
||||
"""标准实体或子项不存在。"""
|
||||
|
||||
|
||||
def _norm_token(value: Any) -> str:
|
||||
return str(value or "").strip().lower()
|
||||
|
||||
|
||||
def normalize_entity_key(raw: Any, *, field: str = "key") -> str:
|
||||
if not isinstance(raw, str) or not _KEY_RE.match(raw.strip()):
|
||||
raise EntityInvalid("实体标识(key)须以小写字母开头,仅含小写字母、数字和下划线(最长 64 字符)", field=field)
|
||||
return raw.strip()
|
||||
|
||||
|
||||
def _clean_str_list(raw: Any, *, field: str) -> list[str]:
|
||||
if raw in (None, ""):
|
||||
return []
|
||||
if not isinstance(raw, list):
|
||||
raise EntityInvalid(f"字段「{field}」需要是字符串列表", field=field)
|
||||
seen: set[str] = set()
|
||||
out: list[str] = []
|
||||
for idx, item in enumerate(raw):
|
||||
if not isinstance(item, str):
|
||||
raise EntityInvalid(f"字段「{field}[{idx}]」需要是字符串", field=f"{field}[{idx}]")
|
||||
val = item.strip()
|
||||
if val and val.lower() not in seen:
|
||||
seen.add(val.lower())
|
||||
out.append(val)
|
||||
return out
|
||||
|
||||
|
||||
def normalize_member(raw: Any, *, index: int = 0) -> dict[str, Any]:
|
||||
at = f"members[{index}]"
|
||||
if not isinstance(raw, Mapping):
|
||||
raise EntityInvalid("实体成员需要是对象", field=at)
|
||||
value = str(raw.get("value") or "").strip()
|
||||
if not value:
|
||||
raise EntityInvalid("实体成员缺少标准值(value)", field=f"{at}.value")
|
||||
label = str(raw.get("label") or value).strip() or value
|
||||
aliases = _clean_str_list(raw.get("aliases"), field=f"{at}.aliases")
|
||||
raw_codes = raw.get("source_codes")
|
||||
source_codes: dict[str, str] = {}
|
||||
if raw_codes not in (None, {}):
|
||||
if not isinstance(raw_codes, Mapping):
|
||||
raise EntityInvalid("跨源编码(source_codes)需要是「数据源名 → 编码」对象", field=f"{at}.source_codes")
|
||||
for src, code in raw_codes.items():
|
||||
src_key = str(src or "").strip()
|
||||
code_val = str(code or "").strip()
|
||||
if src_key and code_val:
|
||||
source_codes[src_key] = code_val
|
||||
return {"value": value, "label": label, "aliases": aliases, "source_codes": source_codes}
|
||||
|
||||
|
||||
def normalize_binding(raw: Any, *, index: int = 0, actor: str | None = None,
|
||||
force_proposed: bool = False) -> dict[str, Any]:
|
||||
at = f"bindings[{index}]"
|
||||
if not isinstance(raw, Mapping):
|
||||
raise EntityInvalid("物理表列绑定需要是对象", field=at)
|
||||
source_name = str(raw.get("source_name") or "").strip()
|
||||
source_id = str(raw.get("source_id") or "").strip() or None
|
||||
table_name = str(raw.get("table_name") or "").strip()
|
||||
column_name = str(raw.get("column_name") or "").strip()
|
||||
if not source_name:
|
||||
raise EntityInvalid("物理绑定缺少数据源名称(source_name)", field=f"{at}.source_name")
|
||||
if not table_name:
|
||||
raise EntityInvalid("物理绑定缺少表名(table_name)", field=f"{at}.table_name")
|
||||
if not column_name:
|
||||
raise EntityInvalid("物理绑定缺少列名(column_name)", field=f"{at}.column_name")
|
||||
source = str(raw.get("source") or "human").strip()
|
||||
if source not in ENTITY_SOURCES:
|
||||
raise EntityInvalid(f"绑定来源「{source}」无效,可选:{', '.join(ENTITY_SOURCES)}", field=f"{at}.source")
|
||||
status = "proposed" if (force_proposed or source in ("ai", "profile") and "status" not in raw) else str(
|
||||
raw.get("status") or "confirmed"
|
||||
).strip()
|
||||
if force_proposed:
|
||||
status = "proposed"
|
||||
if status not in ENTITY_STATUSES:
|
||||
raise EntityInvalid(f"绑定状态「{status}」无效,可选:{', '.join(ENTITY_STATUSES)}", field=f"{at}.status")
|
||||
return {
|
||||
"id": str(raw.get("id") or "").strip() or new_id()[:12],
|
||||
"source_id": source_id,
|
||||
"source_name": source_name,
|
||||
"table_name": table_name,
|
||||
"column_name": column_name,
|
||||
"status": status,
|
||||
"source": source,
|
||||
"note": str(raw.get("note") or "").strip(),
|
||||
"reason": str(raw.get("reason") or "").strip(),
|
||||
"updated_at": str(raw.get("updated_at") or utcnow().isoformat()),
|
||||
"updated_by": str(raw.get("updated_by") or actor or "").strip() or None,
|
||||
}
|
||||
|
||||
|
||||
def normalize_caliber_param(raw: Any, *, index: int = 0, force_proposed: bool = False) -> dict[str, Any]:
|
||||
at = f"caliber_params[{index}]"
|
||||
if not isinstance(raw, Mapping):
|
||||
raise EntityInvalid("口径参数映射需要是对象", field=at)
|
||||
workflow_id = str(raw.get("workflow_id") or "").strip()
|
||||
card_id = str(raw.get("card_id") or "").strip()
|
||||
param_name = str(raw.get("param_name") or "").strip()
|
||||
if not workflow_id or not param_name:
|
||||
raise EntityInvalid("口径参数映射须写明工作流 ID(workflow_id)与参数名(param_name)", field=at)
|
||||
source = str(raw.get("source") or "human").strip()
|
||||
if source not in ENTITY_SOURCES:
|
||||
raise EntityInvalid(f"口径参数映射来源「{source}」无效", field=f"{at}.source")
|
||||
status = "proposed" if force_proposed else str(raw.get("status") or "confirmed").strip()
|
||||
if status not in ENTITY_STATUSES:
|
||||
raise EntityInvalid(f"口径参数映射状态「{status}」无效", field=f"{at}.status")
|
||||
return {
|
||||
"id": str(raw.get("id") or "").strip() or new_id()[:12],
|
||||
"workflow_id": workflow_id,
|
||||
"card_id": card_id,
|
||||
"param_name": param_name,
|
||||
"status": status,
|
||||
"source": source,
|
||||
"note": str(raw.get("note") or "").strip(),
|
||||
}
|
||||
|
||||
|
||||
def normalize_entity_payload(payload: Mapping[str, Any], *, partial: bool = False,
|
||||
actor: str | None = None) -> dict[str, Any]:
|
||||
if not isinstance(payload, Mapping):
|
||||
raise EntityInvalid("请求内容需要是对象")
|
||||
out: dict[str, Any] = {}
|
||||
if not partial or "key" in payload:
|
||||
out["key"] = normalize_entity_key(payload.get("key"))
|
||||
if not partial or "label" in payload:
|
||||
label = str(payload.get("label") or "").strip()
|
||||
if not label:
|
||||
raise EntityInvalid("实体名称(label)不能为空", field="label")
|
||||
if len(label) > 100:
|
||||
raise EntityInvalid("实体名称不能超过 100 个字符", field="label")
|
||||
out["label"] = label
|
||||
if not partial or "description" in payload:
|
||||
out["description"] = str(payload.get("description") or "").strip()
|
||||
if not partial or "aliases" in payload:
|
||||
out["aliases"] = _clean_str_list(payload.get("aliases"), field="aliases")
|
||||
if not partial or "members" in payload:
|
||||
raw_members = payload.get("members") or []
|
||||
if not isinstance(raw_members, list):
|
||||
raise EntityInvalid("成员列表(members)需要是数组", field="members")
|
||||
seen_vals: set[str] = set()
|
||||
members: list[dict[str, Any]] = []
|
||||
for idx, m in enumerate(raw_members):
|
||||
norm = normalize_member(m, index=idx)
|
||||
vkey = norm["value"].lower()
|
||||
if vkey in seen_vals:
|
||||
raise EntityInvalid(f"实体成员值「{norm['value']}」重复", field=f"members[{idx}].value")
|
||||
seen_vals.add(vkey)
|
||||
members.append(norm)
|
||||
out["members"] = members
|
||||
if not partial or "bindings" in payload:
|
||||
raw_bindings = payload.get("bindings") or []
|
||||
if not isinstance(raw_bindings, list):
|
||||
raise EntityInvalid("物理绑定列表(bindings)需要是数组", field="bindings")
|
||||
seen_cols: set[tuple[str, str, str]] = set()
|
||||
bindings: list[dict[str, Any]] = []
|
||||
for idx, b in enumerate(raw_bindings):
|
||||
norm = normalize_binding(b, index=idx, actor=actor)
|
||||
col_key = (norm["source_name"].lower(), norm["table_name"].lower(), norm["column_name"].lower())
|
||||
if col_key in seen_cols:
|
||||
raise EntityInvalid(
|
||||
f"数据源「{norm['source_name']}」表「{norm['table_name']}.{norm['column_name']}」重复绑定",
|
||||
field=f"bindings[{idx}]",
|
||||
)
|
||||
seen_cols.add(col_key)
|
||||
bindings.append(norm)
|
||||
out["bindings"] = bindings
|
||||
if not partial or "caliber_params" in payload:
|
||||
raw_params = payload.get("caliber_params") or []
|
||||
if not isinstance(raw_params, list):
|
||||
raise EntityInvalid("口径参数映射(caliber_params)需要是数组", field="caliber_params")
|
||||
out["caliber_params"] = [normalize_caliber_param(p, index=idx) for idx, p in enumerate(raw_params)]
|
||||
return out
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 运行时对齐与状态过滤(只认 verified / confirmed)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _as_dict(entity: CanonicalEntity | Mapping[str, Any]) -> dict[str, Any]:
|
||||
if isinstance(entity, Mapping):
|
||||
return dict(entity)
|
||||
return {
|
||||
"id": entity.id,
|
||||
"key": entity.key,
|
||||
"label": entity.label,
|
||||
"description": entity.description or "",
|
||||
"aliases": list(entity.aliases or []),
|
||||
"members": list(entity.members or []),
|
||||
"bindings": list(entity.bindings or []),
|
||||
"caliber_params": list(entity.caliber_params or []),
|
||||
"version": int(entity.version or 1),
|
||||
"created_by": entity.created_by,
|
||||
"updated_by": entity.updated_by,
|
||||
"created_at": entity.created_at.isoformat() if getattr(entity, "created_at", None) else None,
|
||||
"updated_at": entity.updated_at.isoformat() if getattr(entity, "updated_at", None) else None,
|
||||
}
|
||||
|
||||
|
||||
def entity_to_dict(entity: CanonicalEntity | Mapping[str, Any], *, restricted: bool = False) -> dict[str, Any]:
|
||||
"""序列化标准实体;受限视角(user 角色)隐去物理表名、列名、数据源内部 ID 与底层编码。"""
|
||||
data = _as_dict(entity)
|
||||
if not restricted:
|
||||
return data
|
||||
return {
|
||||
"id": data["id"],
|
||||
"key": data["key"],
|
||||
"label": data["label"],
|
||||
"description": data["description"],
|
||||
"aliases": list(data.get("aliases") or []),
|
||||
"members": [
|
||||
{"value": m.get("value"), "label": m.get("label"), "aliases": list(m.get("aliases") or [])}
|
||||
for m in (data.get("members") or []) if isinstance(m, Mapping)
|
||||
],
|
||||
"version": data["version"],
|
||||
}
|
||||
|
||||
|
||||
def confirmed_bindings(
|
||||
entity: CanonicalEntity | Mapping[str, Any],
|
||||
*,
|
||||
source_id: str | None = None,
|
||||
source_name: str | None = None,
|
||||
table_name: str | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""返回已生效(verified / confirmed)的物理列绑定;proposed 与 rejected 一律排除。"""
|
||||
data = _as_dict(entity)
|
||||
out: list[dict[str, Any]] = []
|
||||
for item in data.get("bindings") or []:
|
||||
if not isinstance(item, Mapping) or item.get("status") not in ACTIVE_ENTITY_STATUSES:
|
||||
continue
|
||||
if source_id and str(item.get("source_id") or "") != source_id:
|
||||
continue
|
||||
if source_name and _norm_token(item.get("source_name")) != _norm_token(source_name):
|
||||
continue
|
||||
if table_name and _norm_token(item.get("table_name")) != _norm_token(table_name):
|
||||
continue
|
||||
out.append(dict(item))
|
||||
return out
|
||||
|
||||
|
||||
def confirmed_caliber_params(
|
||||
entity: CanonicalEntity | Mapping[str, Any],
|
||||
*,
|
||||
workflow_id: str | None = None,
|
||||
card_id: str | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""返回已生效(verified / confirmed)的口径输入参数映射。"""
|
||||
data = _as_dict(entity)
|
||||
out: list[dict[str, Any]] = []
|
||||
for item in data.get("caliber_params") or []:
|
||||
if not isinstance(item, Mapping) or item.get("status") not in ACTIVE_ENTITY_STATUSES:
|
||||
continue
|
||||
if workflow_id and str(item.get("workflow_id") or "") != workflow_id:
|
||||
continue
|
||||
if card_id and item.get("card_id") and str(item.get("card_id") or "") != card_id:
|
||||
continue
|
||||
out.append(dict(item))
|
||||
return out
|
||||
|
||||
|
||||
def resolve_caliber_param(
|
||||
entity: CanonicalEntity | Mapping[str, Any],
|
||||
*,
|
||||
workflow_id: str,
|
||||
card_id: str | None = None,
|
||||
) -> str | None:
|
||||
"""把标准业务实体解析为目标口径的入口参数名(优先匹配指定 card_id,其次匹配同工作流通用参数)。"""
|
||||
matches = confirmed_caliber_params(entity, workflow_id=workflow_id)
|
||||
if card_id:
|
||||
for item in matches:
|
||||
if str(item.get("card_id") or "") == card_id:
|
||||
return str(item["param_name"])
|
||||
for item in matches:
|
||||
if not item.get("card_id") or card_id is None:
|
||||
return str(item["param_name"])
|
||||
return str(matches[0]["param_name"]) if matches else None
|
||||
|
||||
|
||||
def translate_entity_value(
|
||||
entity: CanonicalEntity | Mapping[str, Any],
|
||||
raw_value: Any,
|
||||
*,
|
||||
from_source: str | None = None,
|
||||
to_source: str | None = None,
|
||||
) -> dict[str, str] | None:
|
||||
"""把源侧实体值确定性地翻译成标准值与目标源编码。
|
||||
|
||||
- 有标准成员表(members 非空)时:严格按 from_source 的 source_codes、标准 value、label、aliases 精确匹配,
|
||||
再用 to_source 的 source_codes(缺省回退标准 value)给出 target_value;找不到返回 None,绝不模糊猜测。
|
||||
- 成员表为空(开放式编码域,如千万级订单号/批次号)时:仅当两侧数据源在该实体上均有已确认绑定时原样透传。
|
||||
"""
|
||||
probe = str(raw_value or "").strip()
|
||||
if not probe:
|
||||
return None
|
||||
data = _as_dict(entity)
|
||||
members = [m for m in (data.get("members") or []) if isinstance(m, Mapping)]
|
||||
probe_norm = probe.lower()
|
||||
|
||||
if members:
|
||||
matched: Mapping[str, Any] | None = None
|
||||
if from_source:
|
||||
src_norm = _norm_token(from_source)
|
||||
for member in members:
|
||||
codes = member.get("source_codes")
|
||||
if isinstance(codes, Mapping):
|
||||
for k, v in codes.items():
|
||||
if _norm_token(k) == src_norm and _norm_token(v) == probe_norm:
|
||||
matched = member
|
||||
break
|
||||
if matched is not None:
|
||||
break
|
||||
if matched is None:
|
||||
for member in members:
|
||||
candidates = [member.get("value"), member.get("label"), *(member.get("aliases") or [])]
|
||||
codes = member.get("source_codes")
|
||||
if not from_source and isinstance(codes, Mapping):
|
||||
candidates.extend(codes.values())
|
||||
if any(_norm_token(c) == probe_norm for c in candidates if c is not None):
|
||||
matched = member
|
||||
break
|
||||
if matched is None:
|
||||
return None
|
||||
canonical = str(matched.get("value") or "").strip()
|
||||
label = str(matched.get("label") or canonical).strip()
|
||||
target_value = canonical
|
||||
if to_source and isinstance(matched.get("source_codes"), Mapping):
|
||||
dst_norm = _norm_token(to_source)
|
||||
for k, v in matched["source_codes"].items():
|
||||
if _norm_token(k) == dst_norm and str(v or "").strip():
|
||||
target_value = str(v).strip()
|
||||
break
|
||||
return {"canonical": canonical, "label": label, "target_value": target_value}
|
||||
|
||||
if from_source and not confirmed_bindings(data, source_name=from_source):
|
||||
return None
|
||||
if to_source and not confirmed_bindings(data, source_name=to_source):
|
||||
return None
|
||||
return {"canonical": probe, "label": probe, "target_value": probe}
|
||||
|
||||
|
||||
def entities_bridging_sources(
|
||||
entities: Iterable[CanonicalEntity | Mapping[str, Any]],
|
||||
*,
|
||||
source_name: str,
|
||||
tables: Iterable[str] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""找出在当前数据源(及可选表集合)上有已确认绑定、并且同时桥接到其他数据源的标准实体。"""
|
||||
table_keys = {_norm_token(t) for t in tables} if tables is not None else None
|
||||
out: list[dict[str, Any]] = []
|
||||
for entity in entities:
|
||||
data = _as_dict(entity)
|
||||
active = confirmed_bindings(data)
|
||||
here = [
|
||||
b for b in active
|
||||
if _norm_token(b.get("source_name")) == _norm_token(source_name)
|
||||
and (table_keys is None or _norm_token(b.get("table_name")) in table_keys)
|
||||
]
|
||||
if not here:
|
||||
continue
|
||||
peers = [b for b in active if _norm_token(b.get("source_name")) != _norm_token(source_name)]
|
||||
out.append({
|
||||
"entity_id": data["id"],
|
||||
"key": data["key"],
|
||||
"label": data["label"],
|
||||
"local_bindings": here,
|
||||
"peer_bindings": peers,
|
||||
"peer_sources": sorted({str(b["source_name"]) for b in peers}),
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 数据库读写与乐观锁审阅
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def list_entities(
|
||||
session: AsyncSession,
|
||||
*,
|
||||
source_name: str | None = None,
|
||||
only_active_bindings: bool = False,
|
||||
) -> list[CanonicalEntity]:
|
||||
rows = list((await session.execute(select(CanonicalEntity).order_by(CanonicalEntity.key.asc()))).scalars())
|
||||
if not source_name:
|
||||
return rows
|
||||
src_norm = _norm_token(source_name)
|
||||
matched: list[CanonicalEntity] = []
|
||||
for row in rows:
|
||||
bindings = confirmed_bindings(row) if only_active_bindings else (row.bindings or [])
|
||||
if any(isinstance(b, Mapping) and _norm_token(b.get("source_name")) == src_norm for b in bindings):
|
||||
matched.append(row)
|
||||
return matched
|
||||
|
||||
|
||||
async def get_entity(session: AsyncSession, entity_id: str) -> CanonicalEntity | None:
|
||||
return await session.get(CanonicalEntity, str(entity_id or "").strip())
|
||||
|
||||
|
||||
async def get_entity_by_key(session: AsyncSession, key: str) -> CanonicalEntity | None:
|
||||
clean = str(key or "").strip().lower()
|
||||
if not clean:
|
||||
return None
|
||||
return (
|
||||
await session.execute(select(CanonicalEntity).where(CanonicalEntity.key == clean).limit(1))
|
||||
).scalar_one_or_none()
|
||||
|
||||
|
||||
async def create_entity(
|
||||
session: AsyncSession,
|
||||
payload: Mapping[str, Any],
|
||||
*,
|
||||
actor: str | None = None,
|
||||
) -> CanonicalEntity:
|
||||
fields = normalize_entity_payload(payload, partial=False, actor=actor)
|
||||
existing = await get_entity_by_key(session, fields["key"])
|
||||
if existing is not None:
|
||||
raise EntityConflict(f"标准实体标识「{fields['key']}」已存在", field="key")
|
||||
row = CanonicalEntity(
|
||||
key=fields["key"],
|
||||
label=fields["label"],
|
||||
description=fields["description"],
|
||||
aliases=fields["aliases"],
|
||||
members=fields["members"],
|
||||
bindings=fields["bindings"],
|
||||
caliber_params=fields["caliber_params"],
|
||||
version=1,
|
||||
created_by=actor,
|
||||
updated_by=actor,
|
||||
)
|
||||
session.add(row)
|
||||
await session.flush()
|
||||
return row
|
||||
|
||||
|
||||
async def update_entity(
|
||||
session: AsyncSession,
|
||||
entity_id: str,
|
||||
payload: Mapping[str, Any],
|
||||
*,
|
||||
actor: str | None = None,
|
||||
) -> CanonicalEntity:
|
||||
row = await get_entity(session, entity_id)
|
||||
if row is None:
|
||||
raise EntityNotFound(f"标准实体「{entity_id}」不存在")
|
||||
expected_version = payload.get("version")
|
||||
if expected_version is not None:
|
||||
if isinstance(expected_version, bool) or not isinstance(expected_version, int) or expected_version != row.version:
|
||||
raise EntityConflict(
|
||||
f"标准实体「{row.label}」已被其他人修改(当前版本 v{row.version}),请刷新后再保存",
|
||||
field="version",
|
||||
)
|
||||
fields = normalize_entity_payload(payload, partial=True, actor=actor)
|
||||
if "key" in fields and fields["key"] != row.key:
|
||||
clash = await get_entity_by_key(session, fields["key"])
|
||||
if clash is not None and clash.id != row.id:
|
||||
raise EntityConflict(f"标准实体标识「{fields['key']}」已存在", field="key")
|
||||
row.key = fields["key"]
|
||||
for attr in ("label", "description", "aliases", "members", "bindings", "caliber_params"):
|
||||
if attr in fields:
|
||||
setattr(row, attr, fields[attr])
|
||||
row.version = int(row.version or 1) + 1
|
||||
row.updated_by = actor
|
||||
await session.flush()
|
||||
return row
|
||||
|
||||
|
||||
async def review_entity_item(
|
||||
session: AsyncSession,
|
||||
entity_id: str,
|
||||
*,
|
||||
target_kind: str,
|
||||
item_id: str,
|
||||
status: str,
|
||||
reason: str = "",
|
||||
actor: str | None = None,
|
||||
expected_version: int | None = None,
|
||||
) -> CanonicalEntity:
|
||||
"""审阅单条物理表列绑定或口径参数映射(确认 confirmed / 驳回 rejected / 验证通过 verified)。"""
|
||||
if target_kind not in ("binding", "caliber_param"):
|
||||
raise EntityInvalid("审阅对象类型须为 binding 或 caliber_param", field="target_kind")
|
||||
if status not in ENTITY_STATUSES:
|
||||
raise EntityInvalid(f"状态「{status}」无效,可选:{', '.join(ENTITY_STATUSES)}", field="status")
|
||||
row = await get_entity(session, entity_id)
|
||||
if row is None:
|
||||
raise EntityNotFound(f"标准实体「{entity_id}」不存在")
|
||||
if expected_version is not None and int(expected_version) != int(row.version or 1):
|
||||
raise EntityConflict(f"标准实体「{row.label}」版本已变更(当前 v{row.version}),请刷新后重试", field="version")
|
||||
|
||||
attr = "bindings" if target_kind == "binding" else "caliber_params"
|
||||
items = [dict(x) for x in (getattr(row, attr) or []) if isinstance(x, Mapping)]
|
||||
found = False
|
||||
now = utcnow().isoformat()
|
||||
for item in items:
|
||||
if str(item.get("id") or "") == str(item_id or "").strip():
|
||||
item["status"] = status
|
||||
if reason.strip():
|
||||
item["reason"] = reason.strip()
|
||||
item["updated_at"] = now
|
||||
item["updated_by"] = actor
|
||||
found = True
|
||||
break
|
||||
if not found:
|
||||
raise EntityNotFound(f"标准实体「{row.label}」中不存在子项「{item_id}」")
|
||||
setattr(row, attr, items)
|
||||
row.version = int(row.version or 1) + 1
|
||||
row.updated_by = actor
|
||||
await session.flush()
|
||||
return row
|
||||
|
||||
|
||||
async def delete_entity(session: AsyncSession, entity_id: str) -> bool:
|
||||
row = await get_entity(session, entity_id)
|
||||
if row is None:
|
||||
return False
|
||||
await session.delete(row)
|
||||
await session.flush()
|
||||
return True
|
||||
@@ -0,0 +1,942 @@
|
||||
"""实体助手(E2):在标准业务实体页与使用者流式对话,跨数据源与业务文档对齐维护 CanonicalEntity。
|
||||
|
||||
核心规矩:
|
||||
1. 只提结构化提案(propose_entity_changes),绝不静默直接写库;使用者在界面看过后点击「保存修改」才经
|
||||
apply_entity_patch(mode="confirmed") 落库。
|
||||
2. 跨源只读探查(run_query)仅对已开启数据剖析的数据源放行,严格经 guard.check(readonly=True) 与遮罩列脱敏
|
||||
(mask_query_result);遮罩列只给列名,不给注释或码值。
|
||||
3. 支持直接读取本次对话挂载的上传附件或知识库文档片段(read_document),并要求在提案 reason 中区分「使用者说:……」、
|
||||
「文档《…》写着:……」与「数据探查显示:……」,同时主动向使用者汇报文档码表与实表取值的差异。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import AsyncIterator, Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.data import catalog, catalog_assistant, catalog_profile, entities, entity_docs, entity_patch, introspect, table_versions
|
||||
from app.data.draft_model import DRAFT_MODEL_PLACE
|
||||
from app.data.engine import SnapshotTampered, masked_columns
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import DataSource, Workflow, WorkflowVersion
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
MAX_MESSAGES = 40
|
||||
MESSAGE_MAX = 4000
|
||||
MAX_ROUNDS = 8
|
||||
MAX_QUERIES = 6
|
||||
QUERY_ROWS = 50
|
||||
SHOWN_ROWS = 10
|
||||
CELL_MAX = 300
|
||||
TOOL_CHARS = 12_000
|
||||
RETRY_DELAYS_S = (3.0, 10.0)
|
||||
HEARTBEAT_S = 10.0
|
||||
|
||||
INSPECT_TOOL = "inspect_sources"
|
||||
READ_DOC_TOOL = "read_document"
|
||||
QUERY_TOOL = "run_query"
|
||||
PROPOSE_TOOL = "propose_entity_changes"
|
||||
|
||||
CODE_MODEL = "assistant_model_error"
|
||||
CODE_TRUNCATED = "assistant_truncated"
|
||||
CODE_TIMEOUT = "assistant_timeout"
|
||||
CODE_ROUNDS = "assistant_rounds"
|
||||
CODE_FAILED = "assistant_failed"
|
||||
|
||||
# ---- 面向界面展示的错误文案
|
||||
ERROR_MODEL = (
|
||||
"调用模型「{model}」失败:{reason}。请稍后重试;如持续失败,请在{place}中换一个模型,"
|
||||
"或到「设置 → 模型接入」测试该模型"
|
||||
)
|
||||
ERROR_TRUNCATED = "{cut},回答没有写完。可以把问题拆小一点再问,或在{place}中换一个模型"
|
||||
ERROR_TIMEOUT = "模型 {seconds} 秒内没有回答完,已停止这一轮。可以把问题拆小一点再问,或在{place}中换一个模型"
|
||||
ERROR_ROUNDS = "这一轮已经调用了 {n} 次模型仍没有答完,已停止。可以换个问法,或者分几次问"
|
||||
ERROR_FAILED = "实体助手出错:{line}"
|
||||
LAST_NOT_USER = "对话的最后一条应当是使用者的话,请重新发送"
|
||||
QUERY_TAMPERED = "数据源「{name}」的{error}。请联系管理员检查数据目录"
|
||||
|
||||
# ---- 面向模型的提示词(命名含 _SYSTEM / _PROMPT)
|
||||
ENTITY_ASSISTANT_SYSTEM = """你是 AgentLab 的「实体助手」,协助搭建者围绕当前选定范围的数据源与业务文档建立、完善「标准业务实体」(CanonicalEntity):
|
||||
包括实体的标准标识(key)、中文名称(label)、常用叫法(aliases)、物理表列绑定(bindings)、标准成员编码映射(members.source_codes)与口径入口参数映射(caliber_params)。
|
||||
|
||||
核心规矩:
|
||||
1. 你不能修改业务数据库数据,也不直接写入标准业务实体:一切新增或修改都通过 propose_entity_changes 工具整理成结构化提案,由使用者在界面审阅后点击保存才生效。
|
||||
2. 严格限定在「当前分析范围的数据源」内工作:只探查、查询和绑定范围内的数据源,绝不提及、猜测或扩展到未纳入范围的其他数据源。若当前范围只有 1 个数据源,聚焦盘点该数据源内的核心业务实体、多表共享主外键、码值字典与口径入参;若选了多个数据源,则重点对齐这些选定数据源之间的同义实体与编码映射。
|
||||
3. 表名、列名、数据源名、工作流 ID、口径参数名必须先通过 inspect_sources 核实,严禁猜测不存在的表名或列名。
|
||||
4. 若挂载了业务文档或编码对照表,优先用 read_document 阅读原文定义或码值映射,并结合 inspect_sources / run_query 去真实数据源交叉核验:
|
||||
- 如果文档里提到的编码与数据库实表抽样不一致(例如实表多出文档未登记的新编码,或文档编码在实表未出现),必须在回答正文中明确告知使用者,询问是否一并纳入。
|
||||
- 如果拿不准两套编码如何对应,先简短具体地向使用者提问,不要硬猜。
|
||||
5. 提案中每条操作的 reason 字段必须如实写明依据来源:
|
||||
- 来自本次对话使用者亲口说明的,写「使用者说:……」;
|
||||
- 来自挂载文档或知识库文档的,写「文档《文档名》写着:……」;
|
||||
- 来自表目录或只读 SQL 核查的,写「数据探查显示:……」。
|
||||
6. 标着「已确认」或「已验证」的既有绑定,除非使用者明确要求调整,否则不要擅自改动或删除;标着「已驳回」的绑定使用者曾明确拒绝过,不要重复提议。
|
||||
7. 标着「遮罩列」的敏感字段禁止查询明细取值,也禁止为其生成成员编码映射(members.source_codes)。
|
||||
8. 回答用简短直接的中文。
|
||||
|
||||
propose_entity_changes 的 operations 数组支持以下 5 种增量操作:
|
||||
- upsert_entity: {"op": "upsert_entity", "entity_key": "小写英文下划线标识", "label": "中文名称", "description": "业务粒度说明", "aliases": ["常用叫法"], "reason": "依据"}
|
||||
- upsert_binding: {"op": "upsert_binding", "entity_key": "实体标识", "entity_label": "中文名(新建时必填)", "source_name": "数据源名", "table_name": "物理表名", "column_name": "物理列名", "note": "备注", "reason": "依据"}
|
||||
- remove_binding: {"op": "remove_binding", "entity_key": "实体标识", "source_name": "数据源名", "table_name": "物理表名", "column_name": "物理列名", "reason": "依据"}
|
||||
- upsert_member: {"op": "upsert_member", "entity_key": "实体标识", "entity_label": "中文名", "value": "标准值", "label": "标准展示名", "aliases": ["成员别名"], "source_codes": {"数据源名": "该源底层编码"}, "reason": "依据"}
|
||||
- upsert_caliber_param: {"op": "upsert_caliber_param", "entity_key": "实体标识", "entity_label": "中文名", "workflow_id": "工作流ID", "card_id": "可选口径卡节点ID", "param_name": "入口参数名", "note": "备注", "reason": "依据"}
|
||||
"""
|
||||
|
||||
ENTITY_QUERY_PROMPT = (
|
||||
"跨源只读查询:可对开启了数据剖析的数据源调用 run_query 执行只读 SQL(如 SELECT DISTINCT 核对编码取值与行数分布),"
|
||||
"单次最多返回 {rows} 行,本轮最多执行 {limit} 次。可查询的数据源:{sources}。"
|
||||
)
|
||||
ENTITY_NO_QUERY_PROMPT = "当前范围内没有开启数据剖析的数据源,不能执行 run_query 查实表数据,请根据表结构、数据目录与业务文档推断对齐。"
|
||||
ENTITY_OPENING_PROMPT = (
|
||||
"(使用者刚打开实体助手,尚未发言。)请结合当前分析范围的数据源、既有实体与挂载文档概况,用两三句话指出当前范围内最值得优先建立或补齐的业务实体,并提出一两个具体问题;先不要调用 propose_entity_changes。"
|
||||
)
|
||||
ENTITY_OPENED_PROMPT = "(使用者打开了实体助手。)"
|
||||
|
||||
ENTITY_TOOL_PROMPT = {
|
||||
"query_disabled_source": "数据源「{source}」未开启数据剖析或不存在,无法执行只读查询。可查询的数据源:{allowed}",
|
||||
"query_capped": "这一轮的只读查询次数已达上限({n} 次),请根据已有信息回答或提出提案。",
|
||||
"empty_sql": "请填写要执行的只读 SQL 语句。",
|
||||
"query_failed": "查询未成功:{error}",
|
||||
"no_ops": "operations 为空,请提供至少一条修改操作。",
|
||||
"proposed": "已通过核验并向界面提交 {n} 项实体变更提案:{items}",
|
||||
"nothing": "没有产生新的有效变更(可能与当前状态完全相同,或全部未通过核验)。",
|
||||
"dropped": "以下项未通过平台核验,请核对表列名或参数后修正:{reasons}",
|
||||
"no_docs": "当前对话未挂载任何业务文档或对照表附件。",
|
||||
"unknown_tool": "未知工具「{name}」,可用工具:{tools}",
|
||||
"last_round": "注意:下一轮是最后一次模型调用,请直接用中文向使用者总结结论,不要再调用工具。",
|
||||
}
|
||||
|
||||
INSPECT_TOOL_PROMPT: dict[str, Any] = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": INSPECT_TOOL,
|
||||
"description": "在已启用的数据源表结构、数据目录(列中文名、含义、码值)及已发布口径入口参数中检索与业务实体相关的表列和入参。",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"keyword": {
|
||||
"type": "string",
|
||||
"description": "检索关键词(如「产线」「line」「仓库」「region」);留空则返回各数据源表概览与口径入参清单。",
|
||||
},
|
||||
"source_name": {
|
||||
"type": "string",
|
||||
"description": "可选:限定只查看某一个数据源名称。",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
READ_DOC_TOOL_PROMPT: dict[str, Any] = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": READ_DOC_TOOL,
|
||||
"description": "读取本次对话挂载的业务文档、数据字典或编码对照表片段。",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {
|
||||
"type": "string",
|
||||
"description": "按关键词检索相关段落或表格行;留空则按顺序读取分段。",
|
||||
},
|
||||
"doc_ref": {
|
||||
"type": "string",
|
||||
"description": "可选:限定某一份文档的 ID 或标题关键词。",
|
||||
},
|
||||
"start_chunk": {
|
||||
"type": "integer",
|
||||
"description": "顺序读取时的起始段号(从 0 开始)。",
|
||||
},
|
||||
"limit": {
|
||||
"type": "integer",
|
||||
"description": "最多返回几段(默认 6,最多 15)。",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
QUERY_TOOL_PROMPT: dict[str, Any] = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": QUERY_TOOL,
|
||||
"description": "在指定数据源上执行一条只读 SQL(如 SELECT DISTINCT code, name FROM ... LIMIT 50),用于核对跨源成员编码与分布。",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"source_name": {"type": "string", "description": "要查询的数据源名称"},
|
||||
"sql": {"type": "string", "description": "只读 SELECT 查询语句"},
|
||||
},
|
||||
"required": ["source_name", "sql"],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
PROPOSE_TOOL_PROMPT: dict[str, Any] = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": PROPOSE_TOOL,
|
||||
"description": "提交一批标准业务实体的新建或修改提案(经平台核验后展示在界面待保存区,不直接写库)。",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"operations": {
|
||||
"type": "array",
|
||||
"description": "增量操作数组(upsert_entity / upsert_binding / remove_binding / upsert_member / upsert_caliber_param)",
|
||||
"items": {"type": "object"},
|
||||
},
|
||||
},
|
||||
"required": ["operations"],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _SourceEntry:
|
||||
row: DataSource
|
||||
resolved: Any
|
||||
kind: str
|
||||
masked: set[str]
|
||||
settings: catalog_profile.ProfileSettings
|
||||
can_query: bool
|
||||
catalog_notes: dict[str, Any]
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class EntityAssistantContext:
|
||||
sources: dict[str, _SourceEntry]
|
||||
workflows: list[dict[str, Any]]
|
||||
doc_bundles: list[entity_docs.EntityDocBundle]
|
||||
pending_ops: list[dict[str, Any]] = field(default_factory=list)
|
||||
system: str = ""
|
||||
|
||||
@property
|
||||
def queryable_sources(self) -> list[str]:
|
||||
return [entry.row.name for entry in self.sources.values() if entry.can_query]
|
||||
|
||||
def tools(self) -> list[dict[str, Any]]:
|
||||
items = [INSPECT_TOOL_PROMPT]
|
||||
if self.doc_bundles:
|
||||
items.append(READ_DOC_TOOL_PROMPT)
|
||||
if self.queryable_sources:
|
||||
items.append(QUERY_TOOL_PROMPT)
|
||||
items.append(PROPOSE_TOOL_PROMPT)
|
||||
return items
|
||||
|
||||
|
||||
resolve_model = catalog_assistant.resolve_model
|
||||
|
||||
|
||||
def conversation_problem(messages: Iterable[Mapping[str, Any]]) -> str | None:
|
||||
said = [m for m in messages if str(m.get("content") or "").strip()]
|
||||
if any(m.get("role") == "user" for m in said) and said[-1].get("role") != "user":
|
||||
return LAST_NOT_USER
|
||||
return None
|
||||
|
||||
|
||||
def _history(messages: Iterable[Mapping[str, Any]]) -> list[Any]:
|
||||
from langchain_core.messages import AIMessage, HumanMessage
|
||||
|
||||
merged: list[tuple[str, str]] = []
|
||||
for m in messages:
|
||||
text = str(m.get("content") or "").strip()
|
||||
if not text:
|
||||
continue
|
||||
role = "user" if m.get("role") == "user" else "assistant"
|
||||
if merged and merged[-1][0] == role:
|
||||
merged[-1] = (role, f"{merged[-1][1]}\n\n{text}")
|
||||
else:
|
||||
merged.append((role, text))
|
||||
out: list[Any] = []
|
||||
if merged and merged[0][0] == "assistant":
|
||||
out.append(HumanMessage(content=ENTITY_OPENED_PROMPT))
|
||||
for role, text in merged:
|
||||
out.append(HumanMessage(content=text) if role == "user" else AIMessage(content=text))
|
||||
if not any(role == "user" for role, _ in merged):
|
||||
out.append(HumanMessage(content=ENTITY_OPENING_PROMPT))
|
||||
return out
|
||||
|
||||
|
||||
def _extract_wf_summary(
|
||||
wf_rows: Sequence[Workflow],
|
||||
ver_rows: Sequence[WorkflowVersion],
|
||||
*,
|
||||
allowed_sources: set[str] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
from app.tools.datasource import source_of_tool
|
||||
|
||||
published_by_wf: dict[str, WorkflowVersion] = {}
|
||||
for ver in ver_rows:
|
||||
cur = published_by_wf.get(ver.workflow_id)
|
||||
if cur is None or int(ver.version or 0) > int(cur.version or 0):
|
||||
published_by_wf[ver.workflow_id] = ver
|
||||
|
||||
summaries: list[dict[str, Any]] = []
|
||||
for wf in wf_rows:
|
||||
ver = published_by_wf.get(wf.id)
|
||||
graph = (ver.graph if ver and isinstance(ver.graph, Mapping) else wf.graph) or {}
|
||||
if not isinstance(graph, Mapping):
|
||||
continue
|
||||
cards: list[dict[str, Any]] = []
|
||||
inputs: list[dict[str, Any]] = []
|
||||
wf_sources: set[str] = set()
|
||||
for node in graph.get("nodes") or []:
|
||||
if not isinstance(node, Mapping):
|
||||
continue
|
||||
ntype = str(node.get("type") or "")
|
||||
nid = str(node.get("id") or "")
|
||||
data = node.get("data") if isinstance(node.get("data"), Mapping) else {}
|
||||
cfg = data.get("config") if isinstance(data.get("config"), Mapping) else (
|
||||
node.get("config") if isinstance(node.get("config"), Mapping) else {}
|
||||
)
|
||||
t_single = str(cfg.get("tool") or "").strip()
|
||||
if s := source_of_tool(t_single):
|
||||
if "{{" not in s:
|
||||
wf_sources.add(s.lower())
|
||||
for t_item in cfg.get("tools") or []:
|
||||
if isinstance(t_item, str) and (s := source_of_tool(t_item.strip())) and "{{" not in s:
|
||||
wf_sources.add(s.lower())
|
||||
for skey in ("source", "source_name", "datasource", "source_id"):
|
||||
sv = str(cfg.get(skey) or "").strip()
|
||||
if sv and "{{" not in sv:
|
||||
wf_sources.add(sv.lower())
|
||||
|
||||
if ntype == "metrics" and nid:
|
||||
m_names = [
|
||||
str(m.get("name") or m.get("id") or "")
|
||||
for m in (cfg.get("metrics") or [])
|
||||
if isinstance(m, Mapping)
|
||||
]
|
||||
cards.append({"card_id": nid, "metrics": [x for x in m_names if x][:8]})
|
||||
elif ntype == "input":
|
||||
for f in cfg.get("fields") or []:
|
||||
if isinstance(f, Mapping) and str(f.get("name") or "").strip():
|
||||
inputs.append({
|
||||
"name": str(f["name"]).strip(),
|
||||
"label": str(f.get("label") or f.get("description") or "").strip(),
|
||||
"type": str(f.get("type") or "string"),
|
||||
})
|
||||
if allowed_sources is not None:
|
||||
if wf_sources:
|
||||
if not (wf_sources & allowed_sources):
|
||||
continue
|
||||
elif not allowed_sources:
|
||||
continue
|
||||
if cards and inputs:
|
||||
summaries.append({
|
||||
"workflow_id": wf.id,
|
||||
"name": wf.name,
|
||||
"published": ver is not None,
|
||||
"inputs": inputs,
|
||||
"cards": cards,
|
||||
"sources": sorted(wf_sources),
|
||||
})
|
||||
return summaries
|
||||
|
||||
|
||||
async def prepare(
|
||||
session: AsyncSession,
|
||||
*,
|
||||
source_ids: Sequence[str] | None = None,
|
||||
focus_entity_key: str | None = None,
|
||||
attachment_ids: Sequence[str] = (),
|
||||
document_ids: Sequence[str] = (),
|
||||
pending_ops: Sequence[Mapping[str, Any]] = (),
|
||||
) -> EntityAssistantContext:
|
||||
"""组装实体助手上下文:按选定范围过滤的数据源表目录、既有标准实体、口径入参、挂载文档摘要与待保存提案。"""
|
||||
all_enabled = list(
|
||||
(await session.execute(select(DataSource).where(DataSource.enabled.is_(True)).order_by(DataSource.name))).scalars()
|
||||
)
|
||||
if source_ids is not None:
|
||||
wanted = {str(s or "").strip().lower() for s in source_ids if str(s or "").strip()}
|
||||
src_rows = [r for r in all_enabled if r.id.lower() in wanted or r.name.lower() in wanted]
|
||||
else:
|
||||
src_rows = all_enabled
|
||||
|
||||
allowed_sources: set[str] | None = (
|
||||
{x for r in src_rows for x in (r.id.lower(), r.name.lower())}
|
||||
if source_ids is not None
|
||||
else None
|
||||
)
|
||||
|
||||
sources: dict[str, _SourceEntry] = {}
|
||||
source_lines: list[str] = ["当前分析范围的数据源:"]
|
||||
for row in src_rows:
|
||||
try:
|
||||
resolved = await table_versions.resolve_source(session, row)
|
||||
except table_versions.SnapshotMissing:
|
||||
resolved = row
|
||||
opts = dict(row.options or {}) if isinstance(row.options, Mapping) else {}
|
||||
masked = {c.lower() for c in masked_columns(opts)}
|
||||
p_settings = catalog_profile.profile_settings(opts)
|
||||
can_query = bool(p_settings.enabled and row.enabled)
|
||||
cat_entries = await catalog.read_catalog(session, resolved.id)
|
||||
notes_map = {tname: entry.notes for tname, entry in cat_entries.items()}
|
||||
sources[row.name.lower()] = _SourceEntry(
|
||||
row=row,
|
||||
resolved=resolved,
|
||||
kind=str(row.kind or ""),
|
||||
masked=masked,
|
||||
settings=p_settings,
|
||||
can_query=can_query,
|
||||
catalog_notes=notes_map,
|
||||
)
|
||||
tables = ((resolved.schema_cache or {}).get("tables") or {}) if isinstance(resolved.schema_cache, Mapping) else {}
|
||||
q_flag = "可只读查询" if can_query else "未开剖析"
|
||||
source_lines.append(f"- {row.name}({row.kind},共 {len(tables)} 张表,{q_flag}){(':' + row.description) if row.description else ''}")
|
||||
if len(source_lines) == 1:
|
||||
source_lines.append("- (当前未选定任何数据源,仅基于挂载文档与既有实体分析)")
|
||||
|
||||
clean_pending = [dict(op) for op in (pending_ops or ()) if isinstance(op, Mapping)]
|
||||
pending_keys = {
|
||||
str(op.get("entity_key") or "").strip().lower()
|
||||
for op in clean_pending
|
||||
if str(op.get("entity_key") or "").strip()
|
||||
}
|
||||
|
||||
ent_rows = await entities.list_entities(session)
|
||||
focus_norm = (focus_entity_key or "").strip().lower()
|
||||
in_scope_entities: list[Any] = []
|
||||
other_domain_keys: list[str] = []
|
||||
for erow in ent_rows:
|
||||
ekey_low = erow.key.lower()
|
||||
b_list = [b for b in (erow.bindings or []) if isinstance(b, Mapping)]
|
||||
if allowed_sources is None or ekey_low == focus_norm or ekey_low in pending_keys or not b_list:
|
||||
in_scope_entities.append(erow)
|
||||
continue
|
||||
touches_scope = any(
|
||||
str(b.get("source_name") or "").strip().lower() in allowed_sources
|
||||
or str(b.get("source_id") or "").strip().lower() in allowed_sources
|
||||
for b in b_list
|
||||
)
|
||||
if touches_scope:
|
||||
in_scope_entities.append(erow)
|
||||
else:
|
||||
other_domain_keys.append(f"{erow.key}({erow.label})")
|
||||
|
||||
entity_lines: list[str] = ["", f"当前范围相关的标准业务实体(共 {len(in_scope_entities)} 个):"]
|
||||
for erow in in_scope_entities[:60]:
|
||||
b_list = [b for b in (erow.bindings or []) if isinstance(b, Mapping)]
|
||||
if allowed_sources is not None and erow.key.lower() != focus_norm:
|
||||
b_list = [
|
||||
b for b in b_list
|
||||
if str(b.get("source_name") or "").strip().lower() in allowed_sources
|
||||
or str(b.get("source_id") or "").strip().lower() in allowed_sources
|
||||
]
|
||||
active_b = [f"{b.get('source_name')}.{b.get('table_name')}.{b.get('column_name')}" for b in b_list if b.get("status") in ("confirmed", "verified")]
|
||||
prop_b = [f"{b.get('source_name')}.{b.get('table_name')}.{b.get('column_name')}" for b in b_list if b.get("status") == "proposed"]
|
||||
rej_b = [f"{b.get('source_name')}.{b.get('table_name')}.{b.get('column_name')}" for b in b_list if b.get("status") == "rejected"]
|
||||
m_count = len(erow.members or [])
|
||||
head = f"- {erow.label}(key={erow.key},v{erow.version},成员 {m_count} 项)"
|
||||
if active_b:
|
||||
head += f"|已确认绑定:{', '.join(active_b[:8])}"
|
||||
if prop_b:
|
||||
head += f"|待确认:{', '.join(prop_b[:6])}"
|
||||
if rej_b:
|
||||
head += f"|已驳回(勿重提):{', '.join(rej_b[:6])}"
|
||||
entity_lines.append(head)
|
||||
if focus_norm and erow.key.lower() == focus_norm:
|
||||
entity_lines.append(f" 【当前使用者正在查看该实体】说明:{erow.description or '无'};别名:{', '.join(erow.aliases or []) or '无'}")
|
||||
for m in (erow.members or [])[:25]:
|
||||
if isinstance(m, Mapping):
|
||||
codes_txt = ", ".join(f"{k}={v}" for k, v in (m.get("source_codes") or {}).items())
|
||||
entity_lines.append(f" · 成员 {m.get('value')}({m.get('label')}):{codes_txt or '无跨源编码'}")
|
||||
if other_domain_keys:
|
||||
entity_lines.append(
|
||||
f"(其他未选定数据源已占用的全局实体 key,避免重名冲突,无需分析未选定源:{', '.join(other_domain_keys[:30])})"
|
||||
)
|
||||
|
||||
wf_rows = list((await session.execute(select(Workflow))).scalars())
|
||||
ver_rows = list((await session.execute(select(WorkflowVersion).where(WorkflowVersion.published_at.is_not(None)))).scalars())
|
||||
workflows = _extract_wf_summary(wf_rows, ver_rows, allowed_sources=allowed_sources)
|
||||
|
||||
doc_bundles = await entity_docs.load_doc_bundles(session, attachment_ids=attachment_ids, document_ids=document_ids)
|
||||
doc_lines: list[str] = []
|
||||
if doc_bundles:
|
||||
doc_lines.append("")
|
||||
doc_lines.append(f"本次对话挂载的业务文档/对照表(共 {len(doc_bundles)} 份,可用 read_document 检索详情):")
|
||||
for b in doc_bundles:
|
||||
preview = b.chunks[0][:300].replace("\n", " ") if b.chunks else ""
|
||||
doc_lines.append(f"- 《{b.title}》(ref={b.ref_id},共 {len(b.chunks)} 段,{b.char_count} 字)首段预览:{preview}…")
|
||||
|
||||
pending_lines: list[str] = []
|
||||
if clean_pending:
|
||||
pending_lines.append("")
|
||||
pending_lines.append(f"界面待保存区已有的修改提案(共 {len(clean_pending)} 项,避免重复提议):")
|
||||
for pop in clean_pending[-40:]:
|
||||
pending_lines.append(f"- {json.dumps(pop, ensure_ascii=False, default=str)}")
|
||||
|
||||
queryable = [e.row.name for e in sources.values() if e.can_query]
|
||||
query_prompt = (
|
||||
ENTITY_QUERY_PROMPT.format(rows=QUERY_ROWS, limit=MAX_QUERIES, sources="、".join(queryable))
|
||||
if queryable
|
||||
else ENTITY_NO_QUERY_PROMPT
|
||||
)
|
||||
|
||||
system = "\n\n".join(
|
||||
part
|
||||
for part in [
|
||||
ENTITY_ASSISTANT_SYSTEM,
|
||||
query_prompt,
|
||||
"\n".join([*source_lines, *entity_lines, *doc_lines, *pending_lines]),
|
||||
]
|
||||
if part
|
||||
)
|
||||
return EntityAssistantContext(
|
||||
sources=sources,
|
||||
workflows=workflows,
|
||||
doc_bundles=doc_bundles,
|
||||
pending_ops=clean_pending,
|
||||
system=system,
|
||||
)
|
||||
|
||||
|
||||
def _cell(value: Any, limit: int | None = None) -> Any:
|
||||
if value is None or isinstance(value, (bool, int, float)):
|
||||
return value
|
||||
if isinstance(value, (datetime, date)):
|
||||
text = value.isoformat()
|
||||
elif isinstance(value, Decimal):
|
||||
text = str(value)
|
||||
elif isinstance(value, (bytes, bytearray)):
|
||||
text = f"<{len(value)} 字节>"
|
||||
else:
|
||||
text = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False, default=str)
|
||||
if limit is not None and len(text) > limit:
|
||||
return text[:limit] + "…"
|
||||
return text
|
||||
|
||||
|
||||
def _rows(rows: Any, n: int, limit: int | None) -> list[list[Any]]:
|
||||
out: list[list[Any]] = []
|
||||
for row in (rows or [])[:n]:
|
||||
cells = row if isinstance(row, list) else list(row.values()) if isinstance(row, dict) else [row]
|
||||
out.append([_cell(v, limit) for v in cells])
|
||||
return out
|
||||
|
||||
|
||||
class _EntityTools:
|
||||
def __init__(self, ctx: EntityAssistantContext) -> None:
|
||||
from app.engine.masking import SeenValues
|
||||
|
||||
self.ctx = ctx
|
||||
self.queries = 0
|
||||
self.seen_by_source: dict[str, Any] = {k: SeenValues() for k in ctx.sources}
|
||||
|
||||
def inspect(self, keyword: Any, source_name: Any) -> str:
|
||||
kw = str(keyword or "").strip().lower()
|
||||
src_filter = str(source_name or "").strip().lower()
|
||||
lines: list[str] = []
|
||||
|
||||
for skey, entry in self.ctx.sources.items():
|
||||
if src_filter and skey != src_filter and entry.row.id.lower() != src_filter:
|
||||
continue
|
||||
tables = ((entry.resolved.schema_cache or {}).get("tables") or {}) if isinstance(entry.resolved.schema_cache, Mapping) else {}
|
||||
if not tables:
|
||||
lines.append(f"数据源 {entry.row.name}:尚未探查表结构")
|
||||
continue
|
||||
matched_tables: list[str] = []
|
||||
for tname, tmeta in tables.items():
|
||||
if not isinstance(tmeta, Mapping):
|
||||
continue
|
||||
notes = entry.catalog_notes.get(tname) if isinstance(entry.catalog_notes.get(tname), Mapping) else {}
|
||||
t_label_item = notes.get("label") if isinstance(notes.get("label"), Mapping) else {}
|
||||
t_label = str(t_label_item.get("value") or "") if t_label_item.get("status") != "rejected" else ""
|
||||
col_notes = notes.get("columns") if isinstance(notes.get("columns"), Mapping) else {}
|
||||
cols = introspect.table_columns(dict(tmeta))
|
||||
|
||||
col_hits: list[str] = []
|
||||
for col in cols:
|
||||
cname = str(col.get("name") or "")
|
||||
is_masked = (
|
||||
cname.lower() in entry.masked
|
||||
or f"{tname.lower()}.{cname.lower()}" in entry.masked
|
||||
)
|
||||
cnote = col_notes.get(cname) if isinstance(col_notes.get(cname), Mapping) else {}
|
||||
clabel = ""
|
||||
cmeaning = ""
|
||||
ccodes = ""
|
||||
if not is_masked and isinstance(cnote, Mapping):
|
||||
li = cnote.get("label")
|
||||
if isinstance(li, Mapping) and li.get("status") != "rejected":
|
||||
clabel = str(li.get("value") or "")
|
||||
mi = cnote.get("meaning")
|
||||
if isinstance(mi, Mapping) and mi.get("status") != "rejected":
|
||||
cmeaning = str(mi.get("value") or "")
|
||||
ci = cnote.get("codes")
|
||||
if isinstance(ci, Mapping) and ci.get("status") != "rejected" and isinstance(ci.get("value"), Mapping):
|
||||
pairs = [f"{k}={v}" for k, v in list(ci["value"].items())[:12]]
|
||||
ccodes = f" 码值[{', '.join(pairs)}]" if pairs else ""
|
||||
comment = "" if is_masked else str(col.get("comment") or "")
|
||||
haystack = " ".join([tname, t_label, cname, clabel, cmeaning, comment, ccodes]).lower()
|
||||
if not kw or kw in haystack:
|
||||
desc_parts = [p for p in [clabel, cmeaning, comment] if p]
|
||||
mask_tag = " [遮罩列]" if is_masked else ""
|
||||
col_hits.append(
|
||||
f" · {cname} ({col.get('type') or 'unknown'}){mask_tag}"
|
||||
+ (f":{' / '.join(desc_parts)}" if desc_parts else "")
|
||||
+ ccodes
|
||||
)
|
||||
if col_hits:
|
||||
head = f" 表 {tname}" + (f"({t_label})" if t_label else "")
|
||||
matched_tables.append(head + "\n" + "\n".join(col_hits[:25]))
|
||||
if matched_tables:
|
||||
lines.append(f"【数据源 {entry.row.name}】\n" + "\n".join(matched_tables[:15]))
|
||||
|
||||
wf_hits: list[str] = []
|
||||
for wf in self.ctx.workflows:
|
||||
hay = json.dumps(wf, ensure_ascii=False).lower()
|
||||
if not kw or kw in hay:
|
||||
in_txt = ", ".join(f"{i['name']}({i['label'] or i['type']})" for i in wf["inputs"])
|
||||
cd_txt = ", ".join(f"{c['card_id']}[{'/'.join(c['metrics'])}]" for c in wf["cards"])
|
||||
pub_tag = "已发布" if wf["published"] else "草稿"
|
||||
wf_hits.append(f"- 工作流 {wf['name']}(workflow_id={wf['workflow_id']},{pub_tag})|入口参数:{in_txt}|口径卡:{cd_txt}")
|
||||
if wf_hits:
|
||||
lines.append("【口径工作流与入口参数】\n" + "\n".join(wf_hits[:20]))
|
||||
|
||||
if not lines:
|
||||
return f"未找到与「{keyword}」匹配的表列或口径参数,请尝试更短的词根或留空 keyword 查看全景概览。"
|
||||
text = "\n\n".join(lines)
|
||||
return text[:TOOL_CHARS] + ("…(已截断)" if len(text) > TOOL_CHARS else "")
|
||||
|
||||
def read_doc(self, query: Any, doc_ref: Any, start_chunk: Any, limit: Any) -> str:
|
||||
if not self.ctx.doc_bundles:
|
||||
return ENTITY_TOOL_PROMPT["no_docs"]
|
||||
hits = entity_docs.search_doc_bundles(
|
||||
self.ctx.doc_bundles,
|
||||
query=str(query or ""),
|
||||
doc_ref=str(doc_ref or ""),
|
||||
start_chunk=int(start_chunk) if isinstance(start_chunk, int) else 0,
|
||||
limit=int(limit) if isinstance(limit, int) else 6,
|
||||
)
|
||||
if not hits:
|
||||
return "未检索到匹配的文档段落,可清空 query 按 start_chunk=0 顺序浏览。"
|
||||
blocks = [
|
||||
f"【{h['citation']}(共 {h['total_chunks']} 段,ref={h['doc_ref']})】\n{h['content']}"
|
||||
for h in hits
|
||||
]
|
||||
text = "\n\n".join(blocks)
|
||||
return text[:TOOL_CHARS] + ("…(已截断)" if len(text) > TOOL_CHARS else "")
|
||||
|
||||
async def query(self, source_name: Any, sql: Any) -> tuple[dict[str, Any] | None, str]:
|
||||
prompts = ENTITY_TOOL_PROMPT
|
||||
src_key = str(source_name or "").strip().lower()
|
||||
entry = self.ctx.sources.get(src_key)
|
||||
if entry is None or not entry.can_query:
|
||||
allowed = "、".join(self.ctx.queryable_sources) or "无"
|
||||
return None, prompts["query_disabled_source"].format(source=source_name or "", allowed=allowed)
|
||||
if self.queries >= MAX_QUERIES:
|
||||
return None, prompts["query_capped"].format(n=MAX_QUERIES)
|
||||
clean_sql = sql.strip() if isinstance(sql, str) else ""
|
||||
if not clean_sql:
|
||||
return None, prompts["empty_sql"]
|
||||
|
||||
self.queries += 1
|
||||
seen = self.seen_by_source[src_key]
|
||||
mask = masked_columns(dict(entry.row.options or {}) if isinstance(entry.row.options, Mapping) else None)
|
||||
known = await asyncio.to_thread(seen.pooled)
|
||||
known = known if known.strong else None
|
||||
error: str | None = None
|
||||
result: Any = None
|
||||
try:
|
||||
result = await catalog_profile.run_single_query(
|
||||
entry.resolved,
|
||||
clean_sql,
|
||||
settings=entry.settings,
|
||||
max_rows=QUERY_ROWS,
|
||||
kind=entry.kind,
|
||||
)
|
||||
except catalog_profile.QueryFailed as exc:
|
||||
error = exc.detail
|
||||
except SnapshotTampered as exc:
|
||||
error = QUERY_TAMPERED.format(name=entry.row.name, error=exc)
|
||||
except ValueError as exc:
|
||||
error = str(exc)
|
||||
|
||||
if error is not None:
|
||||
if mask or known is not None:
|
||||
from app.engine.masking import mask_error_text
|
||||
|
||||
error = mask_error_text(error, clean_sql, known)
|
||||
event = {
|
||||
"type": "query",
|
||||
"source": entry.row.name,
|
||||
"sql": clean_sql,
|
||||
"columns": [],
|
||||
"rows": [],
|
||||
"row_count": 0,
|
||||
"truncated": False,
|
||||
"error": error,
|
||||
}
|
||||
return event, prompts["query_failed"].format(error=error)
|
||||
|
||||
payload = result.to_payload()
|
||||
if mask or known is not None:
|
||||
from app.engine.masking import mask_query_result
|
||||
|
||||
schema = getattr(entry.resolved, "schema_cache", None)
|
||||
done = await asyncio.to_thread(mask_query_result, payload, mask, entry.kind, known, schema=schema)
|
||||
payload = done.payload
|
||||
await asyncio.to_thread(seen.add, done.secrets)
|
||||
|
||||
columns = [str(c) for c in payload.get("columns") or []]
|
||||
rows = payload.get("rows") or []
|
||||
event = {
|
||||
"type": "query",
|
||||
"source": entry.row.name,
|
||||
"sql": str(payload.get("sql") or clean_sql),
|
||||
"columns": columns,
|
||||
"rows": _rows(rows, SHOWN_ROWS, None),
|
||||
"row_count": int(payload.get("row_count") or len(rows)),
|
||||
"truncated": bool(payload.get("truncated")),
|
||||
"error": None,
|
||||
}
|
||||
given = {
|
||||
"source": entry.row.name,
|
||||
"columns": columns,
|
||||
"rows": _rows(rows, QUERY_ROWS, CELL_MAX),
|
||||
"row_count": event["row_count"],
|
||||
"truncated": event["truncated"],
|
||||
}
|
||||
text = json.dumps(given, ensure_ascii=False, default=str)
|
||||
while len(text) > TOOL_CHARS and given["rows"]:
|
||||
given["rows"] = given["rows"][: max(1, len(given["rows"]) // 2)] if len(given["rows"]) > 1 else []
|
||||
text = json.dumps(given, ensure_ascii=False, default=str)
|
||||
return event, text
|
||||
|
||||
async def propose(self, operations: Any) -> tuple[dict[str, Any] | None, str]:
|
||||
prompts = ENTITY_TOOL_PROMPT
|
||||
if not isinstance(operations, list) or not operations:
|
||||
return None, prompts["no_ops"]
|
||||
async with SessionLocal() as session:
|
||||
plan = await entity_patch.plan_entity_patch(
|
||||
session,
|
||||
operations,
|
||||
pending_ops=self.ctx.pending_ops,
|
||||
mode="confirmed",
|
||||
)
|
||||
said: list[str] = []
|
||||
event: dict[str, Any] | None = None
|
||||
if plan.changes:
|
||||
for ch in plan.changes:
|
||||
self.ctx.pending_ops.append(ch.op_payload)
|
||||
event = {
|
||||
"type": "proposal",
|
||||
"changes": [ch.as_dict() for ch in plan.changes],
|
||||
"expected_versions": plan.expected_versions,
|
||||
}
|
||||
said.append(
|
||||
prompts["proposed"].format(
|
||||
n=len(plan.changes),
|
||||
items=";".join(ch.summary for ch in plan.changes),
|
||||
)
|
||||
)
|
||||
else:
|
||||
said.append(prompts["nothing"])
|
||||
if plan.problems:
|
||||
said.append(prompts["dropped"].format(reasons=";".join(plan.problems)))
|
||||
return event, "\n".join(said)
|
||||
|
||||
|
||||
_SENTENCE_END = ("。", "!", "?", ";", ":", ".", "!", "?", ";", ":")
|
||||
|
||||
|
||||
def _sse(obj: Mapping[str, Any]) -> str:
|
||||
return f"data: {json.dumps(obj, ensure_ascii=False, default=str)}\n\n"
|
||||
|
||||
|
||||
class _CallFailed(Exception):
|
||||
def __init__(self, code: str, message: str) -> None:
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
self.message = message
|
||||
|
||||
|
||||
def _seconds(value: Any) -> str:
|
||||
number = float(value)
|
||||
return str(int(number)) if number.is_integer() else f"{number:g}"
|
||||
|
||||
|
||||
def _transient(exc: BaseException) -> bool:
|
||||
from app.core.errors import NETWORK, RATE_LIMIT, SERVER, classify
|
||||
|
||||
return classify(exc) in (RATE_LIMIT, SERVER, NETWORK)
|
||||
|
||||
|
||||
async def _pump(runner: Any, history: list[Any], queue: asyncio.Queue[tuple[str, Any]]) -> None:
|
||||
try:
|
||||
async for chunk in runner.astream(history):
|
||||
await queue.put(("chunk", chunk))
|
||||
await queue.put(("end", None))
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001
|
||||
await queue.put(("error", exc))
|
||||
|
||||
|
||||
async def _round(runner: Any, history: list[Any], model_id: str) -> AsyncIterator[tuple[str, Any]]:
|
||||
from app.core.errors import TIMEOUT, classify, describe_exception
|
||||
from app.engine.state import message_text
|
||||
from app.providers.budget import cut_off
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
attempt = 0
|
||||
while True:
|
||||
queue: asyncio.Queue[tuple[str, Any]] = asyncio.Queue()
|
||||
task = asyncio.ensure_future(_pump(runner, history, queue))
|
||||
deadline = loop.time() + float(catalog.MODEL_TIMEOUT_S)
|
||||
message: Any = None
|
||||
cut = False
|
||||
started = False
|
||||
failure: BaseException | None = None
|
||||
try:
|
||||
while True:
|
||||
remaining = deadline - loop.time()
|
||||
if remaining <= 0:
|
||||
raise _CallFailed(
|
||||
CODE_TIMEOUT,
|
||||
ERROR_TIMEOUT.format(seconds=_seconds(catalog.MODEL_TIMEOUT_S), place=DRAFT_MODEL_PLACE),
|
||||
)
|
||||
try:
|
||||
kind, value = await asyncio.wait_for(queue.get(), timeout=min(remaining, HEARTBEAT_S))
|
||||
except asyncio.TimeoutError:
|
||||
yield "beat", None
|
||||
continue
|
||||
if kind == "end":
|
||||
break
|
||||
if kind == "error":
|
||||
failure = value
|
||||
break
|
||||
message = value if message is None else message + value
|
||||
cut = cut or bool(cut_off(value))
|
||||
text = message_text(value)
|
||||
if text:
|
||||
started = True
|
||||
yield "text", text
|
||||
finally:
|
||||
if not task.done():
|
||||
task.cancel()
|
||||
with contextlib.suppress(asyncio.CancelledError, Exception):
|
||||
await task
|
||||
if failure is None:
|
||||
yield "message", (message, cut)
|
||||
return
|
||||
if not started and attempt < len(RETRY_DELAYS_S) and _transient(failure):
|
||||
logger.info("实体助手:调用模型出错(%s),%s 秒后重试", type(failure).__name__, RETRY_DELAYS_S[attempt])
|
||||
await asyncio.sleep(RETRY_DELAYS_S[attempt])
|
||||
attempt += 1
|
||||
continue
|
||||
if isinstance(failure, TimeoutError) or classify(failure) == TIMEOUT:
|
||||
raise _CallFailed(
|
||||
CODE_TIMEOUT,
|
||||
ERROR_TIMEOUT.format(seconds=_seconds(catalog.MODEL_TIMEOUT_S), place=DRAFT_MODEL_PLACE),
|
||||
) from failure
|
||||
raise _CallFailed(
|
||||
CODE_MODEL,
|
||||
ERROR_MODEL.format(model=model_id or "", reason=describe_exception(failure), place=DRAFT_MODEL_PLACE),
|
||||
) from failure
|
||||
|
||||
|
||||
async def stream(
|
||||
model: Any,
|
||||
model_id: str,
|
||||
ctx: EntityAssistantContext,
|
||||
messages: Iterable[Mapping[str, Any]],
|
||||
) -> AsyncIterator[str]:
|
||||
"""实体助手流式生成器:逐行产出 SSE data 事件与心跳注释。"""
|
||||
from langchain_core.messages import SystemMessage, ToolMessage, message_chunk_to_message
|
||||
|
||||
from app.providers.budget import cut_words
|
||||
|
||||
history: list[Any] = [SystemMessage(content=ctx.system), *_history(messages)]
|
||||
tools = _EntityTools(ctx)
|
||||
usage = {"input_tokens": 0, "output_tokens": 0}
|
||||
wrote = ""
|
||||
tool_defs = ctx.tools()
|
||||
names = [t["function"]["name"] for t in tool_defs]
|
||||
try:
|
||||
runner = model.bind_tools(tool_defs)
|
||||
for round_no in range(MAX_ROUNDS):
|
||||
fresh = True
|
||||
message: Any = None
|
||||
cut = False
|
||||
async with contextlib.aclosing(_round(runner, history, model_id)) as rounds:
|
||||
async for kind, value in rounds:
|
||||
if kind == "beat":
|
||||
yield ": keep-alive\n\n"
|
||||
elif kind == "text":
|
||||
if fresh and wrote.endswith(_SENTENCE_END):
|
||||
yield _sse({"type": "text", "delta": "\n\n"})
|
||||
fresh = False
|
||||
wrote = value
|
||||
yield _sse({"type": "text", "delta": value})
|
||||
else:
|
||||
message, cut = value
|
||||
meta = getattr(message, "usage_metadata", None) or {}
|
||||
usage["input_tokens"] += int(meta.get("input_tokens") or 0)
|
||||
usage["output_tokens"] += int(meta.get("output_tokens") or 0)
|
||||
if cut:
|
||||
limit = getattr(model, "max_tokens", None)
|
||||
yield _sse({
|
||||
"type": "error",
|
||||
"code": CODE_TRUNCATED,
|
||||
"message": ERROR_TRUNCATED.format(cut=cut_words("实体助手", limit), place=DRAFT_MODEL_PLACE),
|
||||
})
|
||||
return
|
||||
calls = list(getattr(message, "tool_calls", None) or []) if message is not None else []
|
||||
if not calls:
|
||||
yield _sse({"type": "done", "usage": usage, "queries": tools.queries})
|
||||
return
|
||||
if round_no == MAX_ROUNDS - 1:
|
||||
yield _sse({"type": "error", "code": CODE_ROUNDS, "message": ERROR_ROUNDS.format(n=MAX_ROUNDS)})
|
||||
return
|
||||
history.append(message_chunk_to_message(message))
|
||||
for call in calls:
|
||||
name = str(call.get("name") or "")
|
||||
args = call.get("args") if isinstance(call.get("args"), Mapping) else {}
|
||||
call_id = str(call.get("id") or f"{name}-{round_no}-{len(history)}")
|
||||
event: dict[str, Any] | None = None
|
||||
if name == INSPECT_TOOL:
|
||||
content = tools.inspect(args.get("keyword"), args.get("source_name"))
|
||||
elif name == READ_DOC_TOOL:
|
||||
content = tools.read_doc(
|
||||
args.get("query"),
|
||||
args.get("doc_ref"),
|
||||
args.get("start_chunk"),
|
||||
args.get("limit"),
|
||||
)
|
||||
elif name == QUERY_TOOL:
|
||||
event, content = await tools.query(args.get("source_name"), args.get("sql"))
|
||||
elif name == PROPOSE_TOOL:
|
||||
event, content = await tools.propose(args.get("operations"))
|
||||
else:
|
||||
content = ENTITY_TOOL_PROMPT["unknown_tool"].format(name=name, tools="、".join(names))
|
||||
if event is not None:
|
||||
yield _sse(event)
|
||||
if round_no == MAX_ROUNDS - 2:
|
||||
content = f"{content}\n{ENTITY_TOOL_PROMPT['last_round']}"
|
||||
history.append(ToolMessage(content=content, tool_call_id=call_id))
|
||||
except _CallFailed as exc:
|
||||
yield _sse({"type": "error", "code": exc.code, "message": exc.message})
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001
|
||||
from app.core.errors import describe_exception, first_line
|
||||
|
||||
logger.exception("实体助手执行异常")
|
||||
yield _sse({
|
||||
"type": "error",
|
||||
"code": CODE_FAILED,
|
||||
"message": ERROR_FAILED.format(line=first_line(exc) or describe_exception(exc)),
|
||||
})
|
||||
@@ -0,0 +1,687 @@
|
||||
"""后台主动发现候选标准业务实体任务(E2:扫描发现候选实体)。
|
||||
|
||||
扫描范围:
|
||||
1. 所选已启用数据源的表结构(主键、外键、标识/属性/状态列)与数据目录(中文名、含义、非遮罩码值);
|
||||
2. 已发布与草稿口径工作流的入口参数(entry_fields / input 节点)及口径卡;
|
||||
3. 可选挂载的业务文档或跨系统编码对照表(知识库 Document 或临时上传附件)。
|
||||
|
||||
铁律:
|
||||
- 发现结果一律经 entity_patch.apply_entity_patch(..., mode="proposed") 写入,所有表列绑定与口径入参映射强制标记为
|
||||
status="proposed", source="ai",绝不直接参与线上跨源穿透,也绝不覆盖人工已确认、已验证或已驳回的项。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from collections.abc import Mapping, Sequence
|
||||
from datetime import timezone
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.errors import describe_exception, first_line
|
||||
from app.data import catalog, catalog_assistant, entities, entity_assistant, entity_docs, entity_patch, introspect, table_versions
|
||||
from app.data.engine import masked_columns
|
||||
from app.db.base import SessionLocal, utcnow
|
||||
from app.db.models import DataSource, Workflow, WorkflowVersion
|
||||
from app.db.models_ontology import EntityDiscoveryJob
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
OPEN_STATUSES = ("queued", "running")
|
||||
NOTE_MAX = 1000
|
||||
MAX_DOC_CHARS_IN_DISCOVERY = 28_000
|
||||
MAX_SCHEMA_CHARS_IN_DISCOVERY = 36_000
|
||||
|
||||
DISCOVERY_BUSY_CODE = "discovery_job_busy"
|
||||
DISCOVERY_FINISHED_CODE = "discovery_job_finished"
|
||||
|
||||
INTERRUPTED_ERROR = "服务重启,候选实体扫描中断"
|
||||
JOB_FAILED_TEXT = "扫描候选实体失败:{reason}"
|
||||
|
||||
_FENCE_RE = re.compile(r"```(?:json)?\s*([\s\S]*?)```", re.IGNORECASE)
|
||||
_ACTIVE_TASKS: dict[str, asyncio.Task[Any]] = {}
|
||||
_CREATE_LOCK = asyncio.Lock()
|
||||
|
||||
|
||||
class DiscoveryBusy(Exception):
|
||||
"""已有正在排队或执行的候选实体扫描任务。"""
|
||||
|
||||
|
||||
class DiscoveryFinished(Exception):
|
||||
"""要取消的候选实体扫描任务已经结束。"""
|
||||
|
||||
|
||||
class DiscoveryEmptyScope(ValueError):
|
||||
"""扫描范围内没有任何可用的数据源表结构或业务文档。"""
|
||||
|
||||
|
||||
DISCOVERY_SYSTEM_PROMPT = """你是 AgentLab 的「业务实体发现专家」。请根据给定的数据源表结构与数据目录、口径工作流入口参数以及业务文档/编码对照表,识别共用的「标准业务实体」(如产线、工厂、车间、仓库、门店、销售大区、渠道、客户等级、不良代码等),并输出增量操作 JSON。
|
||||
|
||||
硬性纪律:
|
||||
1. 只输出一个合法 JSON 对象,格式为 {"operations": [...]},不要输出任何额外解释文字。
|
||||
2. 若输入包含多个数据源,优先发现在多源之间形成桥接的核心业务对象;若仅包含单个数据源,则发现在该源多张业务表、口径入口参数或业务文档之间共用的核心业务维度实体。纯技术自增流水主键不要提议为业务实体。
|
||||
3. 表名、列名、数据源名、工作流 ID、参数名必须逐字取自输入材料,严禁编造不存在的字段。
|
||||
4. 标着「[遮罩列]」的敏感字段严禁为其生成 upsert_member 编码映射;标着「已驳回」的历史绑定严禁重复提议。
|
||||
5. 每条操作必须填写简明的中文 reason:
|
||||
- 来自文档或对照表的写「文档《文档名》写着:……」;
|
||||
- 来自表目录或字段语义对齐的写「目录语义对齐:……」。
|
||||
|
||||
operations 支持以下 4 种结构:
|
||||
- {"op": "upsert_entity", "entity_key": "小写英文下划线标识(如 production_line)", "label": "中文名称", "description": "粒度与业务对齐说明", "aliases": ["同义词"], "reason": "依据"}
|
||||
- {"op": "upsert_binding", "entity_key": "实体标识", "entity_label": "中文名称", "source_name": "数据源名称", "table_name": "物理表名", "column_name": "物理列名", "note": "字段角色备注", "reason": "依据"}
|
||||
- {"op": "upsert_member", "entity_key": "实体标识", "entity_label": "中文名称", "value": "标准编码值", "label": "标准展示名", "aliases": ["别名"], "source_codes": {"数据源名称": "该源编码值"}, "reason": "依据"}
|
||||
- {"op": "upsert_caliber_param", "entity_key": "实体标识", "entity_label": "中文名称", "workflow_id": "工作流ID", "card_id": "口径卡节点ID(可空串)", "param_name": "入口参数名", "note": "参数对齐说明", "reason": "依据"}
|
||||
"""
|
||||
|
||||
DISCOVERY_USER_PROMPT = """请扫描以下数据源结构、口径参数与业务文档,提取候选标准业务实体及其跨源绑定与编码映射,输出 {{"operations": [...]}} JSON:
|
||||
|
||||
{body}
|
||||
"""
|
||||
|
||||
|
||||
def _iso(val: Any) -> str | None:
|
||||
if val is None or not hasattr(val, "isoformat"):
|
||||
return None
|
||||
if getattr(val, "tzinfo", None) is None:
|
||||
val = val.replace(tzinfo=timezone.utc)
|
||||
return val.isoformat()
|
||||
|
||||
|
||||
def job_to_dict(row: EntityDiscoveryJob) -> dict[str, Any]:
|
||||
return {
|
||||
"id": row.id,
|
||||
"status": row.status,
|
||||
"source_ids": list(row.source_ids or []),
|
||||
"document_ids": list(row.document_ids or []),
|
||||
"attachment_ids": list(row.attachment_ids or []),
|
||||
"note": row.note or "",
|
||||
"phase": row.phase or "",
|
||||
"summary": dict(row.summary or {}),
|
||||
"error": row.error,
|
||||
"started_by": row.started_by,
|
||||
"started_at": _iso(row.started_at),
|
||||
"finished_at": _iso(row.finished_at),
|
||||
"created_at": _iso(row.created_at),
|
||||
"updated_at": _iso(row.updated_at),
|
||||
}
|
||||
|
||||
|
||||
def parse_discovery_operations(raw_text: str) -> list[dict[str, Any]]:
|
||||
"""从模型回复中提取 operations 列表。"""
|
||||
text = (raw_text or "").strip()
|
||||
if not text:
|
||||
return []
|
||||
m = _FENCE_RE.search(text)
|
||||
candidate = m.group(1).strip() if m else text
|
||||
if not candidate.startswith(("{", "[")):
|
||||
brace = candidate.find("{")
|
||||
bracket = candidate.find("[")
|
||||
starts = [i for i in (brace, bracket) if i >= 0]
|
||||
if starts:
|
||||
candidate = candidate[min(starts) :]
|
||||
try:
|
||||
parsed = json.loads(candidate)
|
||||
except ValueError:
|
||||
# 尝试截取最后一个匹配括号
|
||||
end_brace = candidate.rfind("}")
|
||||
if end_brace > 0:
|
||||
try:
|
||||
parsed = json.loads(candidate[: end_brace + 1])
|
||||
except ValueError:
|
||||
return []
|
||||
else:
|
||||
return []
|
||||
if isinstance(parsed, list):
|
||||
return [dict(x) for x in parsed if isinstance(x, Mapping)]
|
||||
if isinstance(parsed, Mapping):
|
||||
ops = parsed.get("operations")
|
||||
if isinstance(ops, list):
|
||||
return [dict(x) for x in ops if isinstance(x, Mapping)]
|
||||
return []
|
||||
|
||||
|
||||
async def get_active_or_latest_job(session: AsyncSession) -> dict[str, Any]:
|
||||
active = (
|
||||
await session.execute(
|
||||
select(EntityDiscoveryJob)
|
||||
.where(EntityDiscoveryJob.status.in_(OPEN_STATUSES))
|
||||
.order_by(EntityDiscoveryJob.created_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
latest = (
|
||||
await session.execute(
|
||||
select(EntityDiscoveryJob)
|
||||
.order_by(EntityDiscoveryJob.created_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
return {
|
||||
"job": job_to_dict(active) if active is not None else None,
|
||||
"latest": job_to_dict(latest) if latest is not None else None,
|
||||
}
|
||||
|
||||
|
||||
async def get_job(session: AsyncSession, job_id: str) -> EntityDiscoveryJob | None:
|
||||
return await session.get(EntityDiscoveryJob, str(job_id or "").strip())
|
||||
|
||||
|
||||
async def _build_discovery_prompt_body(
|
||||
session: AsyncSession,
|
||||
*,
|
||||
source_ids: Sequence[str],
|
||||
document_ids: Sequence[str],
|
||||
attachment_ids: Sequence[str],
|
||||
note: str,
|
||||
) -> tuple[str, dict[str, Any]]:
|
||||
"""返回 (prompt_body, 扫描范围量化统计)。"""
|
||||
stmt = select(DataSource).where(DataSource.enabled.is_(True)).order_by(DataSource.name)
|
||||
if source_ids:
|
||||
stmt = stmt.where(DataSource.id.in_(list(source_ids)))
|
||||
src_rows = list((await session.execute(stmt)).scalars())
|
||||
|
||||
schema_blocks: list[str] = []
|
||||
valid_sources = 0
|
||||
source_names: list[str] = []
|
||||
tables_scanned = 0
|
||||
columns_scanned = 0
|
||||
for row in src_rows:
|
||||
try:
|
||||
resolved = await table_versions.resolve_source(session, row)
|
||||
except table_versions.SnapshotMissing:
|
||||
resolved = row
|
||||
tables = ((resolved.schema_cache or {}).get("tables") or {}) if isinstance(resolved.schema_cache, Mapping) else {}
|
||||
if not tables:
|
||||
continue
|
||||
valid_sources += 1
|
||||
source_names.append(row.name)
|
||||
opts = dict(row.options or {}) if isinstance(row.options, Mapping) else {}
|
||||
masked = {c.lower() for c in masked_columns(opts)}
|
||||
cat_entries = await catalog.read_catalog(session, resolved.id)
|
||||
t_lines = [f"### 数据源:{row.name}({row.kind})"]
|
||||
for tname, tmeta in list(tables.items())[:40]:
|
||||
if not isinstance(tmeta, Mapping):
|
||||
continue
|
||||
entry = cat_entries.get(tname)
|
||||
notes = entry.notes if entry else {}
|
||||
t_label_item = notes.get("label") if isinstance(notes.get("label"), Mapping) else {}
|
||||
t_label = str(t_label_item.get("value") or "") if t_label_item.get("status") != "rejected" else ""
|
||||
col_notes = notes.get("columns") if isinstance(notes.get("columns"), Mapping) else {}
|
||||
cols = introspect.table_columns(dict(tmeta))
|
||||
c_parts: list[str] = []
|
||||
for col in cols[:40]:
|
||||
cname = str(col.get("name") or "")
|
||||
is_masked = cname.lower() in masked or f"{tname.lower()}.{cname.lower()}" in masked
|
||||
cnote = col_notes.get(cname) if isinstance(cnote := col_notes.get(cname), Mapping) else {}
|
||||
measure = ""
|
||||
clabel = ""
|
||||
cmeaning = ""
|
||||
ccodes = ""
|
||||
if not is_masked and isinstance(cnote, Mapping):
|
||||
mi = cnote.get("measure")
|
||||
if isinstance(mi, Mapping) and mi.get("status") != "rejected":
|
||||
measure = str(mi.get("value") or "")
|
||||
# 跳过明显的纯度量浮点流量列,聚焦维度/标识/状态/外键或未分类列
|
||||
if measure in ("flow", "stock", "ratio"):
|
||||
continue
|
||||
li = cnote.get("label")
|
||||
if isinstance(li, Mapping) and li.get("status") != "rejected":
|
||||
clabel = str(li.get("value") or "")
|
||||
mgi = cnote.get("meaning")
|
||||
if isinstance(mgi, Mapping) and mgi.get("status") != "rejected":
|
||||
cmeaning = str(mgi.get("value") or "")
|
||||
ci = cnote.get("codes")
|
||||
if isinstance(ci, Mapping) and ci.get("status") != "rejected" and isinstance(ci.get("value"), Mapping):
|
||||
pairs = [f"{k}={v}" for k, v in list(ci["value"].items())[:15]]
|
||||
ccodes = f" 码值[{', '.join(pairs)}]" if pairs else ""
|
||||
comment = "" if is_masked else str(col.get("comment") or "")
|
||||
tags = [p for p in [clabel, cmeaning, comment] if p]
|
||||
pk_tag = "[PK]" if col.get("pk") else ""
|
||||
mask_tag = "[遮罩列]" if is_masked else ""
|
||||
c_parts.append(
|
||||
f" - {cname} ({col.get('type') or ''}){pk_tag}{mask_tag}"
|
||||
+ (f":{' / '.join(tags)}" if tags else "")
|
||||
+ ccodes
|
||||
)
|
||||
if c_parts:
|
||||
tables_scanned += 1
|
||||
columns_scanned += len(c_parts)
|
||||
t_lines.append(f"- 表 {tname}" + (f"({t_label})" if t_label else "") + "\n" + "\n".join(c_parts))
|
||||
block = "\n".join(t_lines)
|
||||
schema_blocks.append(block)
|
||||
|
||||
schema_text = "\n\n".join(schema_blocks)
|
||||
if len(schema_text) > MAX_SCHEMA_CHARS_IN_DISCOVERY:
|
||||
schema_text = schema_text[:MAX_SCHEMA_CHARS_IN_DISCOVERY] + "\n…(表结构摘要已截断)"
|
||||
|
||||
allowed_sources: set[str] | None = (
|
||||
{x for r in src_rows for x in (r.id.lower(), r.name.lower())}
|
||||
if source_ids
|
||||
else None
|
||||
)
|
||||
|
||||
wf_rows = list((await session.execute(select(Workflow))).scalars())
|
||||
ver_rows = list((await session.execute(select(WorkflowVersion).where(WorkflowVersion.published_at.is_not(None)))).scalars())
|
||||
workflows = entity_assistant._extract_wf_summary(wf_rows, ver_rows, allowed_sources=allowed_sources)
|
||||
wf_lines: list[str] = []
|
||||
if workflows:
|
||||
wf_lines.append("## 口径工作流与入口参数")
|
||||
for wf in workflows[:30]:
|
||||
ins = ", ".join(f"{i['name']}({i['label'] or i['type']})" for i in wf["inputs"])
|
||||
cds = ", ".join(f"{c['card_id']}[{'/'.join(c['metrics'])}]" for c in wf["cards"])
|
||||
wf_lines.append(f"- {wf['name']}(workflow_id={wf['workflow_id']})|入口参数:{ins}|口径卡:{cds}")
|
||||
|
||||
ent_rows = await entities.list_entities(session)
|
||||
ent_lines: list[str] = []
|
||||
if ent_rows:
|
||||
ent_lines.append("## 既有标准业务实体(请复用同域已有 entity_key 增补绑定,勿重提已驳回项)")
|
||||
other_keys: list[str] = []
|
||||
for erow in ent_rows[:60]:
|
||||
b_list = [b for b in (erow.bindings or []) if isinstance(b, Mapping)]
|
||||
if allowed_sources is not None and b_list:
|
||||
b_scoped = [
|
||||
b for b in b_list
|
||||
if str(b.get("source_name") or "").strip().lower() in allowed_sources
|
||||
or str(b.get("source_id") or "").strip().lower() in allowed_sources
|
||||
]
|
||||
if not b_scoped:
|
||||
other_keys.append(f"{erow.key}({erow.label})")
|
||||
continue
|
||||
b_list = b_scoped
|
||||
active_b = [f"{b.get('source_name')}.{b.get('table_name')}.{b.get('column_name')}" for b in b_list if b.get("status") != "rejected"]
|
||||
rej_b = [f"{b.get('source_name')}.{b.get('table_name')}.{b.get('column_name')}" for b in b_list if b.get("status") == "rejected"]
|
||||
line = f"- {erow.label}(entity_key={erow.key})|已有绑定:{', '.join(active_b) or '无'}"
|
||||
if rej_b:
|
||||
line += f"|已驳回:{', '.join(rej_b)}"
|
||||
ent_lines.append(line)
|
||||
if other_keys:
|
||||
ent_lines.append(f"(其他未选定数据源已占用的实体标识:{', '.join(other_keys[:30])})")
|
||||
|
||||
bundles = await entity_docs.load_doc_bundles(session, attachment_ids=attachment_ids, document_ids=document_ids)
|
||||
doc_blocks: list[str] = []
|
||||
used_doc_chars = 0
|
||||
for bundle in bundles:
|
||||
piece_text = "\n\n".join(bundle.chunks)
|
||||
remain = MAX_DOC_CHARS_IN_DISCOVERY - used_doc_chars
|
||||
if remain <= 0:
|
||||
break
|
||||
clipped = piece_text[:remain]
|
||||
used_doc_chars += len(clipped)
|
||||
doc_blocks.append(f"### 文档《{bundle.title}》\n{clipped}")
|
||||
|
||||
sections: list[str] = []
|
||||
if note.strip():
|
||||
sections.append(f"## 使用者补充的扫描重点\n{note.strip()}")
|
||||
if schema_text:
|
||||
sections.append(f"## 数据源与表目录\n{schema_text}")
|
||||
if wf_lines:
|
||||
sections.append("\n".join(wf_lines))
|
||||
if ent_lines:
|
||||
sections.append("\n".join(ent_lines))
|
||||
if doc_blocks:
|
||||
sections.append("## 业务文档与主数据对照表\n" + "\n\n".join(doc_blocks))
|
||||
|
||||
scope_stats: dict[str, Any] = {
|
||||
"sources_scanned": valid_sources,
|
||||
"source_names": source_names,
|
||||
"tables_scanned": tables_scanned,
|
||||
"columns_scanned": columns_scanned,
|
||||
"workflows_scanned": len(workflows),
|
||||
"documents_scanned": len(bundles),
|
||||
"doc_titles": [b.title for b in bundles[:15]],
|
||||
"chunks_scanned": sum(len(b.chunks) for b in bundles),
|
||||
}
|
||||
return "\n\n".join(sections), scope_stats
|
||||
|
||||
|
||||
async def create_job(
|
||||
session: AsyncSession,
|
||||
*,
|
||||
source_ids: Sequence[str] | None = None,
|
||||
document_ids: Sequence[str] | None = None,
|
||||
attachment_ids: Sequence[str] | None = None,
|
||||
note: str | None = None,
|
||||
actor: str | None = None,
|
||||
) -> EntityDiscoveryJob:
|
||||
clean_note = (note or "").strip()
|
||||
if len(clean_note) > NOTE_MAX:
|
||||
raise ValueError(f"扫描补充说明最多 {NOTE_MAX} 字,当前 {len(clean_note)} 字")
|
||||
|
||||
# 建任务前先校验起草模型可用性
|
||||
await catalog_assistant.resolve_model(session)
|
||||
|
||||
async with _CREATE_LOCK:
|
||||
existing = (
|
||||
await session.execute(
|
||||
select(EntityDiscoveryJob).where(EntityDiscoveryJob.status.in_(OPEN_STATUSES)).limit(1)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if existing is not None:
|
||||
raise DiscoveryBusy("已有一个正在执行的候选实体扫描任务,请等待完成后再发起")
|
||||
|
||||
clean_sources = list(dict.fromkeys(str(s).strip() for s in (source_ids or []) if str(s or "").strip()))
|
||||
clean_docs = list(dict.fromkeys(str(d).strip() for d in (document_ids or []) if str(d or "").strip()))
|
||||
clean_atts = list(dict.fromkeys(str(a).strip() for a in (attachment_ids or []) if str(a or "").strip()))
|
||||
|
||||
prompt_body, scope_stats = await _build_discovery_prompt_body(
|
||||
session,
|
||||
source_ids=clean_sources,
|
||||
document_ids=clean_docs,
|
||||
attachment_ids=clean_atts,
|
||||
note=clean_note,
|
||||
)
|
||||
if scope_stats["sources_scanned"] == 0 and scope_stats["documents_scanned"] == 0 or not prompt_body.strip():
|
||||
raise DiscoveryEmptyScope("所选范围内没有已探查表结构的数据源或有效业务文档,请先探查表结构或添加文档")
|
||||
|
||||
job = EntityDiscoveryJob(
|
||||
status="queued",
|
||||
source_ids=clean_sources,
|
||||
document_ids=clean_docs,
|
||||
attachment_ids=clean_atts,
|
||||
note=clean_note,
|
||||
phase="已加入扫描队列…",
|
||||
summary={
|
||||
**scope_stats,
|
||||
"step_index": 1,
|
||||
"step_total": 4,
|
||||
"progress_pct": 8,
|
||||
},
|
||||
started_by=actor,
|
||||
)
|
||||
session.add(job)
|
||||
await session.commit()
|
||||
await session.refresh(job)
|
||||
|
||||
task = asyncio.create_task(_execute_job(job.id))
|
||||
_ACTIVE_TASKS[job.id] = task
|
||||
task.add_done_callback(lambda _t, jid=job.id: _ACTIVE_TASKS.pop(jid, None))
|
||||
return job
|
||||
|
||||
|
||||
async def _flush_job_progress(job_id: str, *, phase: str, summary: Mapping[str, Any]) -> None:
|
||||
async with SessionLocal() as session:
|
||||
await session.execute(
|
||||
update(EntityDiscoveryJob)
|
||||
.where(EntityDiscoveryJob.id == job_id, EntityDiscoveryJob.status.in_(OPEN_STATUSES))
|
||||
.values(phase=phase, summary=dict(summary))
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def _execute_job(job_id: str) -> None:
|
||||
from langchain_core.messages import HumanMessage, SystemMessage
|
||||
|
||||
from app.engine.state import message_text
|
||||
|
||||
try:
|
||||
async with SessionLocal() as session:
|
||||
job = await session.get(EntityDiscoveryJob, job_id)
|
||||
if job is None or job.status not in OPEN_STATUSES:
|
||||
return
|
||||
job.status = "running"
|
||||
job.started_at = utcnow()
|
||||
base_summary = dict(job.summary or {})
|
||||
base_summary.update({"step_index": 1, "step_total": 4, "progress_pct": 18})
|
||||
job.summary = base_summary
|
||||
job.phase = "步骤 1/4:正在读取数据源表目录与口径参数…"
|
||||
await session.commit()
|
||||
|
||||
prompt_body, scope_stats = await _build_discovery_prompt_body(
|
||||
session,
|
||||
source_ids=job.source_ids or [],
|
||||
document_ids=job.document_ids or [],
|
||||
attachment_ids=job.attachment_ids or [],
|
||||
note=job.note or "",
|
||||
)
|
||||
model, model_id = await catalog_assistant.resolve_model(session)
|
||||
doc_note = (
|
||||
f"、{scope_stats['documents_scanned']} 份文档({scope_stats['chunks_scanned']} 段)"
|
||||
if scope_stats["documents_scanned"] > 0
|
||||
else ""
|
||||
)
|
||||
live_summary: dict[str, Any] = {
|
||||
**scope_stats,
|
||||
"model_id": model_id,
|
||||
"step_index": 3,
|
||||
"step_total": 4,
|
||||
"progress_pct": 38,
|
||||
"model_chars": 0,
|
||||
"ops_detected": 0,
|
||||
"candidate_keys": [],
|
||||
}
|
||||
job.summary = live_summary
|
||||
job.phase = (
|
||||
f"步骤 3/4:已汇总 {scope_stats['sources_scanned']} 个源({scope_stats['tables_scanned']} 张表){doc_note},"
|
||||
"正在调用模型识别候选实体…"
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
msgs = [
|
||||
SystemMessage(content=DISCOVERY_SYSTEM_PROMPT),
|
||||
HumanMessage(content=DISCOVERY_USER_PROMPT.format(body=prompt_body)),
|
||||
]
|
||||
|
||||
first_token_seen = False
|
||||
call_started = time.monotonic()
|
||||
|
||||
async def _thinking_heartbeat() -> None:
|
||||
nonlocal live_summary
|
||||
while not first_token_seen:
|
||||
await asyncio.sleep(2.0)
|
||||
if first_token_seen:
|
||||
break
|
||||
waited = int(time.monotonic() - call_started)
|
||||
pct = min(52, 38 + waited // 2)
|
||||
live_summary = {
|
||||
**live_summary,
|
||||
"step_index": 3,
|
||||
"step_total": 4,
|
||||
"progress_pct": pct,
|
||||
}
|
||||
await _flush_job_progress(
|
||||
job_id,
|
||||
phase="步骤 3/4:模型正在深度分析表目录与业务关系,等待生成结果…",
|
||||
summary=live_summary,
|
||||
)
|
||||
|
||||
hb_task = asyncio.create_task(_thinking_heartbeat())
|
||||
try:
|
||||
if hasattr(model, "astream") and not getattr(model, "invoke_reply", ""):
|
||||
pieces: list[str] = []
|
||||
last_flush = time.monotonic()
|
||||
async for chunk in model.astream(msgs):
|
||||
piece = message_text(chunk)
|
||||
if piece:
|
||||
first_token_seen = True
|
||||
pieces.append(piece)
|
||||
now_mono = time.monotonic()
|
||||
if now_mono - last_flush >= 0.8 and pieces:
|
||||
last_flush = now_mono
|
||||
so_far = "".join(pieces)
|
||||
op_count = len(re.findall(r'"op"\s*:', so_far))
|
||||
found_keys = list(dict.fromkeys(re.findall(r'"entity_key"\s*:\s*"([a-z][a-z0-9_]{0,63})"', so_far)))
|
||||
pct = min(86, 53 + min(33, len(so_far) // 80))
|
||||
live_summary = {
|
||||
**live_summary,
|
||||
"step_index": 3,
|
||||
"step_total": 4,
|
||||
"progress_pct": pct,
|
||||
"model_chars": len(so_far),
|
||||
"ops_detected": op_count,
|
||||
"candidate_keys": found_keys[:15],
|
||||
}
|
||||
phase_txt = (
|
||||
f"步骤 3/4:模型正在识别候选实体(已识别 {op_count} 项操作,输出 {len(so_far)} 字)…"
|
||||
if op_count > 0
|
||||
else f"步骤 3/4:模型正在流式输出候选结果(已输出 {len(so_far)} 字)…"
|
||||
)
|
||||
await _flush_job_progress(job_id, phase=phase_txt, summary=live_summary)
|
||||
raw_text = "".join(pieces)
|
||||
else:
|
||||
reply = await model.ainvoke(msgs)
|
||||
first_token_seen = True
|
||||
raw_text = message_text(reply)
|
||||
finally:
|
||||
first_token_seen = True
|
||||
hb_task.cancel()
|
||||
with contextlib.suppress(asyncio.CancelledError, Exception):
|
||||
await hb_task
|
||||
|
||||
ops = parse_discovery_operations(raw_text)
|
||||
found_keys_final = list(
|
||||
dict.fromkeys(
|
||||
str(op.get("entity_key") or "").strip().lower()
|
||||
for op in ops
|
||||
if isinstance(op, Mapping) and str(op.get("entity_key") or "").strip()
|
||||
)
|
||||
)
|
||||
|
||||
async with SessionLocal() as session:
|
||||
job = await session.get(EntityDiscoveryJob, job_id)
|
||||
if job is None or job.status == "cancelled":
|
||||
return
|
||||
job.phase = f"步骤 4/4:正在核验 {len(ops)} 项候选的表列真实性并写入待确认列表…"
|
||||
job.summary = {
|
||||
**live_summary,
|
||||
"step_index": 4,
|
||||
"step_total": 4,
|
||||
"progress_pct": 90,
|
||||
"model_chars": len(raw_text),
|
||||
"ops_detected": len(ops),
|
||||
"candidate_keys": found_keys_final[:15],
|
||||
}
|
||||
await session.commit()
|
||||
|
||||
if ops:
|
||||
applied = await entity_patch.apply_entity_patch(
|
||||
session,
|
||||
ops,
|
||||
mode="proposed",
|
||||
actor=job.started_by,
|
||||
)
|
||||
else:
|
||||
applied = {
|
||||
"applied_count": 0,
|
||||
"created_entities": 0,
|
||||
"updated_entities": 0,
|
||||
"bindings_changed": 0,
|
||||
"caliber_params_changed": 0,
|
||||
"members_changed": 0,
|
||||
"entities": [],
|
||||
"problems": [],
|
||||
}
|
||||
touched_entities = [
|
||||
{"id": e["id"], "key": e["key"], "label": e["label"]}
|
||||
for e in (applied.get("entities") or [])[:30]
|
||||
if isinstance(e, Mapping) and e.get("id") and e.get("key")
|
||||
]
|
||||
problems = [str(p) for p in (applied.get("problems") or [])]
|
||||
job.status = "succeeded"
|
||||
job.phase = "扫描完成"
|
||||
job.finished_at = utcnow()
|
||||
job.summary = {
|
||||
**scope_stats,
|
||||
"model_id": model_id,
|
||||
"step_index": 4,
|
||||
"step_total": 4,
|
||||
"progress_pct": 100,
|
||||
"model_chars": len(raw_text),
|
||||
"ops_detected": len(ops),
|
||||
"candidate_keys": found_keys_final[:15],
|
||||
"applied_count": applied["applied_count"],
|
||||
"created_entities": applied["created_entities"],
|
||||
"updated_entities": applied["updated_entities"],
|
||||
"proposed_bindings": applied["bindings_changed"],
|
||||
"proposed_params": applied["caliber_params_changed"],
|
||||
"added_members": applied["members_changed"],
|
||||
"skipped_count": len(problems),
|
||||
"skipped_reasons": problems[:15],
|
||||
"touched_entities": touched_entities,
|
||||
}
|
||||
await session.commit()
|
||||
except asyncio.CancelledError:
|
||||
async with SessionLocal() as session:
|
||||
job = await session.get(EntityDiscoveryJob, job_id)
|
||||
if job is not None and job.status in OPEN_STATUSES:
|
||||
job.status = "cancelled"
|
||||
job.phase = "已停止扫描"
|
||||
job.finished_at = utcnow()
|
||||
await session.commit()
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.exception("候选实体扫描任务 %s 失败", job_id)
|
||||
async with SessionLocal() as session:
|
||||
job = await session.get(EntityDiscoveryJob, job_id)
|
||||
if job is not None and job.status in OPEN_STATUSES:
|
||||
job.status = "failed"
|
||||
job.phase = "扫描失败"
|
||||
job.error = JOB_FAILED_TEXT.format(reason=first_line(exc) or describe_exception(exc))
|
||||
job.finished_at = utcnow()
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def cancel_job(session: AsyncSession, job_id: str) -> EntityDiscoveryJob:
|
||||
job = await session.get(EntityDiscoveryJob, str(job_id or "").strip())
|
||||
if job is None:
|
||||
raise LookupError("候选实体扫描任务不存在")
|
||||
if job.status not in OPEN_STATUSES:
|
||||
raise DiscoveryFinished("该候选实体扫描任务已经结束")
|
||||
task = _ACTIVE_TASKS.get(job.id)
|
||||
if task is not None and not task.done():
|
||||
task.cancel()
|
||||
with contextlib.suppress(asyncio.CancelledError, Exception):
|
||||
await asyncio.wait_for(asyncio.shield(task), timeout=2.0)
|
||||
await session.refresh(job)
|
||||
if job.status in OPEN_STATUSES:
|
||||
job.status = "cancelled"
|
||||
job.phase = "已停止扫描"
|
||||
job.finished_at = utcnow()
|
||||
await session.commit()
|
||||
await session.refresh(job)
|
||||
return job
|
||||
|
||||
|
||||
async def startup(*, live: Any = None) -> int:
|
||||
"""服务启动或清理孤儿任务时,将上次残留的未结束扫描任务标为 interrupted。"""
|
||||
_ = live
|
||||
running_ids = set(_ACTIVE_TASKS.keys())
|
||||
async with SessionLocal() as session:
|
||||
rows = list(
|
||||
(
|
||||
await session.execute(
|
||||
select(EntityDiscoveryJob.id).where(EntityDiscoveryJob.status.in_(OPEN_STATUSES))
|
||||
)
|
||||
).scalars()
|
||||
)
|
||||
stale_ids = [jid for jid in rows if jid not in running_ids]
|
||||
if stale_ids:
|
||||
await session.execute(
|
||||
update(EntityDiscoveryJob)
|
||||
.where(EntityDiscoveryJob.id.in_(stale_ids), EntityDiscoveryJob.status.in_(OPEN_STATUSES))
|
||||
.values(
|
||||
status="interrupted",
|
||||
phase="扫描中断",
|
||||
error=INTERRUPTED_ERROR,
|
||||
finished_at=utcnow(),
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
return len(stale_ids)
|
||||
|
||||
|
||||
async def shutdown() -> None:
|
||||
"""服务关停时取消进程内正在执行的扫描任务并标为 interrupted。"""
|
||||
tasks = list(_ACTIVE_TASKS.values())
|
||||
for task in tasks:
|
||||
if not task.done():
|
||||
task.cancel()
|
||||
for task in tasks:
|
||||
with contextlib.suppress(asyncio.CancelledError, Exception):
|
||||
await task
|
||||
await startup()
|
||||
@@ -0,0 +1,334 @@
|
||||
"""标准业务实体维护的业务文档与主数据对照表附件服务(E1)。
|
||||
|
||||
支持两类文档来源:
|
||||
1. 临时上传附件(POST /api/ontology/entities/attachments):复用 memory/parsing.extract(..., tables=True)
|
||||
提取 .md / .txt / .docx / .pdf / .pptx / .html 以及 .xlsx / .csv / .tsv 编码对照表正文,
|
||||
按 kb.chunk_text 切块(跨块自动保留 Markdown 表头)并存入内容寻址工件(entity_doc_attachment);
|
||||
若勾选 save_to_kb=True,同步创建知识库 Document 并在后台完成切块向量索引,返回 document_id 供后续直接复用。
|
||||
2. 已有知识库文档(Document + Chunk):若文档仍在后台切块中,自动回退按 doc.content 即时切块,无需等待向量计算。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from fastapi import BackgroundTasks, UploadFile
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core import artifact_store
|
||||
from app.core.config import settings
|
||||
from app.core.errors import explain, raw as raw_err
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import Chunk, Document
|
||||
from app.memory import kb
|
||||
from app.memory.parsing import UnsupportedDocument, extract
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
ATTACHMENT_KIND = "entity_doc_attachment"
|
||||
MAX_DOC_SEARCH_LIMIT = 15
|
||||
|
||||
#: 持有后台切块任务引用,防止事件循环提前回收
|
||||
_KB_TASKS: set[asyncio.Task[Any]] = set()
|
||||
|
||||
_TOKEN_RE = re.compile(r"[A-Za-z0-9_]+|[\u4e00-\u9fff]{2,}")
|
||||
|
||||
|
||||
class AttachmentTooLarge(ValueError):
|
||||
"""上传附件超过系统大小上限。"""
|
||||
|
||||
|
||||
class AttachmentInvalid(ValueError):
|
||||
"""上传附件格式不支持或无有效文本。"""
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class EntityDocBundle:
|
||||
ref_id: str
|
||||
kind: str # "attachment" | "kb"
|
||||
title: str
|
||||
mime: str
|
||||
char_count: int
|
||||
chunks: list[str]
|
||||
document_id: str | None = None
|
||||
|
||||
|
||||
async def _index_kb_document(doc_id: str) -> None:
|
||||
"""后台对新建的知识库文档切块并建立向量与倒排索引。"""
|
||||
async with SessionLocal() as session:
|
||||
doc = await session.get(Document, doc_id)
|
||||
if not doc:
|
||||
return
|
||||
try:
|
||||
await kb.process_document(session, doc)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("实体文档存入知识库索引失败(%s):%s", doc_id, raw_err(exc))
|
||||
await session.rollback()
|
||||
doc = await session.get(Document, doc_id)
|
||||
if doc:
|
||||
reason, hint = explain(exc)
|
||||
doc.status = "failed"
|
||||
doc.error = (f"文档分块或生成向量时出错:{reason}" + (f"。{hint}" if hint else ""))[:500]
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def read_upload_limited(file: UploadFile) -> bytes:
|
||||
limit = settings.max_upload_mb * 1024 * 1024
|
||||
pieces: list[bytes] = []
|
||||
total = 0
|
||||
while chunk := await file.read(1024 * 1024):
|
||||
total += len(chunk)
|
||||
if total > limit:
|
||||
raise AttachmentTooLarge(f"文件超过大小上限({settings.max_upload_mb} MB)")
|
||||
pieces.append(chunk)
|
||||
return b"".join(pieces)
|
||||
|
||||
|
||||
async def ingest_entity_attachment(
|
||||
session: AsyncSession,
|
||||
*,
|
||||
raw_bytes: bytes,
|
||||
filename: str,
|
||||
mime: str = "",
|
||||
save_to_kb: bool = False,
|
||||
collection: str = "default",
|
||||
background: BackgroundTasks | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""解析上传的业务文档或主数据表格,生成附件工件并按需存入知识库。"""
|
||||
clean_name = (filename or "上传文档").strip() or "上传文档"
|
||||
clean_mime = (mime or "text/plain").strip() or "text/plain"
|
||||
if not raw_bytes:
|
||||
raise AttachmentInvalid("上传文件为空,请选择包含有效内容的文件")
|
||||
try:
|
||||
content = extract(raw_bytes, clean_name, clean_mime, tables=True)
|
||||
except UnsupportedDocument as exc:
|
||||
raise AttachmentInvalid(str(exc)) from None
|
||||
|
||||
content = (content or "").strip()
|
||||
if not content:
|
||||
raise AttachmentInvalid("未能从文件中提取出有效文字或表格内容")
|
||||
|
||||
chunks = kb.chunk_text(content)
|
||||
if not chunks:
|
||||
chunks = [content[:4000]]
|
||||
|
||||
doc_id: str | None = None
|
||||
clean_collection = (collection or "default").strip() or "default"
|
||||
if save_to_kb:
|
||||
doc = await kb.create_document(
|
||||
session,
|
||||
collection=clean_collection,
|
||||
title=clean_name,
|
||||
content=content,
|
||||
source=clean_name,
|
||||
mime=clean_mime,
|
||||
meta={"origin": "entity_agent"},
|
||||
status="processing",
|
||||
)
|
||||
doc_id = doc.id
|
||||
if background is not None:
|
||||
background.add_task(_index_kb_document, doc.id)
|
||||
else:
|
||||
task = asyncio.create_task(_index_kb_document(doc.id))
|
||||
_KB_TASKS.add(task)
|
||||
task.add_done_callback(_KB_TASKS.discard)
|
||||
|
||||
payload = {
|
||||
"title": clean_name,
|
||||
"filename": clean_name,
|
||||
"mime": clean_mime,
|
||||
"char_count": len(content),
|
||||
"chunks": chunks,
|
||||
"document_id": doc_id,
|
||||
}
|
||||
attachment_id = await artifact_store.put_json(
|
||||
payload,
|
||||
kind=ATTACHMENT_KIND,
|
||||
meta={"title": clean_name, "document_id": doc_id},
|
||||
)
|
||||
return {
|
||||
"attachment_id": attachment_id,
|
||||
"title": clean_name,
|
||||
"mime": clean_mime,
|
||||
"char_count": len(content),
|
||||
"chunk_count": len(chunks),
|
||||
"saved_to_kb": bool(doc_id),
|
||||
"document_id": doc_id,
|
||||
"collection": clean_collection if doc_id else None,
|
||||
}
|
||||
|
||||
|
||||
async def load_doc_bundles(
|
||||
session: AsyncSession,
|
||||
*,
|
||||
attachment_ids: Sequence[str] = (),
|
||||
document_ids: Sequence[str] = (),
|
||||
) -> list[EntityDocBundle]:
|
||||
"""统一加载临时附件与知识库文档的切块,按引用去重。"""
|
||||
bundles: list[EntityDocBundle] = []
|
||||
seen_doc_ids: set[str] = set()
|
||||
seen_att_ids: set[str] = set()
|
||||
|
||||
for raw_att in attachment_ids or ():
|
||||
att_id = str(raw_att or "").strip()
|
||||
if not att_id or att_id in seen_att_ids:
|
||||
continue
|
||||
seen_att_ids.add(att_id)
|
||||
try:
|
||||
loaded = artifact_store.load(att_id)
|
||||
except ValueError:
|
||||
loaded = None
|
||||
if not isinstance(loaded, dict) or not isinstance(loaded.get("chunks"), list):
|
||||
continue
|
||||
chunks = [str(c) for c in loaded["chunks"] if isinstance(c, str) and c.strip()]
|
||||
if not chunks:
|
||||
continue
|
||||
linked_doc_id = str(loaded.get("document_id") or "").strip() or None
|
||||
if linked_doc_id:
|
||||
seen_doc_ids.add(linked_doc_id)
|
||||
bundles.append(
|
||||
EntityDocBundle(
|
||||
ref_id=att_id,
|
||||
kind="attachment",
|
||||
title=str(loaded.get("title") or "上传附件"),
|
||||
mime=str(loaded.get("mime") or "text/plain"),
|
||||
char_count=int(loaded.get("char_count") or sum(len(c) for c in chunks)),
|
||||
chunks=chunks,
|
||||
document_id=linked_doc_id,
|
||||
)
|
||||
)
|
||||
|
||||
clean_doc_ids = [
|
||||
str(d).strip()
|
||||
for d in (document_ids or ())
|
||||
if str(d or "").strip() and str(d).strip() not in seen_doc_ids
|
||||
]
|
||||
if clean_doc_ids:
|
||||
seen_doc_ids.update(clean_doc_ids)
|
||||
docs = {
|
||||
row.id: row
|
||||
for row in (await session.execute(select(Document).where(Document.id.in_(clean_doc_ids)))).scalars()
|
||||
}
|
||||
chunk_rows = list(
|
||||
(
|
||||
await session.execute(
|
||||
select(Chunk)
|
||||
.where(Chunk.document_id.in_(clean_doc_ids))
|
||||
.order_by(Chunk.document_id.asc(), Chunk.ordinal.asc())
|
||||
)
|
||||
).scalars()
|
||||
)
|
||||
by_doc: dict[str, list[str]] = {}
|
||||
for crow in chunk_rows:
|
||||
if crow.content and crow.content.strip():
|
||||
by_doc.setdefault(crow.document_id, []).append(crow.content)
|
||||
|
||||
for doc_id in clean_doc_ids:
|
||||
doc = docs.get(doc_id)
|
||||
if doc is None:
|
||||
continue
|
||||
pieces = by_doc.get(doc_id) or kb.chunk_text(doc.content or "")
|
||||
if not pieces and (doc.content or "").strip():
|
||||
pieces = [(doc.content or "").strip()[:4000]]
|
||||
if not pieces:
|
||||
continue
|
||||
bundles.append(
|
||||
EntityDocBundle(
|
||||
ref_id=doc.id,
|
||||
kind="kb",
|
||||
title=doc.title or doc.source or "知识库文档",
|
||||
mime=doc.mime or "text/plain",
|
||||
char_count=len(doc.content or ""),
|
||||
chunks=pieces,
|
||||
document_id=doc.id,
|
||||
)
|
||||
)
|
||||
return bundles
|
||||
|
||||
|
||||
def _keywords(query: str) -> list[str]:
|
||||
q = (query or "").strip().lower()
|
||||
if not q:
|
||||
return []
|
||||
tokens = [m.group(0) for m in _TOKEN_RE.finditer(q)]
|
||||
out: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for tok in [q, *tokens]:
|
||||
if tok and tok not in seen:
|
||||
seen.add(tok)
|
||||
out.append(tok)
|
||||
return out
|
||||
|
||||
|
||||
def search_doc_bundles(
|
||||
bundles: Sequence[EntityDocBundle],
|
||||
*,
|
||||
query: str = "",
|
||||
doc_ref: str = "",
|
||||
start_chunk: int = 0,
|
||||
limit: int = 6,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""按关键词或分段序号检索已挂载文档片段,返回带文档标题与段号的原文片段。"""
|
||||
cap = max(1, min(int(limit or 6), MAX_DOC_SEARCH_LIMIT))
|
||||
offset = max(0, int(start_chunk or 0))
|
||||
ref_norm = (doc_ref or "").strip().lower()
|
||||
|
||||
scoped = [
|
||||
b
|
||||
for b in bundles
|
||||
if not ref_norm
|
||||
or b.ref_id.lower() == ref_norm
|
||||
or (b.document_id and b.document_id.lower() == ref_norm)
|
||||
or ref_norm in b.title.lower()
|
||||
]
|
||||
if not scoped:
|
||||
scoped = list(bundles)
|
||||
|
||||
terms = _keywords(query)
|
||||
if not terms:
|
||||
out: list[dict[str, Any]] = []
|
||||
for bundle in scoped:
|
||||
for idx in range(offset, len(bundle.chunks)):
|
||||
out.append({
|
||||
"doc_ref": bundle.ref_id,
|
||||
"title": bundle.title,
|
||||
"chunk_index": idx,
|
||||
"total_chunks": len(bundle.chunks),
|
||||
"citation": f"文档《{bundle.title}》第 {idx + 1} 段",
|
||||
"content": bundle.chunks[idx],
|
||||
})
|
||||
if len(out) >= cap:
|
||||
return out
|
||||
return out
|
||||
|
||||
scored: list[tuple[int, int, int, EntityDocBundle, str]] = []
|
||||
for b_idx, bundle in enumerate(scoped):
|
||||
for c_idx, chunk in enumerate(bundle.chunks):
|
||||
if c_idx < offset:
|
||||
continue
|
||||
low = chunk.lower()
|
||||
score = 0
|
||||
for t_idx, term in enumerate(terms):
|
||||
hits = low.count(term)
|
||||
if hits:
|
||||
weight = 5 if t_idx == 0 else (3 if len(term) >= 3 else 1)
|
||||
score += hits * weight
|
||||
if score > 0:
|
||||
scored.append((-score, b_idx, c_idx, bundle, chunk))
|
||||
scored.sort(key=lambda item: (item[0], item[1], item[2]))
|
||||
return [
|
||||
{
|
||||
"doc_ref": bundle.ref_id,
|
||||
"title": bundle.title,
|
||||
"chunk_index": c_idx,
|
||||
"total_chunks": len(bundle.chunks),
|
||||
"citation": f"文档《{bundle.title}》第 {c_idx + 1} 段",
|
||||
"content": chunk,
|
||||
}
|
||||
for _neg, _b, c_idx, bundle, chunk in scored[:cap]
|
||||
]
|
||||
@@ -0,0 +1,841 @@
|
||||
"""标准业务实体的受控增量补丁规划器与落库器(E1)。
|
||||
|
||||
供交互式「实体助手」(Entity Copilot)与后台「扫描发现候选实体」共用同一套核验与合并内核:
|
||||
1. 物理真实性硬校验:表列绑定必须在已启用数据源的 schema_cache 中真实存在;口径参数映射必须在对应工作流入口节点中真实存在。
|
||||
2. 敏感遮罩隔离:遮罩列(masked_columns)严禁写入或更新 members.source_codes 枚举字典。
|
||||
3. 剥离自封审批态:模型传入的 status / approved / verified / actor / version 一律忽略重算。
|
||||
4. 双模式落库:
|
||||
- mode="confirmed"(人工在实体助手点「保存修改」):严格校验 expected_versions CAS 乐观锁,绑定与参数落为 confirmed / human。
|
||||
- mode="proposed"(后台主动发现):新增绑定与参数一律落为 proposed / ai,绝不覆盖人工已确认、已验证或已驳回的项。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Literal
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.artifact_store import canonical_json, content_hash
|
||||
from app.data import entities, introspect
|
||||
from app.data.engine import masked_columns
|
||||
from app.db.base import new_id, utcnow
|
||||
from app.db.models import DataSource, Workflow, WorkflowVersion
|
||||
from app.db.models_ontology import CanonicalEntity
|
||||
|
||||
OP_TYPES = (
|
||||
"upsert_entity",
|
||||
"upsert_binding",
|
||||
"remove_binding",
|
||||
"upsert_member",
|
||||
"upsert_caliber_param",
|
||||
)
|
||||
|
||||
_FORBIDDEN_KEYS = frozenset({
|
||||
"status", "approve", "approved", "approval", "approver", "verified", "verify",
|
||||
"actor", "role", "review", "reviewed", "reviewed_by", "updated_by", "created_by", "version",
|
||||
})
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class EntityPatchChange:
|
||||
id: str
|
||||
op: str
|
||||
entity_key: str
|
||||
entity_label: str
|
||||
summary: str
|
||||
before: Any
|
||||
after: Any
|
||||
reason: str
|
||||
op_payload: dict[str, Any]
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"id": self.id,
|
||||
"op": self.op,
|
||||
"entity_key": self.entity_key,
|
||||
"entity_label": self.entity_label,
|
||||
"summary": self.summary,
|
||||
"before": self.before,
|
||||
"after": self.after,
|
||||
"reason": self.reason,
|
||||
"op_payload": self.op_payload,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class EntityPatchPlan:
|
||||
changes: list[EntityPatchChange] = field(default_factory=list)
|
||||
problems: list[str] = field(default_factory=list)
|
||||
expected_versions: dict[str, int] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _ValidationEnv:
|
||||
sources_by_norm: dict[str, DataSource]
|
||||
masked_by_source: dict[str, set[str]]
|
||||
workflows_by_id: dict[str, Workflow]
|
||||
workflows_by_name: dict[str, Workflow]
|
||||
workflow_inputs: dict[str, set[str]]
|
||||
workflow_cards: dict[str, set[str]]
|
||||
entities_by_key: dict[str, CanonicalEntity]
|
||||
|
||||
|
||||
def _strip_forbidden(raw: Mapping[str, Any]) -> dict[str, Any]:
|
||||
return {k: v for k, v in raw.items() if k not in _FORBIDDEN_KEYS}
|
||||
|
||||
|
||||
def _extract_graph_inputs_and_cards(graph: Mapping[str, Any] | None) -> tuple[set[str], set[str]]:
|
||||
inputs: set[str] = set()
|
||||
cards: set[str] = set()
|
||||
if not isinstance(graph, Mapping):
|
||||
return inputs, cards
|
||||
for node in graph.get("nodes") or []:
|
||||
if not isinstance(node, Mapping):
|
||||
continue
|
||||
ntype = str(node.get("type") or "").strip()
|
||||
nid = str(node.get("id") or "").strip()
|
||||
if ntype == "metrics" and nid:
|
||||
cards.add(nid)
|
||||
if ntype == "input":
|
||||
data = node.get("data") if isinstance(node.get("data"), Mapping) else {}
|
||||
cfg = data.get("config") if isinstance(data.get("config"), Mapping) else (
|
||||
node.get("config") if isinstance(node.get("config"), Mapping) else {}
|
||||
)
|
||||
for f in cfg.get("fields") or []:
|
||||
if isinstance(f, Mapping):
|
||||
fname = str(f.get("name") or "").strip()
|
||||
if fname:
|
||||
inputs.add(fname)
|
||||
return inputs, cards
|
||||
|
||||
|
||||
async def _load_env(session: AsyncSession) -> _ValidationEnv:
|
||||
from app.data import table_versions
|
||||
|
||||
src_rows = list(
|
||||
(await session.execute(select(DataSource).where(DataSource.enabled.is_(True)).order_by(DataSource.name))).scalars()
|
||||
)
|
||||
sources_by_norm: dict[str, Any] = {}
|
||||
masked_by_source: dict[str, set[str]] = {}
|
||||
for row in src_rows:
|
||||
try:
|
||||
resolved = await table_versions.resolve_source(session, row)
|
||||
except table_versions.SnapshotMissing:
|
||||
resolved = row
|
||||
key = row.name.strip().lower()
|
||||
sources_by_norm[key] = resolved
|
||||
sources_by_norm[row.id.strip().lower()] = resolved
|
||||
masked_by_source[row.name.strip().lower()] = {
|
||||
c.strip().lower() for c in masked_columns(row.options if isinstance(row.options, dict) else None)
|
||||
}
|
||||
|
||||
wf_rows = list((await session.execute(select(Workflow))).scalars())
|
||||
ver_rows = list(
|
||||
(
|
||||
await session.execute(
|
||||
select(WorkflowVersion).where(WorkflowVersion.published_at.is_not(None))
|
||||
)
|
||||
).scalars()
|
||||
)
|
||||
workflows_by_id: dict[str, Workflow] = {}
|
||||
name_counts: dict[str, int] = {}
|
||||
for wf in wf_rows:
|
||||
workflows_by_id[wf.id] = wf
|
||||
if wf.name:
|
||||
nkey = wf.name.strip().lower()
|
||||
name_counts[nkey] = name_counts.get(nkey, 0) + 1
|
||||
workflows_by_name = {
|
||||
wf.name.strip().lower(): wf
|
||||
for wf in wf_rows
|
||||
if wf.name and name_counts.get(wf.name.strip().lower()) == 1
|
||||
}
|
||||
|
||||
workflow_inputs: dict[str, set[str]] = {}
|
||||
workflow_cards: dict[str, set[str]] = {}
|
||||
for wf in wf_rows:
|
||||
ins, cds = _extract_graph_inputs_and_cards(wf.graph if isinstance(wf.graph, dict) else None)
|
||||
workflow_inputs.setdefault(wf.id, set()).update(ins)
|
||||
workflow_cards.setdefault(wf.id, set()).update(cds)
|
||||
for ver in ver_rows:
|
||||
ins, cds = _extract_graph_inputs_and_cards(ver.graph if isinstance(ver.graph, dict) else None)
|
||||
workflow_inputs.setdefault(ver.workflow_id, set()).update(ins)
|
||||
workflow_cards.setdefault(ver.workflow_id, set()).update(cds)
|
||||
|
||||
ent_rows = await entities.list_entities(session)
|
||||
entities_by_key = {row.key.strip().lower(): row for row in ent_rows}
|
||||
|
||||
return _ValidationEnv(
|
||||
sources_by_norm=sources_by_norm,
|
||||
masked_by_source=masked_by_source,
|
||||
workflows_by_id=workflows_by_id,
|
||||
workflows_by_name=workflows_by_name,
|
||||
workflow_inputs=workflow_inputs,
|
||||
workflow_cards=workflow_cards,
|
||||
entities_by_key=entities_by_key,
|
||||
)
|
||||
|
||||
|
||||
def _is_column_masked(masked_set: set[str], table_name: str, column_name: str) -> bool:
|
||||
col_low = column_name.strip().lower()
|
||||
tbl_low = table_name.strip().lower()
|
||||
short_tbl = tbl_low.rsplit(".", 1)[-1]
|
||||
return (
|
||||
col_low in masked_set
|
||||
or f"{tbl_low}.{col_low}" in masked_set
|
||||
or f"{short_tbl}.{col_low}" in masked_set
|
||||
)
|
||||
|
||||
|
||||
def _resolve_table_and_column(
|
||||
source: DataSource,
|
||||
raw_table: str,
|
||||
raw_column: str,
|
||||
) -> tuple[str | None, str | None, str | None]:
|
||||
"""返回 (规范表名, 规范列名, 错误说明)。"""
|
||||
cache = source.schema_cache if isinstance(source.schema_cache, Mapping) else {}
|
||||
tables = cache.get("tables") if isinstance(cache.get("tables"), Mapping) else {}
|
||||
if not tables:
|
||||
return None, None, f"数据源「{source.name}」尚未探查表结构,无法核实表「{raw_table}」与列「{raw_column}」"
|
||||
|
||||
t_probe = raw_table.strip()
|
||||
matched_table: str | None = None
|
||||
meta: Mapping[str, Any] | None = None
|
||||
if t_probe in tables:
|
||||
matched_table, meta = t_probe, tables[t_probe]
|
||||
else:
|
||||
t_low = t_probe.lower()
|
||||
t_short = t_low.rsplit(".", 1)[-1]
|
||||
for tname, tmeta in tables.items():
|
||||
if not isinstance(tmeta, Mapping):
|
||||
continue
|
||||
qual = str(tmeta.get("qualified") or tname).strip()
|
||||
if tname.lower() == t_low or qual.lower() == t_low or tname.lower() == t_short:
|
||||
matched_table, meta = tname, tmeta
|
||||
break
|
||||
if meta is None or matched_table is None:
|
||||
sample = "、".join(list(tables.keys())[:15])
|
||||
return None, None, f"数据源「{source.name}」中不存在表「{raw_table}」(已探查表如:{sample})"
|
||||
|
||||
cols = introspect.table_columns(dict(meta))
|
||||
c_low = raw_column.strip().lower()
|
||||
for col in cols:
|
||||
cname = str(col.get("name") or "").strip()
|
||||
if cname.lower() == c_low:
|
||||
return matched_table, cname, None
|
||||
col_sample = "、".join(str(c.get("name")) for c in cols[:20])
|
||||
return None, None, f"数据源「{source.name}」表「{matched_table}」中不存在列「{raw_column}」(可用列:{col_sample})"
|
||||
|
||||
|
||||
def _working_from_row(row: CanonicalEntity) -> dict[str, Any]:
|
||||
return {
|
||||
"id": row.id,
|
||||
"key": row.key,
|
||||
"label": row.label,
|
||||
"description": str(row.description or ""),
|
||||
"aliases": list(row.aliases or []),
|
||||
"members": [copy.deepcopy(dict(m)) for m in (row.members or []) if isinstance(m, Mapping)],
|
||||
"bindings": [copy.deepcopy(dict(b)) for b in (row.bindings or []) if isinstance(b, Mapping)],
|
||||
"caliber_params": [copy.deepcopy(dict(p)) for p in (row.caliber_params or []) if isinstance(p, Mapping)],
|
||||
"version": int(row.version or 1),
|
||||
"is_new": False,
|
||||
}
|
||||
|
||||
|
||||
def _change_id(op: str, entity_key: str, identity: str) -> str:
|
||||
return content_hash(canonical_json({"op": op, "entity_key": entity_key, "id": identity}))[:16]
|
||||
|
||||
|
||||
def _normalize_single_op(
|
||||
raw_op: Any,
|
||||
*,
|
||||
index: int,
|
||||
env: _ValidationEnv,
|
||||
working: dict[str, dict[str, Any]],
|
||||
mode: Literal["confirmed", "proposed"],
|
||||
) -> tuple[EntityPatchChange | None, str | None]:
|
||||
if not isinstance(raw_op, Mapping):
|
||||
return None, f"第 {index + 1} 项操作需要是对象"
|
||||
cleaned = _strip_forbidden(raw_op)
|
||||
op = str(cleaned.get("op") or "").strip()
|
||||
if op not in OP_TYPES:
|
||||
return None, f"第 {index + 1} 项操作类型「{op}」无效,支持:{', '.join(OP_TYPES)}"
|
||||
if mode == "proposed" and op == "remove_binding":
|
||||
return None, "后台发现模式不允许删除既有表列绑定(remove_binding)"
|
||||
|
||||
raw_key = cleaned.get("entity_key") or cleaned.get("key")
|
||||
try:
|
||||
entity_key = entities.normalize_entity_key(raw_key, field="entity_key")
|
||||
except entities.EntityInvalid as exc:
|
||||
return None, f"第 {index + 1} 项:{exc.message}"
|
||||
|
||||
reason = str(cleaned.get("reason") or "").strip()[:500]
|
||||
note = str(cleaned.get("note") or "").strip()[:500]
|
||||
|
||||
# 初始化当前实体工作副本
|
||||
state = working.get(entity_key)
|
||||
if state is None and entity_key in env.entities_by_key:
|
||||
state = _working_from_row(env.entities_by_key[entity_key])
|
||||
working[entity_key] = state
|
||||
|
||||
if op == "upsert_entity":
|
||||
raw_label = str(cleaned.get("label") or cleaned.get("entity_label") or "").strip()
|
||||
if state is None and not raw_label:
|
||||
return None, f"新建标准实体「{entity_key}」缺少中文名称(label)"
|
||||
label = raw_label or (state["label"] if state else entity_key)
|
||||
if len(label) > 100:
|
||||
return None, f"标准实体「{entity_key}」名称不能超过 100 个字符"
|
||||
desc = str(cleaned.get("description") if "description" in cleaned else (state["description"] if state else "") or "").strip()
|
||||
raw_aliases = cleaned.get("aliases")
|
||||
if raw_aliases is None:
|
||||
new_aliases = list(state["aliases"]) if state else []
|
||||
elif isinstance(raw_aliases, list):
|
||||
merged = list(state["aliases"]) if (state and mode == "proposed") else []
|
||||
seen_a = {a.lower() for a in merged}
|
||||
for item in raw_aliases:
|
||||
if isinstance(item, str) and item.strip() and item.strip().lower() not in seen_a:
|
||||
seen_a.add(item.strip().lower())
|
||||
merged.append(item.strip())
|
||||
new_aliases = merged
|
||||
else:
|
||||
return None, f"标准实体「{entity_key}」的 aliases 需要是字符串数组"
|
||||
|
||||
if state is None:
|
||||
working[entity_key] = {
|
||||
"id": None,
|
||||
"key": entity_key,
|
||||
"label": label,
|
||||
"description": desc,
|
||||
"aliases": new_aliases,
|
||||
"members": [],
|
||||
"bindings": [],
|
||||
"caliber_params": [],
|
||||
"version": 0,
|
||||
"is_new": True,
|
||||
}
|
||||
payload = {
|
||||
"op": "upsert_entity",
|
||||
"entity_key": entity_key,
|
||||
"label": label,
|
||||
"description": desc,
|
||||
"aliases": new_aliases,
|
||||
"reason": reason,
|
||||
}
|
||||
return EntityPatchChange(
|
||||
id=_change_id(op, entity_key, "meta"),
|
||||
op=op,
|
||||
entity_key=entity_key,
|
||||
entity_label=label,
|
||||
summary=f"新建标准业务实体「{label}({entity_key})」",
|
||||
before=None,
|
||||
after={"label": label, "description": desc, "aliases": new_aliases},
|
||||
reason=reason,
|
||||
op_payload=payload,
|
||||
), None
|
||||
|
||||
# 更新已有实体元信息(后台发现模式下不覆盖已有中文名与非空说明)
|
||||
target_label = state["label"] if mode == "proposed" else label
|
||||
target_desc = (state["description"] or desc) if mode == "proposed" else desc
|
||||
before_meta = {
|
||||
"label": state["label"],
|
||||
"description": state["description"],
|
||||
"aliases": list(state["aliases"]),
|
||||
}
|
||||
after_meta = {
|
||||
"label": target_label,
|
||||
"description": target_desc,
|
||||
"aliases": new_aliases,
|
||||
}
|
||||
if before_meta == after_meta:
|
||||
return None, None
|
||||
state["label"] = target_label
|
||||
state["description"] = target_desc
|
||||
state["aliases"] = new_aliases
|
||||
payload = {
|
||||
"op": "upsert_entity",
|
||||
"entity_key": entity_key,
|
||||
"label": target_label,
|
||||
"description": target_desc,
|
||||
"aliases": new_aliases,
|
||||
"reason": reason,
|
||||
}
|
||||
return EntityPatchChange(
|
||||
id=_change_id(op, entity_key, "meta"),
|
||||
op=op,
|
||||
entity_key=entity_key,
|
||||
entity_label=target_label,
|
||||
summary=f"更新标准实体「{target_label}({entity_key})」基础信息",
|
||||
before=before_meta,
|
||||
after=after_meta,
|
||||
reason=reason,
|
||||
op_payload=payload,
|
||||
), None
|
||||
|
||||
# 其余子项操作:若实体尚不存在,允许随操作附带 entity_label 自动初始化,否则明确报错
|
||||
if state is None:
|
||||
fallback_label = str(cleaned.get("entity_label") or "").strip()
|
||||
if not fallback_label:
|
||||
return (
|
||||
None,
|
||||
f"标准实体「{entity_key}」尚不存在,请先提供一条 upsert_entity(含 label 中文名称)或在操作中提供 entity_label",
|
||||
)
|
||||
state = {
|
||||
"id": None,
|
||||
"key": entity_key,
|
||||
"label": fallback_label[:100],
|
||||
"description": "",
|
||||
"aliases": [],
|
||||
"members": [],
|
||||
"bindings": [],
|
||||
"caliber_params": [],
|
||||
"version": 0,
|
||||
"is_new": True,
|
||||
}
|
||||
working[entity_key] = state
|
||||
|
||||
if op == "upsert_binding":
|
||||
raw_src = str(cleaned.get("source_name") or "").strip()
|
||||
raw_tbl = str(cleaned.get("table_name") or "").strip()
|
||||
raw_col = str(cleaned.get("column_name") or "").strip()
|
||||
if not raw_src or not raw_tbl or not raw_col:
|
||||
return None, f"实体「{entity_key}」的表列绑定须完整填写 source_name、table_name、column_name"
|
||||
src_row = env.sources_by_norm.get(raw_src.lower())
|
||||
if src_row is None:
|
||||
avail_src = "、".join(sorted({r.name for r in env.sources_by_norm.values()})) or "无"
|
||||
return None, f"未找到已启用的数据源「{raw_src}」(当前可用数据源:{avail_src})"
|
||||
canon_tbl, canon_col, problem = _resolve_table_and_column(src_row, raw_tbl, raw_col)
|
||||
if problem or canon_tbl is None or canon_col is None:
|
||||
return None, problem or "表列核验未通过"
|
||||
|
||||
target_status = "confirmed" if mode == "confirmed" else "proposed"
|
||||
target_source = "human" if mode == "confirmed" else "ai"
|
||||
bkey = (src_row.name.lower(), canon_tbl.lower(), canon_col.lower())
|
||||
existing_idx = next(
|
||||
(
|
||||
i
|
||||
for i, b in enumerate(state["bindings"])
|
||||
if (
|
||||
str(b.get("source_name") or "").lower(),
|
||||
str(b.get("table_name") or "").lower(),
|
||||
str(b.get("column_name") or "").lower(),
|
||||
)
|
||||
== bkey
|
||||
),
|
||||
None,
|
||||
)
|
||||
if existing_idx is not None:
|
||||
cur = state["bindings"][existing_idx]
|
||||
cur_status = str(cur.get("status") or "confirmed")
|
||||
if mode == "proposed" and cur_status in ("confirmed", "verified", "rejected"):
|
||||
return None, None
|
||||
if (
|
||||
cur_status == target_status
|
||||
and str(cur.get("note") or "") == (note or str(cur.get("note") or ""))
|
||||
and str(cur.get("reason") or "") == (reason or str(cur.get("reason") or ""))
|
||||
):
|
||||
return None, None
|
||||
before_val = copy.deepcopy(cur)
|
||||
updated = {
|
||||
**cur,
|
||||
"source_id": src_row.id,
|
||||
"source_name": src_row.name,
|
||||
"table_name": canon_tbl,
|
||||
"column_name": canon_col,
|
||||
"status": target_status,
|
||||
"source": target_source,
|
||||
"note": note or str(cur.get("note") or ""),
|
||||
"reason": reason or str(cur.get("reason") or ""),
|
||||
"updated_at": utcnow().isoformat(),
|
||||
}
|
||||
state["bindings"][existing_idx] = updated
|
||||
else:
|
||||
before_val = None
|
||||
updated = {
|
||||
"id": new_id()[:12],
|
||||
"source_id": src_row.id,
|
||||
"source_name": src_row.name,
|
||||
"table_name": canon_tbl,
|
||||
"column_name": canon_col,
|
||||
"status": target_status,
|
||||
"source": target_source,
|
||||
"note": note,
|
||||
"reason": reason,
|
||||
"updated_at": utcnow().isoformat(),
|
||||
}
|
||||
state["bindings"].append(updated)
|
||||
|
||||
payload = {
|
||||
"op": "upsert_binding",
|
||||
"entity_key": entity_key,
|
||||
"entity_label": state["label"],
|
||||
"source_name": src_row.name,
|
||||
"table_name": canon_tbl,
|
||||
"column_name": canon_col,
|
||||
"note": note,
|
||||
"reason": reason,
|
||||
}
|
||||
action_word = "新增" if before_val is None else "更新"
|
||||
return EntityPatchChange(
|
||||
id=_change_id(op, entity_key, f"{src_row.name}.{canon_tbl}.{canon_col}"),
|
||||
op=op,
|
||||
entity_key=entity_key,
|
||||
entity_label=state["label"],
|
||||
summary=f"{action_word}实体「{state['label']}」表列绑定:{src_row.name}.{canon_tbl}.{canon_col}",
|
||||
before=before_val,
|
||||
after=updated,
|
||||
reason=reason,
|
||||
op_payload=payload,
|
||||
), None
|
||||
|
||||
if op == "remove_binding":
|
||||
raw_src = str(cleaned.get("source_name") or "").strip()
|
||||
raw_tbl = str(cleaned.get("table_name") or "").strip()
|
||||
raw_col = str(cleaned.get("column_name") or "").strip()
|
||||
bkey = (raw_src.lower(), raw_tbl.lower(), raw_col.lower())
|
||||
existing_idx = next(
|
||||
(
|
||||
i
|
||||
for i, b in enumerate(state["bindings"])
|
||||
if (
|
||||
str(b.get("source_name") or "").lower(),
|
||||
str(b.get("table_name") or "").lower(),
|
||||
str(b.get("column_name") or "").lower(),
|
||||
)
|
||||
== bkey
|
||||
),
|
||||
None,
|
||||
)
|
||||
if existing_idx is None:
|
||||
return None, f"实体「{entity_key}」中不存在待移除的绑定「{raw_src}.{raw_tbl}.{raw_col}」"
|
||||
removed = state["bindings"].pop(existing_idx)
|
||||
payload = {
|
||||
"op": "remove_binding",
|
||||
"entity_key": entity_key,
|
||||
"source_name": removed.get("source_name"),
|
||||
"table_name": removed.get("table_name"),
|
||||
"column_name": removed.get("column_name"),
|
||||
"reason": reason,
|
||||
}
|
||||
return EntityPatchChange(
|
||||
id=_change_id(op, entity_key, f"{raw_src}.{raw_tbl}.{raw_col}"),
|
||||
op=op,
|
||||
entity_key=entity_key,
|
||||
entity_label=state["label"],
|
||||
summary=f"移除实体「{state['label']}」表列绑定:{removed.get('source_name')}.{removed.get('table_name')}.{removed.get('column_name')}",
|
||||
before=removed,
|
||||
after=None,
|
||||
reason=reason,
|
||||
op_payload=payload,
|
||||
), None
|
||||
|
||||
if op == "upsert_member":
|
||||
val = str(cleaned.get("value") or "").strip()
|
||||
if not val:
|
||||
return None, f"实体「{entity_key}」成员映射缺少标准值(value)"
|
||||
label = str(cleaned.get("label") or val).strip() or val
|
||||
raw_aliases = cleaned.get("aliases")
|
||||
if raw_aliases is not None and not isinstance(raw_aliases, list):
|
||||
return None, f"实体「{entity_key}」成员「{val}」的 aliases 需要是字符串数组"
|
||||
raw_codes = cleaned.get("source_codes")
|
||||
if raw_codes is not None and not isinstance(raw_codes, Mapping):
|
||||
return None, f"实体「{entity_key}」成员「{val}」的 source_codes 需要是「数据源名 → 编码」对象"
|
||||
|
||||
canon_codes: dict[str, str] = {}
|
||||
for src_k, code_v in (raw_codes or {}).items():
|
||||
s_str = str(src_k or "").strip()
|
||||
c_str = str(code_v or "").strip()
|
||||
if not s_str or not c_str:
|
||||
continue
|
||||
src_row = env.sources_by_norm.get(s_str.lower())
|
||||
canon_src = src_row.name if src_row is not None else s_str
|
||||
# 检查该实体在此数据源上的绑定是否涉及敏感遮罩列
|
||||
masked_set = env.masked_by_source.get(canon_src.lower(), set())
|
||||
if masked_set:
|
||||
for b in state["bindings"]:
|
||||
if str(b.get("source_name") or "").lower() == canon_src.lower():
|
||||
if _is_column_masked(masked_set, str(b.get("table_name") or ""), str(b.get("column_name") or "")):
|
||||
return (
|
||||
None,
|
||||
f"数据源「{canon_src}」表列「{b.get('table_name')}.{b.get('column_name')}」为敏感遮罩列,禁止为其配置成员编码映射",
|
||||
)
|
||||
canon_codes[canon_src] = c_str
|
||||
|
||||
m_idx = next(
|
||||
(i for i, m in enumerate(state["members"]) if str(m.get("value") or "").strip().lower() == val.lower()),
|
||||
None,
|
||||
)
|
||||
if m_idx is not None:
|
||||
cur_m = state["members"][m_idx]
|
||||
before_m = copy.deepcopy(cur_m)
|
||||
merged_aliases = list(cur_m.get("aliases") or [])
|
||||
seen_ma = {a.lower() for a in merged_aliases if isinstance(a, str)}
|
||||
for item in raw_aliases or []:
|
||||
if isinstance(item, str) and item.strip() and item.strip().lower() not in seen_ma:
|
||||
seen_ma.add(item.strip().lower())
|
||||
merged_aliases.append(item.strip())
|
||||
merged_codes = dict(cur_m.get("source_codes") or {})
|
||||
for cs, cv in canon_codes.items():
|
||||
if mode == "proposed" and cs in merged_codes and str(merged_codes[cs]).strip():
|
||||
continue
|
||||
merged_codes[cs] = cv
|
||||
target_mlabel = str(cur_m.get("label") or val) if mode == "proposed" else label
|
||||
after_m = {
|
||||
"value": str(cur_m.get("value") or val),
|
||||
"label": target_mlabel,
|
||||
"aliases": merged_aliases,
|
||||
"source_codes": merged_codes,
|
||||
}
|
||||
if before_m == after_m:
|
||||
return None, None
|
||||
state["members"][m_idx] = after_m
|
||||
else:
|
||||
before_m = None
|
||||
clean_aliases = []
|
||||
seen_ma = set()
|
||||
for item in raw_aliases or []:
|
||||
if isinstance(item, str) and item.strip() and item.strip().lower() not in seen_ma:
|
||||
seen_ma.add(item.strip().lower())
|
||||
clean_aliases.append(item.strip())
|
||||
after_m = {
|
||||
"value": val,
|
||||
"label": label,
|
||||
"aliases": clean_aliases,
|
||||
"source_codes": canon_codes,
|
||||
}
|
||||
state["members"].append(after_m)
|
||||
|
||||
payload = {
|
||||
"op": "upsert_member",
|
||||
"entity_key": entity_key,
|
||||
"entity_label": state["label"],
|
||||
"value": after_m["value"],
|
||||
"label": after_m["label"],
|
||||
"aliases": after_m["aliases"],
|
||||
"source_codes": after_m["source_codes"],
|
||||
"reason": reason,
|
||||
}
|
||||
code_desc = ",".join(f"{k}={v}" for k, v in after_m["source_codes"].items())
|
||||
suffix = f"({code_desc})" if code_desc else ""
|
||||
return EntityPatchChange(
|
||||
id=_change_id(op, entity_key, after_m["value"].lower()),
|
||||
op=op,
|
||||
entity_key=entity_key,
|
||||
entity_label=state["label"],
|
||||
summary=f"{'新增' if before_m is None else '更新'}实体「{state['label']}」成员「{after_m['label']}({after_m['value']})」{suffix}",
|
||||
before=before_m,
|
||||
after=after_m,
|
||||
reason=reason,
|
||||
op_payload=payload,
|
||||
), None
|
||||
|
||||
# upsert_caliber_param
|
||||
raw_wf = str(cleaned.get("workflow_id") or "").strip()
|
||||
raw_card = str(cleaned.get("card_id") or "").strip()
|
||||
raw_param = str(cleaned.get("param_name") or "").strip()
|
||||
if not raw_wf or not raw_param:
|
||||
return None, f"实体「{entity_key}」口径参数映射须提供 workflow_id 与 param_name"
|
||||
wf_row = env.workflows_by_id.get(raw_wf) or env.workflows_by_name.get(raw_wf.lower())
|
||||
if wf_row is None:
|
||||
return None, f"不存在工作流「{raw_wf}」,请核对已发布口径的工作流 ID"
|
||||
valid_inputs = env.workflow_inputs.get(wf_row.id, set())
|
||||
canon_param = next((p for p in valid_inputs if p.lower() == raw_param.lower()), None)
|
||||
if canon_param is None:
|
||||
sample_in = "、".join(sorted(valid_inputs)) or "无入口参数"
|
||||
return None, f"工作流「{wf_row.name}({wf_row.id})」入口节点不存在参数「{raw_param}」(可用参数:{sample_in})"
|
||||
valid_cards = env.workflow_cards.get(wf_row.id, set())
|
||||
canon_card = ""
|
||||
if raw_card:
|
||||
matched_card = next((c for c in valid_cards if c.lower() == raw_card.lower()), None)
|
||||
if matched_card is None:
|
||||
sample_cd = "、".join(sorted(valid_cards)) or "无口径卡节点"
|
||||
return None, f"工作流「{wf_row.name}({wf_row.id})」中不存在口径卡「{raw_card}」(可用口径卡:{sample_cd})"
|
||||
canon_card = matched_card
|
||||
|
||||
target_status = "confirmed" if mode == "confirmed" else "proposed"
|
||||
target_source = "human" if mode == "confirmed" else "ai"
|
||||
pkey = (wf_row.id, canon_card.lower(), canon_param.lower())
|
||||
p_idx = next(
|
||||
(
|
||||
i
|
||||
for i, p in enumerate(state["caliber_params"])
|
||||
if (
|
||||
str(p.get("workflow_id") or ""),
|
||||
str(p.get("card_id") or "").lower(),
|
||||
str(p.get("param_name") or "").lower(),
|
||||
)
|
||||
== pkey
|
||||
),
|
||||
None,
|
||||
)
|
||||
if p_idx is not None:
|
||||
cur_p = state["caliber_params"][p_idx]
|
||||
cur_status = str(cur_p.get("status") or "confirmed")
|
||||
if mode == "proposed" and cur_status in ("confirmed", "verified", "rejected"):
|
||||
return None, None
|
||||
if cur_status == target_status and str(cur_p.get("note") or "") == (note or str(cur_p.get("note") or "")):
|
||||
return None, None
|
||||
before_p = copy.deepcopy(cur_p)
|
||||
after_p = {
|
||||
**cur_p,
|
||||
"workflow_id": wf_row.id,
|
||||
"card_id": canon_card,
|
||||
"param_name": canon_param,
|
||||
"status": target_status,
|
||||
"source": target_source,
|
||||
"note": note or reason or str(cur_p.get("note") or ""),
|
||||
}
|
||||
state["caliber_params"][p_idx] = after_p
|
||||
else:
|
||||
before_p = None
|
||||
after_p = {
|
||||
"id": new_id()[:12],
|
||||
"workflow_id": wf_row.id,
|
||||
"card_id": canon_card,
|
||||
"param_name": canon_param,
|
||||
"status": target_status,
|
||||
"source": target_source,
|
||||
"note": note or reason,
|
||||
}
|
||||
state["caliber_params"].append(after_p)
|
||||
|
||||
payload = {
|
||||
"op": "upsert_caliber_param",
|
||||
"entity_key": entity_key,
|
||||
"entity_label": state["label"],
|
||||
"workflow_id": wf_row.id,
|
||||
"card_id": canon_card,
|
||||
"param_name": canon_param,
|
||||
"note": note or reason,
|
||||
"reason": reason,
|
||||
}
|
||||
card_part = f" · {canon_card}" if canon_card else ""
|
||||
return EntityPatchChange(
|
||||
id=_change_id(op, entity_key, f"{wf_row.id}:{canon_card}:{canon_param}"),
|
||||
op=op,
|
||||
entity_key=entity_key,
|
||||
entity_label=state["label"],
|
||||
summary=f"{'新增' if before_p is None else '更新'}实体「{state['label']}」口径入参映射:{wf_row.name}{card_part} → {canon_param}",
|
||||
before=before_p,
|
||||
after=after_p,
|
||||
reason=reason,
|
||||
op_payload=payload,
|
||||
), None
|
||||
|
||||
|
||||
async def plan_entity_patch(
|
||||
session: AsyncSession,
|
||||
raw_ops: Sequence[Any],
|
||||
*,
|
||||
pending_ops: Sequence[Any] = (),
|
||||
mode: Literal["confirmed", "proposed"] = "confirmed",
|
||||
) -> EntityPatchPlan:
|
||||
"""校验一批增量修改操作并生成结构化变更预览,不写入数据库。"""
|
||||
env = await _load_env(session)
|
||||
working: dict[str, dict[str, Any]] = {}
|
||||
|
||||
# 先回放界面待保存的操作,保证后续新增实体/成员在此基础上叠加
|
||||
for p_idx, pop in enumerate(pending_ops or ()):
|
||||
_normalize_single_op(pop, index=p_idx, env=env, working=working, mode=mode)
|
||||
|
||||
plan = EntityPatchPlan()
|
||||
for idx, item in enumerate(raw_ops or ()):
|
||||
change, problem = _normalize_single_op(item, index=idx, env=env, working=working, mode=mode)
|
||||
if problem:
|
||||
plan.problems.append(problem)
|
||||
continue
|
||||
if change is not None:
|
||||
plan.changes.append(change)
|
||||
|
||||
touched_keys = {c.entity_key for c in plan.changes}
|
||||
for ekey in sorted(touched_keys):
|
||||
existing = env.entities_by_key.get(ekey)
|
||||
plan.expected_versions[ekey] = int(existing.version or 1) if existing is not None else 0
|
||||
return plan
|
||||
|
||||
|
||||
async def apply_entity_patch(
|
||||
session: AsyncSession,
|
||||
raw_ops: Sequence[Any],
|
||||
*,
|
||||
mode: Literal["confirmed", "proposed"],
|
||||
actor: str | None = None,
|
||||
expected_versions: Mapping[str, int] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""将一批增量操作合并并经 entities.create_entity / update_entity 统一落库。"""
|
||||
if not raw_ops:
|
||||
raise entities.EntityInvalid("没有可保存的实体修改项", field="operations")
|
||||
|
||||
env = await _load_env(session)
|
||||
working: dict[str, dict[str, Any]] = {}
|
||||
changes: list[EntityPatchChange] = []
|
||||
problems: list[str] = []
|
||||
|
||||
for idx, item in enumerate(raw_ops):
|
||||
change, problem = _normalize_single_op(item, index=idx, env=env, working=working, mode=mode)
|
||||
if problem:
|
||||
problems.append(problem)
|
||||
continue
|
||||
if change is not None:
|
||||
changes.append(change)
|
||||
|
||||
if mode == "confirmed" and problems:
|
||||
raise entities.EntityInvalid(";".join(problems), field="operations")
|
||||
|
||||
touched_keys = list(dict.fromkeys(c.entity_key for c in changes))
|
||||
if mode == "confirmed" and expected_versions is not None:
|
||||
for ekey in touched_keys:
|
||||
if ekey not in expected_versions:
|
||||
continue
|
||||
exp_ver = int(expected_versions[ekey])
|
||||
cur_row = env.entities_by_key.get(ekey)
|
||||
cur_ver = int(cur_row.version or 1) if cur_row is not None else 0
|
||||
if exp_ver != cur_ver:
|
||||
label = cur_row.label if cur_row is not None else ekey
|
||||
raise entities.EntityConflict(
|
||||
f"标准实体「{label}」在对话期间已被修改(基准 v{exp_ver},当前 v{cur_ver}),请刷新后再应用修改",
|
||||
field="expected_versions",
|
||||
)
|
||||
|
||||
created_count = 0
|
||||
updated_count = 0
|
||||
saved_entities: list[dict[str, Any]] = []
|
||||
|
||||
for ekey in touched_keys:
|
||||
state = working.get(ekey)
|
||||
if state is None:
|
||||
continue
|
||||
payload: dict[str, Any] = {
|
||||
"key": state["key"],
|
||||
"label": state["label"],
|
||||
"description": state["description"],
|
||||
"aliases": state["aliases"],
|
||||
"members": state["members"],
|
||||
"bindings": state["bindings"],
|
||||
"caliber_params": state["caliber_params"],
|
||||
}
|
||||
cur_row = env.entities_by_key.get(ekey)
|
||||
if cur_row is None:
|
||||
row = await entities.create_entity(session, payload, actor=actor)
|
||||
created_count += 1
|
||||
else:
|
||||
payload["version"] = int(cur_row.version or 1)
|
||||
row = await entities.update_entity(session, cur_row.id, payload, actor=actor)
|
||||
updated_count += 1
|
||||
saved_entities.append(entities.entity_to_dict(row))
|
||||
|
||||
proposed_bindings = sum(1 for c in changes if c.op == "upsert_binding")
|
||||
proposed_params = sum(1 for c in changes if c.op == "upsert_caliber_param")
|
||||
added_members = sum(1 for c in changes if c.op == "upsert_member")
|
||||
|
||||
return {
|
||||
"applied_count": len(changes),
|
||||
"created_entities": created_count,
|
||||
"updated_entities": updated_count,
|
||||
"bindings_changed": proposed_bindings,
|
||||
"caliber_params_changed": proposed_params,
|
||||
"members_changed": added_members,
|
||||
"entities": saved_entities,
|
||||
"problems": problems,
|
||||
}
|
||||
@@ -0,0 +1,463 @@
|
||||
"""跨数据源健康与漂移快速探针(source_health,W1-b)。
|
||||
|
||||
用于指标异动与跨源穿透时的第 1 顺位核查(先排数,后归因):
|
||||
1. 读取 `SourceChange` 待处理变化记录(`date_stalled`、`date_gap`、`volume_drop`、`rows_drop`、`codes_new`、
|
||||
`codes_shift`、`coverage_drop`、`table_removed`、`column_removed`、`column_type` 等);
|
||||
2. 对目标表非遮罩的时间列做有界只读鲜度与时间窗覆盖探查(`PROBE_TIMEOUT_S = 10.0s`,`guard.check + run_query`),
|
||||
超时或失败优雅降级为 `unverified`,绝不拖垮主运行;
|
||||
3. 严守遮罩铁律(铁律 3):绝不探查或返回任何 `mask_columns` 列的值;`restricted=True`(普通 user 角色)时自动隐去
|
||||
物理表名、物理列名与底层码值细节,只输出业务数据源说明与高层健康判定。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from collections.abc import Iterable, Mapping
|
||||
from datetime import date, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core import artifact_store
|
||||
from app.data import schema_diff as sd
|
||||
from app.data import source_changes as sc
|
||||
from app.data.coverage import _as_day
|
||||
from app.data.engine import QueryLimits, masked_columns, run_query
|
||||
from app.data.guard import check, is_write
|
||||
from app.data.names import name_key
|
||||
from app.db.models import DataSource, Workflow, WorkflowVersion
|
||||
from app.db.models_sources import SourceChange
|
||||
|
||||
ARTIFACT_KIND = "source_health"
|
||||
LEDGER_KIND = "source_health"
|
||||
CATALOG_PREFIX = "H"
|
||||
PROBE_TIMEOUT_S = 10.0
|
||||
MAX_TABLES_PROBED = 6
|
||||
|
||||
HEALTHY = "healthy"
|
||||
DEGRADED = "degraded"
|
||||
ANOMALOUS = "anomalous"
|
||||
UNVERIFIED = "unverified"
|
||||
VERDICTS = (HEALTHY, DEGRADED, ANOMALOUS, UNVERIFIED)
|
||||
|
||||
_SEVERE_CHANGE_KINDS = frozenset({
|
||||
sd.TABLE_REMOVED,
|
||||
sd.COLUMN_REMOVED,
|
||||
sd.COLUMN_TYPE,
|
||||
sc.DATE_STALLED,
|
||||
sc.DATE_GAP,
|
||||
sc.VOLUME_DROP,
|
||||
sc.ROWS_DROP,
|
||||
sc.COVERAGE_DROP,
|
||||
})
|
||||
_WARN_CHANGE_KINDS = frozenset({
|
||||
sd.TABLE_REFLECT_FAILED,
|
||||
sd.TABLE_LEFT_SCOPE,
|
||||
sc.CODES_NEW,
|
||||
sc.CODES_SHIFT,
|
||||
sc.ROWS_JUMP,
|
||||
})
|
||||
|
||||
|
||||
def _quote_ident(name: str, dialect: str) -> str:
|
||||
clean = str(name)
|
||||
if dialect in ("mysql", "mariadb"):
|
||||
return "`" + clean.replace("`", "``") + "`"
|
||||
return '"' + clean.replace('"', '""') + '"'
|
||||
|
||||
|
||||
def _date_columns_of(table_meta: Mapping[str, Any], masked: set[str]) -> list[str]:
|
||||
cols = table_meta.get("columns")
|
||||
if not isinstance(cols, list):
|
||||
return []
|
||||
out: list[str] = []
|
||||
for col in cols:
|
||||
if not isinstance(col, Mapping):
|
||||
continue
|
||||
cname = str(col.get("name") or "").strip()
|
||||
if not cname or cname.lower() in masked:
|
||||
continue
|
||||
ctype = str(col.get("type") or "").lower()
|
||||
low = cname.lower()
|
||||
if any(tok in ctype for tok in ("date", "time", "timestamp")) or any(
|
||||
low == k or low.endswith(f"_{k}") or low.startswith(f"{k}_")
|
||||
for k in ("date", "day", "dt", "shift_date", "production_day", "biz_date", "created_at")
|
||||
):
|
||||
out.append(cname)
|
||||
return out[:2]
|
||||
|
||||
|
||||
async def _resolve_caliber_targets(
|
||||
session: AsyncSession,
|
||||
*,
|
||||
workflow_id: str,
|
||||
version: int | None,
|
||||
card_id: str | None,
|
||||
metric_id: str | None,
|
||||
) -> list[tuple[str, list[str]]]:
|
||||
"""根据已发布(或指定版本)口径自动反查 (source_name, [table_names])。"""
|
||||
from app.ledger.ontology import workflow_ontology_index
|
||||
|
||||
wf = await session.get(Workflow, workflow_id)
|
||||
if wf is None:
|
||||
return []
|
||||
ver = version if version is not None else wf.published_version or wf.version
|
||||
snap = (
|
||||
await session.execute(
|
||||
select(WorkflowVersion).where(
|
||||
WorkflowVersion.workflow_id == workflow_id,
|
||||
WorkflowVersion.version == int(ver),
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
graph = (snap.graph if snap is not None else wf.graph) or {}
|
||||
idx = workflow_ontology_index(workflow_id=workflow.id if (workflow := wf) else workflow_id, version=int(ver), graph=graph)
|
||||
by_src: dict[str, set[str]] = {}
|
||||
for card in idx["cards"]:
|
||||
if card_id and card["card_id"] != card_id:
|
||||
continue
|
||||
if metric_id:
|
||||
for m in card["metrics"]:
|
||||
if m["metric_id"] == metric_id:
|
||||
for item in m["lineage"]:
|
||||
by_src.setdefault(item["source"], set()).update(item["tables"])
|
||||
else:
|
||||
for src, tbls in card["tables_by_source"].items():
|
||||
by_src.setdefault(src, set()).update(tbls)
|
||||
return [(src, sorted(tbls)) for src, tbls in sorted(by_src.items())]
|
||||
|
||||
|
||||
def _parse_window_day(raw: str | date | None) -> date | None:
|
||||
if raw is None:
|
||||
return None
|
||||
if isinstance(raw, datetime):
|
||||
return raw.date()
|
||||
if isinstance(raw, date):
|
||||
return raw
|
||||
d = _as_day(str(raw).strip())
|
||||
return d.day if d is not None else None
|
||||
|
||||
|
||||
async def _probe_table_coverage(
|
||||
source: DataSource,
|
||||
table_name: str,
|
||||
table_meta: Mapping[str, Any],
|
||||
*,
|
||||
masked: set[str],
|
||||
start_day: date | None,
|
||||
end_day: date | None,
|
||||
) -> dict[str, Any]:
|
||||
"""对单张表做只读行数与非遮罩日期边界探查,超时或报错返回 status='unverified'。"""
|
||||
dialect = (source.kind or "sqlite").lower()
|
||||
q_table = _quote_ident(table_name, dialect)
|
||||
date_cols = _date_columns_of(table_meta, masked)
|
||||
select_parts = ["COUNT(*) AS row_cnt"]
|
||||
for idx, col in enumerate(date_cols):
|
||||
qc = _quote_ident(col, dialect)
|
||||
select_parts.append(f"MIN({qc}) AS min_d{idx}")
|
||||
select_parts.append(f"MAX({qc}) AS max_d{idx}")
|
||||
sql = f"SELECT {', '.join(select_parts)} FROM {q_table}"
|
||||
if is_write(sql, dialect=dialect):
|
||||
return {"table": table_name, "status": UNVERIFIED, "reason": "探查语句未通过只读校验"}
|
||||
check(sql, readonly=True, source_name=source.name, dialect=dialect)
|
||||
|
||||
try:
|
||||
res = await asyncio.wait_for(
|
||||
run_query(source, sql, limits=QueryLimits(max_rows=5, timeout_seconds=PROBE_TIMEOUT_S)),
|
||||
timeout=PROBE_TIMEOUT_S + 1.0,
|
||||
)
|
||||
except Exception: # noqa: BLE001 - 探查失败优雅降级,绝不中断主流程
|
||||
return {"table": table_name, "status": UNVERIFIED, "reason": "实时探查超时或暂时不可达"}
|
||||
|
||||
if not res.rows:
|
||||
return {"table": table_name, "status": ANOMALOUS, "issue": "empty_table", "row_count": 0}
|
||||
first = res.rows[0]
|
||||
row_cnt = int(first[0] or 0) if first and first[0] is not None else 0
|
||||
if row_cnt == 0:
|
||||
return {"table": table_name, "status": ANOMALOUS, "issue": "empty_table", "row_count": 0}
|
||||
|
||||
ranges: list[dict[str, Any]] = []
|
||||
window_issues: list[dict[str, Any]] = []
|
||||
for idx, col in enumerate(date_cols):
|
||||
pos = 1 + idx * 2
|
||||
raw_min = first[pos] if pos < len(first) else None
|
||||
raw_max = first[pos + 1] if (pos + 1) < len(first) else None
|
||||
d_min = _as_day(raw_min)
|
||||
d_max = _as_day(raw_max)
|
||||
ranges.append({
|
||||
"column": col,
|
||||
"min": d_min.day.isoformat() if d_min else (str(raw_min) if raw_min is not None else None),
|
||||
"max": d_max.day.isoformat() if d_max else (str(raw_max) if raw_max is not None else None),
|
||||
})
|
||||
if start_day and d_max and d_max.day < start_day:
|
||||
window_issues.append({
|
||||
"kind": sc.DATE_STALLED,
|
||||
"severity": "action",
|
||||
"column": col,
|
||||
"max_date": d_max.day.isoformat(),
|
||||
"requested_start": start_day.isoformat(),
|
||||
})
|
||||
elif end_day and d_min and d_min.day >= end_day:
|
||||
window_issues.append({
|
||||
"kind": sc.DATE_GAP,
|
||||
"severity": "action",
|
||||
"column": col,
|
||||
"min_date": d_min.day.isoformat(),
|
||||
"requested_end": end_day.isoformat(),
|
||||
})
|
||||
return {
|
||||
"table": table_name,
|
||||
"status": ANOMALOUS if window_issues else HEALTHY,
|
||||
"row_count": row_cnt,
|
||||
"ranges": ranges,
|
||||
"window_issues": window_issues,
|
||||
}
|
||||
|
||||
|
||||
def _render_phrase(source_title: str, verdict: str, issues: list[dict[str, Any]], *, probe_unverified: bool) -> str:
|
||||
if verdict == ANOMALOUS:
|
||||
labels = list(dict.fromkeys(str(i.get("label") or i.get("kind") or "数据异常") for i in issues))
|
||||
shown = "、".join(labels[:3])
|
||||
return f"数据源「{source_title}」检测到数据健康异常({shown}),当前异动优先归因于上游同步或主数据质量问题,不宜直接归结为业务实质变化。"
|
||||
if verdict == DEGRADED:
|
||||
labels = list(dict.fromkeys(str(i.get("label") or i.get("kind") or "取值变化") for i in issues))
|
||||
shown = "、".join(labels[:3])
|
||||
return f"数据源「{source_title}」存在待审阅的数据变化提醒({shown}),跨源对比与归因时需留意码值或分布漂移。"
|
||||
if verdict == UNVERIFIED or probe_unverified:
|
||||
return f"数据源「{source_title}」未发现已登记的变化告警,但实时范围探查未完成(状态记为待核实)。"
|
||||
return f"数据源「{source_title}」同步状态与查询区间覆盖正常,未发现断更、缺口或未处理的结构漂移。"
|
||||
|
||||
|
||||
async def check_source_health(
|
||||
session: AsyncSession,
|
||||
*,
|
||||
source_name: str | None = None,
|
||||
tables: Iterable[str] | None = None,
|
||||
workflow_id: str | None = None,
|
||||
version: int | None = None,
|
||||
card_id: str | None = None,
|
||||
metric_id: str | None = None,
|
||||
start: str | date | None = None,
|
||||
end: str | date | None = None,
|
||||
restricted: bool = False,
|
||||
live_probe: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
"""检查目标数据源(或目标口径底层依赖源)的健康度与数据漂移,返回确定性结构化结论。"""
|
||||
targets: list[tuple[str, list[str]]] = []
|
||||
if source_name and str(source_name).strip():
|
||||
explicit_tables = [str(t).strip() for t in (tables or ()) if str(t).strip()]
|
||||
targets.append((str(source_name).strip(), explicit_tables))
|
||||
elif workflow_id and str(workflow_id).strip():
|
||||
targets = await _resolve_caliber_targets(
|
||||
session,
|
||||
workflow_id=str(workflow_id).strip(),
|
||||
version=version,
|
||||
card_id=card_id,
|
||||
metric_id=metric_id,
|
||||
)
|
||||
if not targets:
|
||||
return {
|
||||
"kind": ARTIFACT_KIND,
|
||||
"verdict": UNVERIFIED,
|
||||
"sources": [],
|
||||
"issues": [],
|
||||
"phrase": "未能定位目标数据源,健康状态记为未核实。",
|
||||
}
|
||||
|
||||
start_day = _parse_window_day(start)
|
||||
end_day = _parse_window_day(end)
|
||||
|
||||
source_reports: list[dict[str, Any]] = []
|
||||
all_issues: list[dict[str, Any]] = []
|
||||
overall_rank = 0 # 0 healthy, 1 unverified, 2 degraded, 3 anomalous
|
||||
rank_to_verdict = {0: HEALTHY, 1: UNVERIFIED, 2: DEGRADED, 3: ANOMALOUS}
|
||||
|
||||
for src_name, target_tables in targets:
|
||||
src = (
|
||||
await session.execute(select(DataSource).where(DataSource.name == src_name).limit(1))
|
||||
).scalar_one_or_none()
|
||||
if src is None or not src.enabled:
|
||||
overall_rank = max(overall_rank, 3 if src is not None and not src.enabled else 1)
|
||||
status = ANOMALOUS if (src is not None and not src.enabled) else UNVERIFIED
|
||||
issue_label = "数据源已停用" if status == ANOMALOUS else "数据源未找到"
|
||||
issue = {"kind": "source_unavailable", "severity": "action", "label": issue_label}
|
||||
if not restricted:
|
||||
issue["source"] = src_name
|
||||
all_issues.append(issue)
|
||||
source_reports.append({
|
||||
"source_label": (src.description or src.name) if src else ("关联数据源" if restricted else src_name),
|
||||
**({} if restricted else {"source": src_name}),
|
||||
"verdict": status,
|
||||
"issues": [issue],
|
||||
})
|
||||
continue
|
||||
|
||||
masked = {c.lower() for c in masked_columns(src.options)}
|
||||
schema_tables = (src.schema_cache or {}).get("tables") if isinstance(src.schema_cache, Mapping) else {}
|
||||
if not isinstance(schema_tables, Mapping):
|
||||
schema_tables = {}
|
||||
|
||||
wanted_keys = {name_key(t) for t in target_tables} if target_tables else None
|
||||
change_rows = list(
|
||||
(
|
||||
await session.execute(
|
||||
select(SourceChange)
|
||||
.where(SourceChange.source_id == src.id, SourceChange.status.in_(sc.PENDING))
|
||||
.order_by(SourceChange.seq.desc())
|
||||
)
|
||||
).scalars()
|
||||
)
|
||||
src_issues: list[dict[str, Any]] = []
|
||||
src_rank = 0
|
||||
for ch in change_rows:
|
||||
if wanted_keys is not None and ch.table_name and name_key(ch.table_name) not in wanted_keys:
|
||||
continue
|
||||
# 遮罩列上的变化不暴露列名与取值细节(铁律 3)
|
||||
if ch.column_name and ch.column_name.lower() in masked:
|
||||
continue
|
||||
label = sc.KIND_LABEL.get(ch.kind, ch.kind)
|
||||
is_severe = ch.kind in _SEVERE_CHANGE_KINDS or ch.severity == "action"
|
||||
src_rank = max(src_rank, 3 if is_severe else 2)
|
||||
entry: dict[str, Any] = {
|
||||
"kind": ch.kind,
|
||||
"severity": ch.severity,
|
||||
"status": ch.status,
|
||||
"label": label,
|
||||
}
|
||||
if not restricted:
|
||||
entry["source"] = src.name
|
||||
if ch.table_name:
|
||||
entry["table"] = ch.table_name
|
||||
if ch.column_name:
|
||||
entry["column"] = ch.column_name
|
||||
entry["summary"] = sc.summary(ch)
|
||||
src_issues.append(entry)
|
||||
|
||||
probe_unverified = False
|
||||
probed_tables: list[dict[str, Any]] = []
|
||||
if live_probe:
|
||||
candidate_names = target_tables or list(schema_tables.keys())[:MAX_TABLES_PROBED]
|
||||
for tbl_name in candidate_names[:MAX_TABLES_PROBED]:
|
||||
# 找到 schema_cache 里精确的表名键
|
||||
matched_name = next((k for k in schema_tables if name_key(k) == name_key(tbl_name)), tbl_name)
|
||||
meta = schema_tables.get(matched_name) if isinstance(schema_tables.get(matched_name), Mapping) else {}
|
||||
probe = await _probe_table_coverage(
|
||||
src,
|
||||
matched_name,
|
||||
meta,
|
||||
masked=masked,
|
||||
start_day=start_day,
|
||||
end_day=end_day,
|
||||
)
|
||||
if probe["status"] == UNVERIFIED:
|
||||
probe_unverified = True
|
||||
if src_rank == 0:
|
||||
src_rank = 1
|
||||
elif probe["status"] == ANOMALOUS:
|
||||
src_rank = max(src_rank, 3)
|
||||
if probe.get("issue") == "empty_table":
|
||||
item: dict[str, Any] = {
|
||||
"kind": sc.ROWS_DROP,
|
||||
"severity": "action",
|
||||
"label": "目标表当前无数据(0 行)",
|
||||
}
|
||||
if not restricted:
|
||||
item["source"] = src.name
|
||||
item["table"] = matched_name
|
||||
src_issues.append(item)
|
||||
for wi in probe.get("window_issues") or []:
|
||||
item = {
|
||||
"kind": wi["kind"],
|
||||
"severity": wi["severity"],
|
||||
"label": sc.KIND_LABEL.get(wi["kind"], wi["kind"]),
|
||||
}
|
||||
if not restricted:
|
||||
item["source"] = src.name
|
||||
item["table"] = matched_name
|
||||
item["column"] = wi["column"]
|
||||
if "max_date" in wi:
|
||||
item["max_date"] = wi["max_date"]
|
||||
if "min_date" in wi:
|
||||
item["min_date"] = wi["min_date"]
|
||||
src_issues.append(item)
|
||||
if not restricted:
|
||||
probed_tables.append(probe)
|
||||
|
||||
src_verdict = rank_to_verdict[src_rank]
|
||||
overall_rank = max(overall_rank, src_rank)
|
||||
all_issues.extend(src_issues)
|
||||
display_title = (src.description or "关联业务源") if restricted else src.name
|
||||
rep: dict[str, Any] = {
|
||||
"source_label": display_title,
|
||||
"verdict": src_verdict,
|
||||
"issue_count": len(src_issues),
|
||||
"issues": src_issues,
|
||||
"phrase": _render_phrase(display_title, src_verdict, src_issues, probe_unverified=probe_unverified),
|
||||
}
|
||||
if not restricted:
|
||||
rep["source"] = src.name
|
||||
rep["tables_probed"] = probed_tables
|
||||
source_reports.append(rep)
|
||||
|
||||
overall_verdict = rank_to_verdict[overall_rank]
|
||||
lead_phrase = source_reports[0]["phrase"] if len(source_reports) == 1 else (
|
||||
";".join(r["phrase"] for r in source_reports)
|
||||
)
|
||||
out: dict[str, Any] = {
|
||||
"kind": ARTIFACT_KIND,
|
||||
"verdict": overall_verdict,
|
||||
"sources": source_reports,
|
||||
"issues": all_issues,
|
||||
"phrase": lead_phrase,
|
||||
}
|
||||
if workflow_id:
|
||||
out["caliber_ref"] = {
|
||||
"workflow_id": workflow_id,
|
||||
**({"version": version} if version is not None else {}),
|
||||
**({"card_id": card_id} if card_id else {}),
|
||||
**({"metric_id": metric_id} if metric_id else {}),
|
||||
}
|
||||
return out
|
||||
|
||||
|
||||
def is_health_result(content: Any) -> bool:
|
||||
return (
|
||||
isinstance(content, Mapping)
|
||||
and content.get("kind") == ARTIFACT_KIND
|
||||
and content.get("verdict") in VERDICTS
|
||||
and isinstance(content.get("sources"), list)
|
||||
)
|
||||
|
||||
|
||||
async def store_source_health(result: Mapping[str, Any], *, run_id: str | None, node_id: str | None = None) -> str:
|
||||
return await artifact_store.put_json(dict(result), kind=ARTIFACT_KIND, run_id=run_id, meta={
|
||||
"verdict": result.get("verdict"),
|
||||
"node_id": node_id or "",
|
||||
})
|
||||
|
||||
|
||||
def ledger_entry(
|
||||
result: Mapping[str, Any],
|
||||
*,
|
||||
node_id: str,
|
||||
exec_no: int,
|
||||
artifact: str,
|
||||
call_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
sources = [
|
||||
str(s.get("source") or s.get("source_label") or "")
|
||||
for s in (result.get("sources") or [])
|
||||
if isinstance(s, Mapping)
|
||||
]
|
||||
entry: dict[str, Any] = {
|
||||
"kind": LEDGER_KIND,
|
||||
"node_id": node_id,
|
||||
"exec": exec_no,
|
||||
"artifact": artifact,
|
||||
"verdict": result.get("verdict"),
|
||||
"sources": [s for s in sources if s],
|
||||
"issue_count": len(result.get("issues") or []),
|
||||
}
|
||||
if call_id is not None:
|
||||
entry["call_id"] = call_id
|
||||
return entry
|
||||
Reference in New Issue
Block a user