diff --git a/backend/app/api/semantic_metrics.py b/backend/app/api/semantic_metrics.py new file mode 100644 index 00000000..5c156c13 --- /dev/null +++ b/backend/app/api/semantic_metrics.py @@ -0,0 +1,418 @@ +"""独立语义指标库（SemanticMetric）与后台口径文档一键拆解工作流 REST 接口（搭建者及以上权限）。 + +- GET /api/semantic-metrics：列出语义指标卡片（支持按 source_id、status、q 过滤） +- POST /api/semantic-metrics：创建单条语义指标卡片（支持单表 expression 与多表参数化 sql_template，保存即刻生效） +- POST /api/semantic-metrics/batch：批量保存勾选的候选语义指标卡片 +- PUT /api/semantic-metrics/{metric_id}：编辑语义指标卡片（支持乐观锁 expected_version） +- DELETE /api/semantic-metrics/{metric_id}：删除语义指标卡片 +- POST /api/semantic-metrics/preview：在线试算（支持已落库 metric_id 或草稿 definition 实时预览 SQL 与结果表） +- POST /api/semantic-metrics/suggest：同步提取候选指标卡片（兼容旧调用） +- POST /api/semantic-metrics/attachments：上传业务口径文档或编码对照表附件（.md/.txt/.pdf/.docx/.xlsx/.csv） +- POST /api/semantic-metrics/decompose-jobs：启动后台文档一键拆解 / 智能提取指标任务（实时阶段进度，关闭弹窗继续执行） +- GET /api/semantic-metrics/decompose-jobs/active：查询正在运行与最近一次拆解任务（供口径台账页顶部横幅轮询） +- GET /api/semantic-metrics/decompose-jobs/{job_id}：查询单个拆解任务的实时进度、分步耗时与待审阅结果 +- POST /api/semantic-metrics/decompose-jobs/{job_id}/cancel：取消排队或执行中的拆解任务 +- POST /api/semantic-metrics/decompose-jobs/{job_id}/apply：一键应用审阅勾选的「字段目录补丁 + 指标口径 + 标准业务实体」 +""" +from __future__ import annotations + +from typing import Any + +from fastapi import APIRouter, BackgroundTasks, Depends, File, Form, Header, HTTPException, Query, UploadFile +from pydantic import BaseModel, Field +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.api.coded import CodedHTTPException +from app.api.deps import actor_of +from app.data import entity_docs +from app.data.semantic_decompose import ( + DECOMPOSE_BUSY_CODE, + DECOMPOSE_FINISHED_CODE, + DECOMPOSE_NOT_FOUND_CODE, + DecomposeBusy, + DecomposeFinished, + DecomposeNotFound, + apply_decompose_job, + cancel_decompose_job, + create_decompose_job, + get_active_or_latest_job, + get_job, + job_to_dict, + list_jobs, +) +from app.data.semantic_metrics import ( + MetricConflict, + MetricInvalid, + MetricNotFound, + create_metric, + delete_metric, + get_metric, + list_metrics, + metric_to_dict, + preview_metric, + suggest_metrics_from_source, + update_metric, +) +from app.db.base import get_session +from app.db.models import DataSource + +router = APIRouter(prefix="/api/semantic-metrics", tags=["semantic-metrics"]) + + +class MetricDimensionIn(BaseModel): + key: str = "" + label: str = "" + column: str = Field(min_length=1) + entity_key: str = "" + + +class MetricCreateIn(BaseModel): + key: str = Field(min_length=1, max_length=64) + name: str = Field(min_length=1, max_length=200) + aliases: list[str] = Field(default_factory=list) + description: str = "" + unit: str = "" + source_id: str = Field(min_length=1) + calc_mode: str = "expression" + table_name: str = "" + sql_expression: str = "" + sql_template: str = "" + date_column: str = "" + filter_sql: str = "" + dimensions: list[MetricDimensionIn] = Field(default_factory=list) + default_time_grain: str = "day" + status: str = "enabled" + source: str = "human" + + +class MetricUpdateIn(BaseModel): + key: str | None = None + name: str | None = None + aliases: list[str] | None = None + description: str | None = None + unit: str | None = None + source_id: str | None = None + calc_mode: str | None = None + table_name: str | None = None + sql_expression: str | None = None + sql_template: str | None = None + date_column: str | None = None + filter_sql: str | None = None + dimensions: list[MetricDimensionIn] | None = None + default_time_grain: str | None = None + status: str | None = None + expected_version: int | None = None + + +class MetricBatchCreateIn(BaseModel): + items: list[MetricCreateIn] = Field(min_length=1, max_length=50) + + +class MetricPreviewIn(BaseModel): + metric_id: str | None = None + definition: dict[str, Any] | None = None + date_from: str | None = None + date_to: str | None = None + group_by_date: bool = False + time_grain: str | None = None + group_by: list[str] = Field(default_factory=list) + filters: dict[str, Any] = Field(default_factory=dict) + limit: int = Field(default=50, ge=1, le=200) + + +class MetricSuggestIn(BaseModel): + source_id: str = Field(min_length=1) + table_names: list[str] | None = None + document_ids: list[str] | None = None + note: str = "" + + +class DecomposeJobCreateIn(BaseModel): + source_id: str = Field(min_length=1) + scopes: list[str] | None = None + table_names: list[str] | None = None + document_ids: list[str] = Field(default_factory=list) + attachment_ids: list[str] = Field(default_factory=list) + doc_title: str = "" + doc_text: str = "" + save_to_kb: bool = True + note: str = "" + + +class DecomposeJobApplyIn(BaseModel): + catalog_patches: list[dict[str, Any]] | None = None + metrics: list[dict[str, Any]] | None = None + entity_operations: list[dict[str, Any]] | None = None + overwrite_existing_metrics: bool = True + + +def _raise_metric_error(exc: Exception) -> HTTPException: + if isinstance(exc, DecomposeNotFound): + return CodedHTTPException(404, str(exc), DECOMPOSE_NOT_FOUND_CODE) + if isinstance(exc, DecomposeBusy): + return CodedHTTPException(409, str(exc), DECOMPOSE_BUSY_CODE) + if isinstance(exc, DecomposeFinished): + return CodedHTTPException(409, str(exc), DECOMPOSE_FINISHED_CODE) + if isinstance(exc, MetricNotFound): + return CodedHTTPException(404, str(exc), "metric_not_found") + if isinstance(exc, MetricConflict): + return CodedHTTPException(409, str(exc), "metric_conflict") + if isinstance(exc, MetricInvalid): + return CodedHTTPException(422, str(exc), "metric_invalid") + return HTTPException(400, str(exc)) + + +async def _source_names_map(session: AsyncSession, source_ids: set[str]) -> dict[str, str]: + if not source_ids: + return {} + rows = (await session.execute(select(DataSource.id, DataSource.name).where(DataSource.id.in_(source_ids)))).all() + return {str(sid): str(sname) for sid, sname in rows} + + +@router.get("") +async def api_list_metrics( + source_id: str | None = Query(default=None), + status: str | None = Query(default=None), + q: str | None = Query(default=None), + session: AsyncSession = Depends(get_session), +) -> dict[str, Any]: + rows = await list_metrics(session, source_id=source_id, status=status, q=q) + src_names = await _source_names_map(session, {r.source_id for r in rows if r.source_id}) + return { + "items": [metric_to_dict(r, source_name=src_names.get(r.source_id, "")) for r in rows], + "total": len(rows), + } + + +@router.post("/attachments") +async def api_upload_decompose_attachment( + background: BackgroundTasks, + file: UploadFile = File(...), + save_to_kb: bool = Form(default=False), + collection: str = Form(default="default"), + session: AsyncSession = Depends(get_session), +) -> dict[str, Any]: + try: + raw_bytes = await entity_docs.read_upload_limited(file) + res = await entity_docs.ingest_entity_attachment( + session, + raw_bytes=raw_bytes, + filename=file.filename or "上传文档", + mime=file.content_type or "", + save_to_kb=save_to_kb, + collection=collection, + background=background, + ) + await session.commit() + return res + except entity_docs.AttachmentTooLarge as exc: + raise CodedHTTPException(413, str(exc), "attachment_too_large") from exc + except entity_docs.AttachmentInvalid as exc: + raise CodedHTTPException(422, str(exc), "attachment_invalid") from exc + + +@router.get("/decompose-jobs/active") +async def api_active_decompose_job( + source_id: str | None = Query(default=None), + session: AsyncSession = Depends(get_session), +) -> dict[str, Any]: + return await get_active_or_latest_job(session, source_id=source_id) + + +@router.get("/decompose-jobs") +async def api_list_decompose_jobs( + source_id: str | None = Query(default=None), + limit: int = Query(default=10, ge=1, le=30), + session: AsyncSession = Depends(get_session), +) -> dict[str, Any]: + items = await list_jobs(session, source_id=source_id, limit=limit) + return {"items": items} + + +@router.post("/decompose-jobs", status_code=202) +async def api_create_decompose_job( + payload: DecomposeJobCreateIn, + session: AsyncSession = Depends(get_session), + x_actor: str | None = Header(default=None), +) -> dict[str, Any]: + try: + job = await create_decompose_job( + session, + source_id=payload.source_id, + scopes=payload.scopes, + table_names=payload.table_names, + document_ids=payload.document_ids, + attachment_ids=payload.attachment_ids, + doc_title=payload.doc_title, + doc_text=payload.doc_text, + save_to_kb=payload.save_to_kb, + note=payload.note, + actor=actor_of(x_actor), + ) + return {"job": job_to_dict(job, include_result=False)} + except Exception as exc: # noqa: BLE001 + await session.rollback() + raise _raise_metric_error(exc) from exc + + +@router.get("/decompose-jobs/{job_id}") +async def api_get_decompose_job( + job_id: str, + session: AsyncSession = Depends(get_session), +) -> dict[str, Any]: + try: + job = await get_job(session, job_id) + return {"job": job_to_dict(job, include_result=True)} + except Exception as exc: # noqa: BLE001 + raise _raise_metric_error(exc) from exc + + +@router.post("/decompose-jobs/{job_id}/cancel") +async def api_cancel_decompose_job( + job_id: str, + session: AsyncSession = Depends(get_session), +) -> dict[str, Any]: + try: + job = await cancel_decompose_job(session, job_id) + return {"job": job_to_dict(job, include_result=False)} + except Exception as exc: # noqa: BLE001 + await session.rollback() + raise _raise_metric_error(exc) from exc + + +@router.post("/decompose-jobs/{job_id}/apply") +async def api_apply_decompose_job( + job_id: str, + payload: DecomposeJobApplyIn, + session: AsyncSession = Depends(get_session), + x_actor: str | None = Header(default=None), +) -> dict[str, Any]: + try: + return await apply_decompose_job( + session, + job_id, + catalog_patches=payload.catalog_patches, + metrics=payload.metrics, + entity_operations=payload.entity_operations, + overwrite_existing_metrics=payload.overwrite_existing_metrics, + actor=actor_of(x_actor), + ) + except Exception as exc: # noqa: BLE001 + await session.rollback() + raise _raise_metric_error(exc) from exc + + +@router.get("/{metric_id}") +async def api_get_metric(metric_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + try: + row = await get_metric(session, metric_id) + except Exception as exc: # noqa: BLE001 + raise _raise_metric_error(exc) from exc + src_names = await _source_names_map(session, {row.source_id} if row.source_id else set()) + return {"metric": metric_to_dict(row, source_name=src_names.get(row.source_id, ""))} + + +@router.post("", status_code=201) +async def api_create_metric( + payload: MetricCreateIn, + session: AsyncSession = Depends(get_session), + x_actor: str | None = Header(default=None), +) -> dict[str, Any]: + try: + row, ds = await create_metric(session, payload.model_dump(), actor=actor_of(x_actor)) + await session.commit() + await session.refresh(row) + except Exception as exc: # noqa: BLE001 + await session.rollback() + raise _raise_metric_error(exc) from exc + return {"metric": metric_to_dict(row, source_name=ds.name)} + + +@router.post("/batch", status_code=201) +async def api_batch_create_metrics( + payload: MetricBatchCreateIn, + session: AsyncSession = Depends(get_session), + x_actor: str | None = Header(default=None), +) -> dict[str, Any]: + actor = actor_of(x_actor) + created: list[dict[str, Any]] = [] + skipped: list[dict[str, str]] = [] + for item in payload.items: + data = item.model_dump() + try: + async with session.begin_nested(): + row, ds = await create_metric(session, data, actor=actor) + created.append(metric_to_dict(row, source_name=ds.name)) + except Exception as exc: # noqa: BLE001 + skipped.append({"key": item.key, "reason": str(exc)}) + await session.commit() + return {"created": created, "skipped": skipped} + + +@router.put("/{metric_id}") +async def api_update_metric( + metric_id: str, + payload: MetricUpdateIn, + session: AsyncSession = Depends(get_session), + x_actor: str | None = Header(default=None), +) -> dict[str, Any]: + data = {k: v for k, v in payload.model_dump().items() if v is not None} + try: + row, ds = await update_metric(session, metric_id, data, actor=actor_of(x_actor)) + await session.commit() + await session.refresh(row) + except Exception as exc: # noqa: BLE001 + await session.rollback() + raise _raise_metric_error(exc) from exc + return {"metric": metric_to_dict(row, source_name=ds.name)} + + +@router.delete("/{metric_id}") +async def api_delete_metric(metric_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + try: + row = await delete_metric(session, metric_id) + await session.commit() + except Exception as exc: # noqa: BLE001 + await session.rollback() + raise _raise_metric_error(exc) from exc + return {"ok": True, "deleted_id": row.id, "deleted_key": row.key} + + +@router.post("/preview") +async def api_preview_metric( + payload: MetricPreviewIn, + session: AsyncSession = Depends(get_session), +) -> dict[str, Any]: + try: + return await preview_metric( + session, + metric_id=payload.metric_id, + definition=payload.definition, + date_from=payload.date_from, + date_to=payload.date_to, + group_by_date=payload.group_by_date, + time_grain=payload.time_grain, + group_by=payload.group_by, + filters=payload.filters, + limit=payload.limit, + ) + except Exception as exc: # noqa: BLE001 + raise _raise_metric_error(exc) from exc + + +@router.post("/suggest") +async def api_suggest_metrics( + payload: MetricSuggestIn, + session: AsyncSession = Depends(get_session), +) -> dict[str, Any]: + try: + return await suggest_metrics_from_source( + session, + source_id=payload.source_id, + table_names=payload.table_names, + document_ids=payload.document_ids, + note=payload.note, + ) + except Exception as exc: # noqa: BLE001 + raise _raise_metric_error(exc) from exc diff --git a/backend/app/auth/chat_policy.py b/backend/app/auth/chat_policy.py index 03384d09..d08a815c 100644 --- a/backend/app/auth/chat_policy.py +++ b/backend/app/auth/chat_policy.py @@ -41,18 +41,19 @@ from typing import Any from app.analysis.contract import TOOL_NAMES from app.ledger.querycore import CALIBER_QUERY +from app.tools.metrics import METRIC_QUERY from app.tools.ontology import ONTOLOGY_TOOLS #: 问数据运行里 user 角色能用的节点类型（engine/schema.py 的 NodeType 取值） CHAT_NODE_TYPES = frozenset({"input", "llm", "agent", "report", "output", "retrieve", "transform", "branch", "metrics", "merge", "analysis", "tool"}) #: Agent 能绑的只读内置工具（tools/builtin/）：分析工具也在内（contract.TOOL_NAMES，不查库），以及口径查询 -#: （querycore.CALIBER_QUERY，只读、按资产授权收窄——见文件开头） +#: （querycore.CALIBER_QUERY，只读、按资产授权收窄——见文件开头）、独立语义指标查询（METRIC_QUERY，只读、经数据源遮罩） #: 和本体邻居/源健康探针（ONTOLOGY_TOOLS，只读、按授权与遮罩收窄）。工具名只从登记处取常量，不在这里另写一个字符串 CHAT_BUILTIN_TOOLS = frozenset({"calculator", "current_time", "json_query", "kb_search", "memory_recall", - CALIBER_QUERY, *ONTOLOGY_TOOLS, *TOOL_NAMES.values()}) + CALIBER_QUERY, METRIC_QUERY, *ONTOLOGY_TOOLS, *TOOL_NAMES.values()}) #: 上面那几个内置工具给人看的说法（拒绝时的报错、提示词里都写它） -CHAT_BUILTIN_LABEL = "计算、当前时间、JSON 取值、知识库检索、记忆读取、分析、口径查询、本体邻居、数据源健康" +CHAT_BUILTIN_LABEL = "计算、当前时间、JSON 取值、知识库检索、记忆读取、分析、口径查询、指标查询、本体邻居、数据源健康" #: 运行时改记忆范围、知识库的配置键（engine/nodes 的 ctx.cfg("memory_scope") / ctx.cfg("collection")，图级 defaults 同样生效） SCOPE_KEYS = ("memory_scope", "collection") SCOPE_REFUSED = "没有权限：使用者在问数据中不能指定记忆范围和知识库，使用的是管理员设好的默认值" diff --git a/backend/app/data/semantic_decompose.py b/backend/app/data/semantic_decompose.py new file mode 100644 index 00000000..ef4175d5 --- /dev/null +++ b/backend/app/data/semantic_decompose.py @@ -0,0 +1,1280 @@ +"""后台口径文档一键拆解与智能指标提取任务引擎（SemanticDecomposeJob）。 + +一次性打通业务文档（粘贴 Markdown、上传附件、选择知识库文档）与数据源目录的四层语义资产： +1. 知识库归档与判异红线留存（KB Document + guardrail_notes）：自动将粘贴或上传的口径总表存入知识库，保留物理红线、异常阈值与「暂无数据源」说明供问数检索。 +2. 字段目录补丁（catalog_patches）：提取表中文名、粒度、业务日期列、有效过滤条件，以及列级符号映射（如 ATTRIBUTE01 → 计划开班时长）、单位、流量或存量角色、存储陷阱备注；经 catalog.plan_patch 严格核验，并在内存里即时注入下一阶段！ +3. 单表与多表参数化指标口径（metric_candidates）：结合最新字段符号映射，同时提取单表聚合表达式（calc_mode="expression"）与跨表 JOIN / CTE / 动态周期天数参数化模板（calc_mode="sql_template"），逐条通过 AST 与只读守卫预编译核验。 +4. 标准业务实体与物理列编码映射（entity_changes）：提取产线、车间、模具、不良代码等共用维度实体及表列绑定，经 entity_patch.plan_entity_patch 核验。 +""" +from __future__ import annotations + +import asyncio +import contextlib +import copy +import json +import logging +import re +import time +from collections.abc import Mapping, Sequence +from datetime import timezone +from typing import Any + +from langchain_core.messages import HumanMessage, SystemMessage +from sqlalchemy import select, update +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.errors import describe_exception, first_line +from app.data import catalog, catalog_assistant, entity_docs, entity_patch +from app.data.engine import masked_columns +from app.data.entities import confirmed_bindings, list_entities +from app.data.names import name_key +from app.data.semantic_metrics import ( + SUGGEST_SYSTEM_PROMPT, + MetricConflict, + MetricInvalid, + MetricNotFound, + _normalize_metric_payload, + _parse_suggested_metrics, + _resolve_source, + create_metric, + list_metrics, + metric_to_dict, + update_metric, +) +from app.db.base import SessionLocal, utcnow +from app.db.models import DataSource, Document +from app.db.models_metrics import SemanticDecomposeJob, SemanticMetric +from app.engine.state import message_text +from app.memory import kb + +logger = logging.getLogger(__name__) + +OPEN_STATUSES = ("queued", "running") +VALID_SCOPES = ("catalog", "metrics", "entities") +MAX_DOC_CHARS = 32_000 +MAX_NOTE_CHARS = 1500 + +DECOMPOSE_BUSY_CODE = "decompose_job_busy" +DECOMPOSE_FINISHED_CODE = "decompose_job_finished" +DECOMPOSE_NOT_FOUND_CODE = "decompose_job_not_found" + +INTERRUPTED_ERROR = "服务重启，后台口径拆解任务中断，请重新发起" +JOB_FAILED_TEXT = "口径拆解任务执行失败：{reason}" + +_FENCE_RE = re.compile(r"```(?:json)?\s*([\s\S]*?)```", re.IGNORECASE) +_ACTIVE_TASKS: dict[str, asyncio.Task[Any]] = {} +_CREATE_LOCK = asyncio.Lock() +_KB_BG_TASKS: set[asyncio.Task[Any]] = set() + + +class DecomposeBusy(Exception): + """当前数据源已有正在执行的口径拆解任务。""" + + +class DecomposeFinished(Exception): + """要取消的口径拆解任务已经结束。""" + + +class DecomposeNotFound(LookupError): + """找不到指定的口径拆解任务。""" + + +CATALOG_DECOMPOSE_SYSTEM_PROMPT = """你是 AgentLab 的「数据目录与字段字典拆解专家」。请阅读数据源物理表结构、现有数据目录以及业务口径文档（如《指标公式总表》），提取需要补充或修正的「表目录与字段字典补丁」，同时摘录文档中的「物理判异红线与缺源提醒」。 + +硬性纪律： +1. 只输出一个合法 JSON 对象，格式为： + { + "tables": [ + { + "table": "物理表名（逐字取自输入表结构）", + "changes": [ + {"path": "路径", "value": "取值", "reason": "简明依据"} + ] + } + ], + "guardrail_notes": [ + {"title": "规则标题", "rule": "判异红线、存储陷阱或暂无数据源说明", "category": "redline | storage | missing_source"} + ] + } + 不要输出任何额外解释文字。 +2. 表名与列名必须 100% 逐字存在于输入提供的表结构中，严禁臆造不存在的表或列；标有「[遮罩列]」的敏感列严禁输出补丁。 +3. changes[].path 仅支持以下规范路径： + - 表级： + · "label"：表中文名称（纯名词短语，勿写公式） + · "grain"：一行代表什么业务粒度（如 "每条产线每个班次每天一行"） + · "business_date"：{"column": "物理日期列名", "rule": "统计归属说明（可空串）"} + · "valid_filter"：默认有效记录 SQL 过滤片段（如 "is_deleted = 0"，不含 WHERE） + - 列级（把 替换为真实物理列名，如 ATTRIBUTE01）： + · "columns..label"：字段中文名或符号含义（纯名称，如 "计划开班时长"，严禁包含 =、/、*、SUM( 等计算公式符号！） + · "columns..unit"：单位（如 "秒"、"%"、"件"、"人次"） + · "columns..measure"：仅限 "flow"（流量，可跨日加总，如时长、产量）、"stock"（存量，不可跨期直接求和）、"ratio"（比率，如良品率）、"identifier"（编码标识）、"status"（状态码）、"attribute"（属性） + · "columns..meaning"：关键业务含义与存储说明（如 "底表直接存百分数如 99.65，引用时须除以 100，切勿重复乘 100"；注意文字中不要写等号公式，用自然中文表述） + · "columns..codes"：枚举码值字典，如 {"1": "正常", "0": "停用"} +4. 特别留意工业宽表中的盲列（如 ATTRIBUTE01～ATTRIBUTE55）：凡文档给出了字段对照或符号说明的，务必逐列提取 label、unit、measure 与 meaning！ +""" + +ENTITY_DECOMPOSE_SYSTEM_PROMPT = """你是 AgentLab 的「标准业务实体拆解专家」。请结合数据源物理表结构（含最新字段说明）与业务口径文档，识别多张表或业务提问中共用的核心维度实体（如产线、车间、机台、产品型号、不良原因、区域等），输出增量操作 JSON。 + +硬性纪律： +1. 只输出一个合法 JSON 对象，格式为 {"operations": [...]}，不要输出任何额外解释文字。 +2. 表名、列名、数据源名必须 100% 逐字存在于输入材料中，严禁编造；纯自增流水主键或纯度量数值列不要建为业务实体；标有「[遮罩列]」的敏感列严禁绑定或提取成员。 +3. 若没有明确可建的标准业务实体，返回 {"operations": []}。 +4. operations 支持以下结构（新建实体时请先输出一条 upsert_entity，再输出其 upsert_binding / upsert_member）： + - {"op": "upsert_entity", "entity_key": "小写英文下划线标识，如 production_line", "label": "产线", "description": "制造产线标准维度", "aliases": ["生产线", "线体"], "reason": "依据"} + - {"op": "upsert_binding", "entity_key": "production_line", "entity_label": "产线", "source_name": "数据源名称", "table_name": "物理表名", "column_name": "物理列名", "note": "产线编码列", "reason": "依据"} + - {"op": "upsert_member", "entity_key": "production_line", "entity_label": "产线", "value": "L1", "label": "一号产线", "aliases": ["1线"], "source_codes": {"数据源名称": "L1"}, "reason": "依据"} +""" + + +def _iso(val: Any) -> str | None: + if val is None or not hasattr(val, "isoformat"): + return None + if getattr(val, "tzinfo", None) is None: + val = val.replace(tzinfo=timezone.utc) + return val.isoformat() + + +def _parse_json_obj(raw_text: str) -> dict[str, Any]: + text = (raw_text or "").strip() + if not text: + return {} + m = _FENCE_RE.search(text) + candidate = m.group(1).strip() if m else text + for snippet in (candidate, text): + start = snippet.find("{") + end = snippet.rfind("}") + if start >= 0 and end > start: + try: + obj = json.loads(snippet[start : end + 1]) + if isinstance(obj, Mapping): + return dict(obj) + except ValueError: + continue + return {} + + +def _normalize_scopes(raw_scopes: Sequence[str] | None, *, has_docs: bool) -> list[str]: + if not raw_scopes: + return ["catalog", "metrics", "entities"] if has_docs else ["metrics"] + out: list[str] = [] + for s in raw_scopes: + clean = str(s or "").strip().lower() + if clean in VALID_SCOPES and clean not in out: + out.append(clean) + return out or (["catalog", "metrics", "entities"] if has_docs else ["metrics"]) + + +def _initial_steps(scopes: Sequence[str], *, has_docs: bool) -> list[dict[str, Any]]: + steps: list[dict[str, Any]] = [ + { + "key": "prepare", + "label": "准备表结构并归档口径文档" if has_docs else "读取数据源表结构与目录上下文", + "status": "pending", + "detail": "等待执行", + "count": 0, + "elapsed_ms": 0, + } + ] + if "catalog" in scopes: + steps.append( + { + "key": "catalog", + "label": "拆解字段符号映射与表目录补丁", + "status": "pending", + "detail": "等待执行", + "count": 0, + "elapsed_ms": 0, + } + ) + if "metrics" in scopes: + steps.append( + { + "key": "metrics", + "label": "生成单表表达式与多表模板指标口径", + "status": "pending", + "detail": "等待执行", + "count": 0, + "elapsed_ms": 0, + } + ) + if "entities" in scopes: + steps.append( + { + "key": "entities", + "label": "识别标准业务实体与物理列映射", + "status": "pending", + "detail": "等待执行", + "count": 0, + "elapsed_ms": 0, + } + ) + return steps + + +def job_to_dict(row: SemanticDecomposeJob, *, include_result: bool = True) -> dict[str, Any]: + data: dict[str, Any] = { + "id": row.id, + "status": row.status, + "source_id": row.source_id, + "source_name": row.source_name or "", + "scopes": list(row.scopes or []), + "document_ids": list(row.document_ids or []), + "attachment_ids": list(row.attachment_ids or []), + "doc_title": row.doc_title or "", + "has_doc_text": bool((row.doc_text or "").strip()), + "save_to_kb": bool(row.save_to_kb), + "note": row.note or "", + "phase": row.phase or "", + "phase_label": row.phase_label or "", + "progress": int(row.progress or 0), + "steps": list(row.steps or []), + "summary": dict(row.summary or {}), + "error": row.error, + "started_by": row.started_by, + "started_at": _iso(row.started_at), + "finished_at": _iso(row.finished_at), + "created_at": _iso(row.created_at), + "updated_at": _iso(row.updated_at), + } + if include_result: + data["result"] = dict(row.result or {}) + return data + + +async def get_job(session: AsyncSession, job_id: str) -> SemanticDecomposeJob: + clean = str(job_id or "").strip() + if not clean: + raise DecomposeNotFound("未指定口径拆解任务编号") + row = await session.get(SemanticDecomposeJob, clean) + if row is None: + raise DecomposeNotFound(f"找不到口径拆解任务「{clean}」") + return row + + +async def get_active_or_latest_job( + session: AsyncSession, + *, + source_id: str | None = None, +) -> dict[str, Any]: + """返回正在排队或运行中的任务，以及最近一次已完成任务（供口径台账页顶部横幅轮询与恢复审阅）。""" + active_stmt = select(SemanticDecomposeJob).where(SemanticDecomposeJob.status.in_(OPEN_STATUSES)) + latest_stmt = select(SemanticDecomposeJob) + if source_id and source_id.strip(): + sid = source_id.strip() + active_stmt = active_stmt.where(SemanticDecomposeJob.source_id == sid) + latest_stmt = latest_stmt.where(SemanticDecomposeJob.source_id == sid) + active_row = ( + await session.execute(active_stmt.order_by(SemanticDecomposeJob.created_at.desc()).limit(1)) + ).scalars().first() + latest_row = ( + await session.execute(latest_stmt.order_by(SemanticDecomposeJob.created_at.desc()).limit(1)) + ).scalars().first() + return { + "active": job_to_dict(active_row, include_result=False) if active_row else None, + "latest": job_to_dict(latest_row, include_result=True) if latest_row else None, + } + + +async def list_jobs( + session: AsyncSession, + *, + source_id: str | None = None, + limit: int = 10, +) -> list[dict[str, Any]]: + stmt = select(SemanticDecomposeJob).order_by(SemanticDecomposeJob.created_at.desc()).limit(max(1, min(limit, 30))) + if source_id and source_id.strip(): + stmt = stmt.where(SemanticDecomposeJob.source_id == source_id.strip()) + rows = list((await session.execute(stmt)).scalars().all()) + return [job_to_dict(r, include_result=False) for r in rows] + + +async def create_decompose_job( + session: AsyncSession, + *, + source_id: str, + scopes: Sequence[str] | None = None, + table_names: Sequence[str] | None = None, + document_ids: Sequence[str] | None = None, + attachment_ids: Sequence[str] | None = None, + doc_title: str = "", + doc_text: str = "", + save_to_kb: bool = True, + note: str = "", + actor: str | None = None, +) -> SemanticDecomposeJob: + """创建并启动后台口径文档一键拆解或智能提取指标任务。""" + ds = await _resolve_source(session, source_id) + cache = ds.schema_cache if isinstance(ds.schema_cache, Mapping) else {} + tables = cache.get("tables") if isinstance(cache, Mapping) else None + if not isinstance(tables, Mapping) or not tables: + raise MetricInvalid(f"数据源「{ds.name}」尚未探查表结构，请先在数据源设置中完成探查", field="source_id") + + clean_docs = [str(d).strip() for d in (document_ids or []) if str(d or "").strip()] + clean_atts = [str(a).strip() for a in (attachment_ids or []) if str(a or "").strip()] + clean_text = (doc_text or "").strip() + clean_title = (doc_title or "").strip()[:240] + has_docs = bool(clean_docs or clean_atts or clean_text) + norm_scopes = _normalize_scopes(scopes, has_docs=has_docs) + clean_tables = [str(t).strip() for t in (table_names or []) if str(t or "").strip()] + + async with _CREATE_LOCK: + existing_open = ( + await session.execute( + select(SemanticDecomposeJob).where( + SemanticDecomposeJob.source_id == ds.id, + SemanticDecomposeJob.status.in_(OPEN_STATUSES), + ) + ) + ).scalars().first() + if existing_open is not None: + raise DecomposeBusy(f"数据源「{ds.name}」已有正在后台执行的拆解任务，请等待完成或先取消当前任务") + + job = SemanticDecomposeJob( + status="queued", + source_id=ds.id, + source_name=ds.name, + scopes=norm_scopes, + document_ids=clean_docs, + attachment_ids=clean_atts, + doc_title=clean_title, + doc_text=clean_text, + save_to_kb=bool(save_to_kb), + note=(note or "").strip()[:MAX_NOTE_CHARS], + phase="queued", + phase_label="已进入后台队列，准备读取数据源与文档", + progress=4, + steps=_initial_steps(norm_scopes, has_docs=has_docs), + result={"table_names": clean_tables} if clean_tables else {}, + summary={}, + started_by=actor, + started_at=utcnow(), + ) + session.add(job) + await session.commit() + await session.refresh(job) + + task = asyncio.create_task(_execute_job(job.id), name=f"semantic-decompose-{job.id}") + _ACTIVE_TASKS[job.id] = task + task.add_done_callback(lambda _t, jid=job.id: _ACTIVE_TASKS.pop(jid, None)) + return job + + +def _update_step_list( + steps: list[dict[str, Any]], + step_key: str, + *, + status: str, + detail: str | None = None, + count: int | None = None, + elapsed_ms: int | None = None, +) -> list[dict[str, Any]]: + updated = [copy.deepcopy(s) for s in steps] + for s in updated: + if s.get("key") == step_key: + s["status"] = status + if detail is not None: + s["detail"] = detail + if count is not None: + s["count"] = count + if elapsed_ms is not None: + s["elapsed_ms"] = elapsed_ms + return updated + + +async def _commit_progress( + session: AsyncSession, + job: SemanticDecomposeJob, + *, + phase: str, + phase_label: str, + progress: int, + steps: list[dict[str, Any]], +) -> None: + job.status = "running" + job.phase = phase + job.phase_label = phase_label + job.progress = max(0, min(99, int(progress))) + job.steps = steps + await session.commit() + + +def _infer_doc_title(doc_title: str, doc_text: str, source_name: str) -> str: + if doc_title.strip(): + return doc_title.strip()[:200] + for line in doc_text.splitlines(): + clean = line.strip().lstrip("#").strip() + if clean: + return clean[:120] + return f"{source_name} 业务指标口径文档" + + +def _build_schema_prompt_block( + ds: DataSource, + tables: Mapping[str, Any], + selected_tables: Sequence[str], + entries: Mapping[str, Any], + masked_set: set[str], + col_entity: Mapping[tuple[str, str], str], + overlay_notes: Mapping[str, Mapping[str, Any]] | None = None, +) -> str: + lines: list[str] = [f"数据源：{ds.name}（方言：{ds.kind}）"] + if (ds.kind or "").strip().lower() == "oracle": + lines.append("方言注意：Oracle 严禁写 LIMIT、严禁写 CAST(x AS TEXT)、表或子查询别名不写 AS；动态周期天数请直接用 {{period_days}} 槽位。") + lines.append("\n【物理表结构与数据目录】") + for tname in selected_tables: + tinfo = tables.get(tname) or {} + base_notes = catalog.visible_notes(entries[tname].notes) if tname in entries else {} + notes = copy.deepcopy(dict(base_notes)) + if overlay_notes and tname in overlay_notes: + ov = overlay_notes[tname] + for k, v in ov.items(): + if k == "columns" and isinstance(v, Mapping): + cols_dict = dict(notes.get("columns") or {}) if isinstance(notes.get("columns"), Mapping) else {} + for ck, cv in v.items(): + prev_c = dict(cols_dict.get(ck) or {}) if isinstance(cols_dict.get(ck), Mapping) else {} + if isinstance(cv, Mapping): + prev_c.update(cv) + cols_dict[ck] = prev_c + notes["columns"] = cols_dict + else: + notes[k] = v + + t_label = (notes.get("label") or {}).get("value") if isinstance(notes.get("label"), Mapping) else "" + t_grain = (notes.get("grain") or {}).get("value") if isinstance(notes.get("grain"), Mapping) else "" + b_date_raw = (notes.get("business_date") or {}).get("value") if isinstance(notes.get("business_date"), Mapping) else "" + if isinstance(b_date_raw, Mapping): + b_col = str(b_date_raw.get("column") or "").strip() + b_rule = str(b_date_raw.get("rule") or "").strip() + b_date = f"{b_col}（{b_rule}）" if (b_col and b_rule) else (b_col or b_rule) + else: + b_date = str(b_date_raw or "").strip() + v_filt = (notes.get("valid_filter") or {}).get("value") if isinstance(notes.get("valid_filter"), Mapping) else "" + + head_bits = [f"表 {tname}"] + qualified_t = str(tinfo.get("qualified") or "").strip() + if qualified_t and qualified_t != tname: + head_bits.append(f"SQL 全名：{qualified_t}") + if t_label: + head_bits.append(f"中文名：{t_label}") + if t_grain: + head_bits.append(f"粒度：{t_grain}") + if b_date: + head_bits.append(f"业务日期列：{b_date}") + if v_filt: + head_bits.append(f"有效过滤：{v_filt}") + lines.append("- " + " ｜ ".join(head_bits)) + + col_notes = notes.get("columns") if isinstance(notes.get("columns"), Mapping) else {} + t_short = name_key(tname.rsplit(".", 1)[-1]) + for col in (tinfo.get("columns") or [])[:75]: + if not isinstance(col, Mapping) or not col.get("name"): + continue + cname = str(col["name"]) + ctype = str(col.get("type") or "") + if name_key(cname) in masked_set: + lines.append(f" · {cname} ({ctype}) [遮罩列]") + continue + cn = col_notes.get(cname) if isinstance(col_notes, Mapping) else None + c_label = (cn.get("label") or {}).get("value") if isinstance(cn, Mapping) and isinstance(cn.get("label"), Mapping) else "" + c_unit = (cn.get("unit") or {}).get("value") if isinstance(cn, Mapping) and isinstance(cn.get("unit"), Mapping) else "" + c_measure = (cn.get("measure") or {}).get("value") if isinstance(cn, Mapping) and isinstance(cn.get("measure"), Mapping) else "" + c_meaning = ( + (cn.get("meaning") or {}).get("value") + if isinstance(cn, Mapping) and isinstance(cn.get("meaning"), Mapping) + else ((cn.get("note") or {}).get("value") if isinstance(cn, Mapping) and isinstance(cn.get("note"), Mapping) else "") + ) + ent_key = col_entity.get((t_short, name_key(cname)), "") + extra_bits = [ + b + for b in ( + c_label, + f"单位：{c_unit}" if c_unit else "", + f"度量：{c_measure}" if c_measure else "", + f"含义：{c_meaning}" if c_meaning else "", + f"已绑定标准实体：{ent_key}" if ent_key else "", + ) + if b + ] + suffix = f" — {'，'.join(extra_bits)}" if extra_bits else "" + lines.append(f" · {cname} ({ctype}){suffix}") + return "\n".join(lines) + + +def _align_catalog_changes_for_table( + raw_changes: Sequence[Any], + *, + tinfo: Mapping[str, Any], + masked_set: set[str], +) -> list[dict[str, Any]]: + """对齐列路径大小写，过滤敏感遮罩列，避免模型把 ATTRIBUTE01 写成 attribute01 导致落空。""" + col_exact_by_key: dict[str, str] = {} + for c in tinfo.get("columns") or []: + if isinstance(c, Mapping) and c.get("name"): + cname = str(c["name"]) + col_exact_by_key[name_key(cname)] = cname + + aligned: list[dict[str, Any]] = [] + for item in raw_changes or []: + if not isinstance(item, Mapping): + continue + raw_path = str(item.get("path") or "").strip() + if not raw_path or "value" not in item: + continue + val = copy.deepcopy(item["value"]) + reason = str(item.get("reason") or "").strip() + + if raw_path.startswith("columns."): + parts = raw_path.split(".") + if len(parts) == 3: + raw_col, slot_field = parts[1], parts[2] + if name_key(raw_col) in masked_set: + continue + if slot_field in ("note", "description"): + slot_field = "meaning" + exact_col = col_exact_by_key.get(name_key(raw_col), raw_col) + raw_path = f"columns.{exact_col}.{slot_field}" + # 若模型不小心在 label 里写了等号，自动清洗掉等号右侧以免被 _formula_like 整条丢弃 + if slot_field == "label" and isinstance(val, str) and "=" in val: + val = val.split("=", 1)[0].strip() + elif raw_path == "business_date" and isinstance(val, Mapping) and val.get("column"): + bcol = str(val["column"]) + if name_key(bcol) in masked_set: + continue + val = {**dict(val), "column": col_exact_by_key.get(name_key(bcol), bcol)} + elif raw_path == "business_date" and isinstance(val, str) and val.strip(): + bcol = val.strip() + if name_key(bcol) in masked_set: + continue + val = {"column": col_exact_by_key.get(name_key(bcol), bcol), "rule": ""} + + aligned.append({"path": raw_path, "value": val, "reason": reason}) + return aligned + + +async def _execute_job(job_id: str) -> None: + try: + async with SessionLocal() as session: + job = await session.get(SemanticDecomposeJob, job_id) + if job is None or job.status not in OPEN_STATUSES: + return + + steps = list(job.steps or []) + scopes = list(job.scopes or ["metrics"]) + requested_tables = list((job.result or {}).get("table_names") or []) + + # ---------------- 阶段 1：准备表结构与归档知识库 ---------------- + t_step = time.perf_counter() + steps = _update_step_list(steps, "prepare", status="running", detail="正在加载数据源表结构与口径文档") + await _commit_progress( + session, + job, + phase="preparing", + phase_label="正在加载数据源结构并整理业务口径文档…", + progress=12, + steps=steps, + ) + + ds = await _resolve_source(session, job.source_id) + cache = ds.schema_cache if isinstance(ds.schema_cache, Mapping) else {} + tables = cache.get("tables") if isinstance(cache, Mapping) else {} + if not isinstance(tables, Mapping) or not tables: + raise MetricInvalid(f"数据源「{ds.name}」尚未探查表结构") + + entries = await catalog.read_catalog(session, ds.id) + masked_set = {name_key(c) for c in masked_columns(ds.options)} + entities_list = await list_entities(session) + existing_metrics = await list_metrics(session, source_id=ds.id) + existing_metric_keys = {m.key for m in existing_metrics} + + col_entity: dict[tuple[str, str], str] = {} + for ent in entities_list: + for b in confirmed_bindings(ent): + if name_key(str(b.get("source_name") or "")) == name_key(ds.name): + t_short = name_key(str(b.get("table_name") or "").rsplit(".", 1)[-1]) + c_key = name_key(str(b.get("column_name") or "")) + col_entity[(t_short, c_key)] = ent.key + + selected_tables: list[str] = [] + if requested_tables: + for raw_t in requested_tables: + resolved = catalog.resolve_table_name(cache, str(raw_t)) + if resolved and resolved not in selected_tables: + selected_tables.append(resolved) + if not selected_tables: + def _table_priority(tname: str) -> tuple[int, str]: + has_cat = 0 if (tname in entries and catalog.visible_notes(entries[tname].notes)) else 1 + return (has_cat, tname) + + selected_tables = sorted(tables.keys(), key=_table_priority)[:32] + + # 组装文档内容，并按需把粘贴文本归档入知识库 + kb_documents: list[dict[str, Any]] = [] + doc_sections: list[str] = [] + doc_budget = MAX_DOC_CHARS + + pasted_text = (job.doc_text or "").strip() + if pasted_text: + inferred_title = _infer_doc_title(job.doc_title or "", pasted_text, ds.name) + created_doc_id: str | None = None + if job.save_to_kb: + existing_doc = ( + await session.execute( + select(Document).where(Document.title == inferred_title).order_by(Document.created_at.desc()).limit(1) + ) + ).scalars().first() + if existing_doc is not None and (existing_doc.content or "").strip() == pasted_text: + created_doc_id = existing_doc.id + kb_documents.append({"id": existing_doc.id, "title": inferred_title, "created_now": False}) + else: + new_doc = await kb.create_document( + session, + collection="default", + title=inferred_title, + content=pasted_text, + source=inferred_title, + mime="text/markdown", + meta={"origin": "semantic_decompose", "source_id": ds.id}, + status="processing", + ) + await session.flush() + try: + await kb.process_document(session, new_doc) + except Exception as kb_exc: # noqa: BLE001 + logger.warning("知识库文档切块索引降级（%s）：%s", new_doc.id, kb_exc) + new_doc.status = "ready" + created_doc_id = new_doc.id + kb_documents.append({"id": new_doc.id, "title": inferred_title, "created_now": True}) + + snippet = pasted_text[: min(len(pasted_text), doc_budget)] + doc_budget -= len(snippet) + doc_sections.append(f"--- 文档《{inferred_title}》 ---\n{snippet}") + if created_doc_id and created_doc_id not in (job.document_ids or []): + job.document_ids = [*(job.document_ids or []), created_doc_id] + + bundles = await entity_docs.load_doc_bundles( + session, + attachment_ids=job.attachment_ids or [], + document_ids=job.document_ids or [], + ) + seen_kb_ids = {d["id"] for d in kb_documents} + for bundle in bundles: + if bundle.document_id and bundle.document_id in seen_kb_ids: + continue + joined = "\n\n".join(bundle.chunks).strip() + if not joined or doc_budget <= 0: + continue + if bundle.kind == "attachment" and job.save_to_kb and not bundle.document_id: + att_doc = await kb.create_document( + session, + collection="default", + title=bundle.title, + content=joined, + source=bundle.title, + mime=bundle.mime, + meta={"origin": "semantic_decompose", "source_id": ds.id}, + status="processing", + ) + await session.flush() + try: + await kb.process_document(session, att_doc) + except Exception as kb_exc: # noqa: BLE001 + logger.warning("附件文档切块索引降级（%s）：%s", att_doc.id, kb_exc) + att_doc.status = "ready" + kb_documents.append({"id": att_doc.id, "title": bundle.title, "created_now": True}) + seen_kb_ids.add(att_doc.id) + elif bundle.document_id and bundle.document_id not in seen_kb_ids: + kb_documents.append({"id": bundle.document_id, "title": bundle.title, "created_now": False}) + seen_kb_ids.add(bundle.document_id) + + snippet = joined[: min(len(joined), doc_budget)] + doc_budget -= len(snippet) + doc_sections.append(f"--- 文档《{bundle.title}》 ---\n{snippet}") + + prep_detail = f"已扫描 {len(selected_tables)} 张物理表" + if doc_sections: + prep_detail += f"，载入 {len(doc_sections)} 份口径文档" + if any(d.get("created_now") for d in kb_documents): + prep_detail += "并同步归档至知识库" + steps = _update_step_list( + steps, + "prepare", + status="succeeded", + detail=prep_detail, + count=len(selected_tables), + elapsed_ms=int((time.perf_counter() - t_step) * 1000), + ) + await _commit_progress( + session, + job, + phase="prepared", + phase_label=prep_detail, + progress=22, + steps=steps, + ) + + model, model_id = await catalog_assistant.resolve_model(session) + + # ---------------- 阶段 2：拆解字段符号映射与表目录补丁 ---------------- + catalog_patches: list[dict[str, Any]] = [] + catalog_skipped: list[str] = [] + guardrail_notes: list[dict[str, str]] = [] + overlay_notes: dict[str, dict[str, Any]] = {} + + if "catalog" in scopes: + t_cat = time.perf_counter() + if not doc_sections and not (job.note or "").strip(): + steps = _update_step_list( + steps, + "catalog", + status="skipped", + detail="未提供业务口径文档，跳过字段目录拆解", + count=0, + elapsed_ms=0, + ) + else: + steps = _update_step_list( + steps, + "catalog", + status="running", + detail="正在识别字段符号映射、单位、流量或存量角色与存储陷阱", + ) + await _commit_progress( + session, + job, + phase="extracting_catalog", + phase_label="正在拆解字段符号映射与数据目录补丁（1/3）…", + progress=34, + steps=steps, + ) + schema_block = _build_schema_prompt_block( + ds, tables, selected_tables, entries, masked_set, col_entity + ) + cat_prompt_parts = [schema_block] + if doc_sections: + cat_prompt_parts.append("\n【业务口径文档】\n" + "\n\n".join(doc_sections)) + if job.note: + cat_prompt_parts.append(f"\n【用户补充要求】\n{job.note}") + + cat_reply = await model.ainvoke( + [ + SystemMessage(content=CATALOG_DECOMPOSE_SYSTEM_PROMPT), + HumanMessage(content="\n".join(cat_prompt_parts)), + ] + ) + cat_json = _parse_json_obj(message_text(cat_reply)) + + for g in cat_json.get("guardrail_notes") or []: + if isinstance(g, Mapping) and str(g.get("rule") or "").strip(): + guardrail_notes.append( + { + "title": str(g.get("title") or "口径规则提醒").strip()[:120], + "rule": str(g.get("rule") or "").strip()[:600], + "category": str(g.get("category") or "redline").strip()[:32], + } + ) + + total_cat_changes = 0 + for tbl_item in cat_json.get("tables") or []: + if not isinstance(tbl_item, Mapping): + continue + raw_tbl = str(tbl_item.get("table") or "").strip() + resolved_tbl = catalog.resolve_table_name(cache, raw_tbl) + if not resolved_tbl: + if raw_tbl: + catalog_skipped.append(f"跳过不存在的表「{raw_tbl}」") + continue + tinfo = tables.get(resolved_tbl) or {} + aligned_changes = _align_catalog_changes_for_table( + tbl_item.get("changes") or [], + tinfo=tinfo, + masked_set=masked_set, + ) + if not aligned_changes: + continue + entry = entries.get(resolved_tbl) + plan = catalog.plan_patch( + entry.notes if entry else {}, + aligned_changes, + table=resolved_tbl, + tables=tables, + ) + catalog_skipped.extend(f"{resolved_tbl}：{p}" for p in plan.problems) + active_changes = [c for c in plan.changes if c.state != "same"] + if not active_changes: + continue + + # 将通过核验的目录补丁写入内存覆盖层，直接供给下一阶段指标提取！ + tbl_ov = overlay_notes.setdefault(resolved_tbl, {"columns": {}}) + serialized_changes: list[dict[str, Any]] = [] + for c_idx, ch in enumerate(active_changes): + slot = catalog.parse_path(ch.path) + slot_label = catalog.describe_slot(slot) if slot[0] != "r" else "关联关系" + serialized_changes.append( + { + "id": f"{resolved_tbl}:{ch.path}:{c_idx}", + "table": resolved_tbl, + "path": ch.path, + "slot_label": slot_label, + "before": ch.before, + "after": ch.after, + "value": ch.value, + "reason": ch.reason, + "state": ch.state, + } + ) + if slot[0] == "t": + tbl_ov[slot[1]] = {"value": ch.after} + elif slot[0] == "c": + col_ov = tbl_ov["columns"].setdefault(slot[1], {}) + col_ov[slot[2]] = {"value": ch.after} + + total_cat_changes += len(serialized_changes) + catalog_patches.append( + { + "table": resolved_tbl, + "if_version": int(entry.version) if entry else 0, + "changes": serialized_changes, + } + ) + + steps = _update_step_list( + steps, + "catalog", + status="succeeded", + detail=f"提取出 {total_cat_changes} 项字段与表目录补丁" + + (f"、{len(guardrail_notes)} 条判异红线" if guardrail_notes else ""), + count=total_cat_changes, + elapsed_ms=int((time.perf_counter() - t_cat) * 1000), + ) + await _commit_progress( + session, + job, + phase="catalog_done", + phase_label="字段目录拆解完成，准备提取语义指标口径…", + progress=52, + steps=steps, + ) + + # ---------------- 阶段 3：提取单表与多表参数化指标口径 ---------------- + metric_candidates: list[dict[str, Any]] = [] + metric_skipped: list[dict[str, str]] = [] + + if "metrics" in scopes: + t_met = time.perf_counter() + steps = _update_step_list( + steps, + "metrics", + status="running", + detail="正在结合字段目录映射生成单表聚合与多表模板指标并预编译校验", + ) + await _commit_progress( + session, + job, + phase="extracting_metrics", + phase_label="正在生成单表与多表参数化 SQL 指标口径并逐条校验…", + progress=64, + steps=steps, + ) + + enriched_schema_block = _build_schema_prompt_block( + ds, + tables, + selected_tables, + entries, + masked_set, + col_entity, + overlay_notes=overlay_notes, + ) + met_lines = [enriched_schema_block] + if existing_metric_keys: + met_lines.append(f"\n已存在的指标标识（若口径需要升级可沿用并更新，否则请勿无故重复）：{', '.join(sorted(existing_metric_keys))}") + if doc_sections: + met_lines.append("\n【参考业务口径文档】\n" + "\n\n".join(doc_sections)) + if job.note: + met_lines.append(f"\n【用户补充要求】\n{job.note}") + + met_reply = await model.ainvoke( + [ + SystemMessage(content=SUGGEST_SYSTEM_PROMPT), + HumanMessage(content="\n".join(met_lines)), + ] + ) + raw_candidates = _parse_suggested_metrics(message_text(met_reply)) + seen_batch_keys: set[str] = set() + for raw in raw_candidates: + candidate = {**raw, "source_id": ds.id, "source": "ai", "status": "enabled"} + raw_k = str(candidate.get("key") or "").strip().lower() + if raw_k and raw_k in seen_batch_keys: + metric_skipped.append({"key": raw_k, "reason": "同批候选指标标识重复"}) + continue + try: + _, norm = await _normalize_metric_payload(session, candidate, existing=None) + seen_batch_keys.add(norm["key"]) + metric_candidates.append( + { + **norm, + "source_name": ds.name, + "exists_in_db": norm["key"] in existing_metric_keys, + } + ) + except Exception as exc: # noqa: BLE001 + metric_skipped.append( + {"key": raw_k or str(candidate.get("name") or "未命名指标"), "reason": str(exc)} + ) + + tpl_cnt = sum(1 for m in metric_candidates if m.get("calc_mode") == "sql_template") + expr_cnt = len(metric_candidates) - tpl_cnt + met_detail = f"核验通过 {len(metric_candidates)} 个指标（单表 {expr_cnt} 个、多表模板 {tpl_cnt} 个）" + steps = _update_step_list( + steps, + "metrics", + status="succeeded", + detail=met_detail, + count=len(metric_candidates), + elapsed_ms=int((time.perf_counter() - t_met) * 1000), + ) + await _commit_progress( + session, + job, + phase="metrics_done", + phase_label=met_detail, + progress=82, + steps=steps, + ) + + # ---------------- 阶段 4：识别标准业务实体与编码映射 ---------------- + entity_changes: list[dict[str, Any]] = [] + entity_skipped: list[str] = [] + + if "entities" in scopes: + t_ent = time.perf_counter() + steps = _update_step_list( + steps, + "entities", + status="running", + detail="正在识别共用业务实体维度与表列绑定", + ) + await _commit_progress( + session, + job, + phase="extracting_entities", + phase_label="正在识别标准业务实体与物理列编码映射…", + progress=88, + steps=steps, + ) + enriched_schema_block = _build_schema_prompt_block( + ds, + tables, + selected_tables, + entries, + masked_set, + col_entity, + overlay_notes=overlay_notes, + ) + ent_lines = [enriched_schema_block] + if entities_list: + ent_summary = "、".join(f"{e.key}({e.label})" for e in entities_list[:30]) + ent_lines.append(f"\n已有标准业务实体：{ent_summary}") + if doc_sections: + ent_lines.append("\n【业务口径文档】\n" + "\n\n".join(doc_sections)) + if job.note: + ent_lines.append(f"\n【用户补充要求】\n{job.note}") + + ent_reply = await model.ainvoke( + [ + SystemMessage(content=ENTITY_DECOMPOSE_SYSTEM_PROMPT), + HumanMessage(content="\n".join(ent_lines)), + ] + ) + ent_json = _parse_json_obj(message_text(ent_reply)) + raw_ops = ent_json.get("operations") if isinstance(ent_json.get("operations"), list) else [] + + safe_ops: list[dict[str, Any]] = [] + for op_item in raw_ops: + if not isinstance(op_item, Mapping): + continue + op_copy = dict(op_item) + if op_copy.get("op") == "upsert_binding": + if not op_copy.get("source_name"): + op_copy["source_name"] = ds.name + col_candidate = str(op_copy.get("column_name") or "").strip() + if name_key(col_candidate) in masked_set: + entity_skipped.append(f"跳过敏感遮罩列绑定「{col_candidate}」") + continue + safe_ops.append(op_copy) + + if safe_ops: + ent_plan = await entity_patch.plan_entity_patch(session, safe_ops, mode="confirmed") + entity_changes = [ch.as_dict() for ch in ent_plan.changes] + entity_skipped.extend(ent_plan.problems) + + steps = _update_step_list( + steps, + "entities", + status="succeeded", + detail=f"识别出 {len(entity_changes)} 项标准实体与绑定变更", + count=len(entity_changes), + elapsed_ms=int((time.perf_counter() - t_ent) * 1000), + ) + + # ---------------- 完成汇总 ---------------- + catalog_change_count = sum(len(t.get("changes") or []) for t in catalog_patches) + summary = { + "model_id": model_id, + "kb_synced_count": sum(1 for d in kb_documents if d.get("created_now")), + "kb_doc_count": len(kb_documents), + "catalog_table_count": len(catalog_patches), + "catalog_change_count": catalog_change_count, + "metric_count": len(metric_candidates), + "metric_template_count": sum(1 for m in metric_candidates if m.get("calc_mode") == "sql_template"), + "metric_skipped_count": len(metric_skipped), + "entity_change_count": len(entity_changes), + "guardrail_count": len(guardrail_notes), + } + job.status = "succeeded" + job.phase = "done" + job.phase_label = ( + f"拆解完成：{len(metric_candidates)} 个指标" + + (f"、{catalog_change_count} 项字段目录补丁" if catalog_change_count else "") + + (f"、{len(entity_changes)} 项实体映射" if entity_changes else "") + ) + job.progress = 100 + job.steps = steps + job.summary = summary + job.result = { + "model_id": model_id, + "kb_documents": kb_documents, + "guardrail_notes": guardrail_notes, + "catalog_patches": catalog_patches, + "catalog_skipped": catalog_skipped, + "metric_candidates": metric_candidates, + "metric_skipped": metric_skipped, + "entity_changes": entity_changes, + "entity_skipped": entity_skipped, + "applied": None, + } + job.finished_at = utcnow() + await session.commit() + + except asyncio.CancelledError: + async with SessionLocal() as session: + job = await session.get(SemanticDecomposeJob, job_id) + if job is not None and job.status in OPEN_STATUSES: + job.status = "cancelled" + job.phase = "cancelled" + job.phase_label = "已取消后台拆解任务" + job.finished_at = utcnow() + await session.commit() + raise + except Exception as exc: # noqa: BLE001 + logger.warning("口径拆解任务执行失败（%s）：%s", job_id, exc, exc_info=True) + async with SessionLocal() as session: + job = await session.get(SemanticDecomposeJob, job_id) + if job is not None and job.status in OPEN_STATUSES: + steps = [ + {**s, "status": "failed", "detail": first_line(exc) or describe_exception(exc)} + if s.get("status") == "running" + else s + for s in (job.steps or []) + ] + job.status = "failed" + job.phase = "failed" + job.phase_label = "拆解任务执行失败" + job.steps = steps + job.error = JOB_FAILED_TEXT.format(reason=first_line(exc) or describe_exception(exc)) + job.finished_at = utcnow() + await session.commit() + + +async def cancel_decompose_job(session: AsyncSession, job_id: str) -> SemanticDecomposeJob: + job = await get_job(session, job_id) + if job.status not in OPEN_STATUSES: + raise DecomposeFinished("该口径拆解任务已经结束") + task = _ACTIVE_TASKS.get(job.id) + if task is not None and not task.done(): + task.cancel() + with contextlib.suppress(asyncio.CancelledError, Exception): + await asyncio.wait_for(asyncio.shield(task), timeout=2.0) + await session.refresh(job) + if job.status in OPEN_STATUSES: + job.status = "cancelled" + job.phase = "cancelled" + job.phase_label = "已取消后台拆解任务" + job.finished_at = utcnow() + await session.commit() + await session.refresh(job) + return job + + +async def apply_decompose_job( + session: AsyncSession, + job_id: str, + *, + catalog_patches: Sequence[Mapping[str, Any]] | None = None, + metrics: Sequence[Mapping[str, Any]] | None = None, + entity_operations: Sequence[Mapping[str, Any]] | None = None, + overwrite_existing_metrics: bool = True, + actor: str | None = None, +) -> dict[str, Any]: + """一键应用用户在审阅抽屉中勾选的「字段目录补丁 + 语义指标口径 + 标准业务实体」。""" + job = await get_job(session, job_id) + if job.status != "succeeded": + raise MetricInvalid("仅已完成的拆解任务支持一键应用") + + ds = await _resolve_source(session, job.source_id) + cache = ds.schema_cache if isinstance(ds.schema_cache, Mapping) else {} + tables = cache.get("tables") if isinstance(cache, Mapping) else {} + + res = dict(job.result or {}) + selected_cat = ( + list(catalog_patches) + if catalog_patches is not None + else [ + { + "table": t["table"], + "changes": [ + {"path": c["path"], "value": c["value"], "reason": c.get("reason", "")} + for c in (t.get("changes") or []) + ], + } + for t in (res.get("catalog_patches") or []) + ] + ) + selected_metrics = list(metrics) if metrics is not None else list(res.get("metric_candidates") or []) + selected_ent_ops = ( + list(entity_operations) + if entity_operations is not None + else [c["op_payload"] for c in (res.get("entity_changes") or []) if isinstance(c, Mapping) and c.get("op_payload")] + ) + + catalog_applied_count = 0 + catalog_errors: list[str] = [] + pending_after_reviews: list[tuple[str, str, Any, Any]] = [] + for tbl_patch in selected_cat: + tbl_name = str(tbl_patch.get("table") or "").strip() + changes = list(tbl_patch.get("changes") or []) + if not tbl_name or not changes: + continue + resolved_tbl = catalog.resolve_table_name(cache, tbl_name) or tbl_name + try: + async with session.begin_nested(): + entry = await catalog.read_entry(session, ds.id, resolved_tbl) + cur_ver = int(entry.version) if entry else 0 + new_notes = catalog.apply_patch( + entry.notes if entry else {}, + changes, + table=resolved_tbl, + tables=tables if isinstance(tables, Mapping) else None, + ) + written = await catalog.write_entry( + session, ds.id, resolved_tbl, new_notes, if_version=cur_ver, actor=actor, commit=False + ) + pending_after_reviews.append((ds.id, resolved_tbl, entry.notes if entry else None, written)) + catalog_applied_count += len(changes) + except Exception as exc: # noqa: BLE001 + catalog_errors.append(f"{resolved_tbl}：{exc}") + + created_metrics: list[dict[str, Any]] = [] + updated_metrics: list[dict[str, Any]] = [] + metric_errors: list[dict[str, str]] = [] + for m_item in selected_metrics: + payload = {**dict(m_item), "source_id": ds.id} + payload.pop("exists_in_db", None) + payload.pop("source_name", None) + m_key = str(payload.get("key") or "").strip().lower() + try: + async with session.begin_nested(): + existing_row = ( + await session.execute(select(SemanticMetric).where(SemanticMetric.key == m_key)) + ).scalar_one_or_none() + if existing_row is not None: + if overwrite_existing_metrics: + row, _ = await update_metric(session, existing_row.id, payload, actor=actor) + updated_metrics.append(metric_to_dict(row, source_name=ds.name)) + else: + metric_errors.append({"key": m_key, "reason": f"指标「{m_key}」已存在，已跳过覆盖"}) + else: + row, _ = await create_metric(session, payload, actor=actor) + created_metrics.append(metric_to_dict(row, source_name=ds.name)) + except (MetricConflict, MetricInvalid, MetricNotFound, Exception) as exc: # noqa: BLE001 + metric_errors.append({"key": m_key or "unknown", "reason": str(exc)}) + + entity_apply_summary: dict[str, Any] = {"applied_count": 0, "problems": []} + if selected_ent_ops: + try: + async with session.begin_nested(): + entity_apply_summary = await entity_patch.apply_entity_patch( + session, + selected_ent_ops, + mode="confirmed", + actor=actor, + ) + except Exception as exc: # noqa: BLE001 + entity_apply_summary = {"applied_count": 0, "problems": [str(exc)]} + + applied_record = { + "applied_at": _iso(utcnow()), + "applied_by": actor, + "catalog_applied_count": catalog_applied_count, + "catalog_errors": catalog_errors, + "metrics_created_count": len(created_metrics), + "metrics_updated_count": len(updated_metrics), + "metric_errors": metric_errors, + "entities_applied_count": int(entity_apply_summary.get("applied_count") or 0), + "entity_errors": list(entity_apply_summary.get("problems") or []), + } + res["applied"] = applied_record + job.result = res + job.summary = {**dict(job.summary or {}), "applied": True, "applied_at": applied_record["applied_at"]} + await session.commit() + + for rev_sid, rev_tbl, rev_before, rev_written in pending_after_reviews: + catalog.after_review(rev_sid, rev_tbl, rev_before, rev_written) + + return { + "job_id": job.id, + "applied": applied_record, + "created_metrics": created_metrics, + "updated_metrics": updated_metrics, + } + + +async def startup(*, live: Any = None) -> int: + """服务启动或清理孤儿任务时，将上次残留未完成的拆解任务标记为 failed（中断说明）。""" + _ = live + running_ids = set(_ACTIVE_TASKS.keys()) + async with SessionLocal() as session: + rows = list( + ( + await session.execute( + select(SemanticDecomposeJob.id).where(SemanticDecomposeJob.status.in_(OPEN_STATUSES)) + ) + ).scalars() + ) + stale_ids = [jid for jid in rows if jid not in running_ids] + if stale_ids: + await session.execute( + update(SemanticDecomposeJob) + .where(SemanticDecomposeJob.id.in_(stale_ids), SemanticDecomposeJob.status.in_(OPEN_STATUSES)) + .values( + status="failed", + phase="interrupted", + phase_label="服务重启导致任务中断", + error=INTERRUPTED_ERROR, + finished_at=utcnow(), + ) + ) + await session.commit() + return len(stale_ids) + + +async def shutdown() -> None: + """服务关停时取消进程内正在执行的拆解任务。""" + tasks = list(_ACTIVE_TASKS.values()) + for task in tasks: + if not task.done(): + task.cancel() + for task in tasks: + with contextlib.suppress(asyncio.CancelledError, Exception): + await task + await startup() diff --git a/backend/app/data/semantic_metrics.py b/backend/app/data/semantic_metrics.py new file mode 100644 index 00000000..2948ad32 --- /dev/null +++ b/backend/app/data/semantic_metrics.py @@ -0,0 +1,1727 @@ +"""独立语义指标库（SemanticMetric）校验、确定性 SQL 编译、试算与 AI 提取服务。 + +核心能力： +1. 即存即用指标卡片 CRUD：严格校验指标 key、物理表存在性、聚合表达式与过滤条件安全性（拒绝写语句、多语句、子查询与遮罩列引用）。 +2. 确定性单条 SQL 编译器（compile_metric_sql）：支持按时间区间（date_from / date_to）、按时间序列趋势（group_by_date）、 + 按预定义业务维度切片（group_by）及维度过滤（filters）；当维度绑定了 CanonicalEntity 时，自动将业务别名确定性翻译为 + 目标库物理编码。 +3. T+1 空结果自动探底：单日查询返回空或全 NULL 时，自动探测底表最新可用业务日期（latest_available_date），省去 Agent 多轮试探。 +4. AI 辅助提取（suggest_metrics_from_source）：结合数据目录（中文名、粒度、业务日期、有效过滤）与知识库文档（如《指标公式总表》）， + 自动生成通过表列与语法核验的候选指标卡片。 +""" +from __future__ import annotations + +import json +import logging +import re +import time +from collections.abc import Mapping, Sequence +from datetime import date, datetime, timedelta, timezone +from functools import lru_cache +from typing import Any + +import sqlglot +from langchain_core.messages import HumanMessage, SystemMessage +from sqlglot import exp +from sqlalchemy import or_, select +from sqlalchemy.dialects import oracle as oracle_dialect +from sqlalchemy.ext.asyncio import AsyncSession + +from app.engine.state import message_text +from app.data import catalog, catalog_assistant +from app.data.engine import QueryLimits, masked_columns, run_query +from app.data.entities import ( + ACTIVE_ENTITY_STATUSES, + confirmed_bindings, + list_entities, + translate_entity_value, +) +from app.data.guard import SqlRejected, _ALWAYS_DENIED, _HIDDEN_WRITE, _SIDE_EFFECT_CALL, check as guard_check, first_verb +from app.data.names import name_key +from app.data.sqlcheck import dialect_of +from app.db.models import DataSource, Document +from app.db.models_metrics import SemanticMetric + +logger = logging.getLogger(__name__) + +METRIC_STATUSES = ("enabled", "draft", "disabled") +METRIC_SOURCES = ("human", "ai") +CALC_MODES = ("expression", "sql_template") +TIME_GRAINS = ("day", "month", "hour", "none") +ALLOWED_TEMPLATE_SLOTS = frozenset( + { + "select_dims", + "where", + "and_where", + "group_by", + "order_by", + "date_from", + "date_to", + "period_days", + "limit", + } +) + +_KEY_RE = re.compile(r"^[a-z][a-z0-9_]{0,63}$") +_IDENT_SAFE_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$|^[\u4e00-\u9fffA-Za-z0-9_]+$") +_ORACLE_SIMPLE_IDENT_RE = re.compile(r"^[A-Za-z][A-Za-z0-9_$#]*$") +_ORACLE_PSEUDOCOLUMNS = frozenset({"rownum", "rowid", "sysdate", "systimestamp", "level"}) +_ISO_DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$") +_FENCE_RE = re.compile(r"```(?:json)?\s*([\s\S]*?)```", re.IGNORECASE) +_SLOT_TOKEN_RE = re.compile(r"\{\{\s*([a-zA-Z0-9_]+)\s*\}\}") +_SELECT_DIMS_SLOT_RE = re.compile(r"\{\{\s*select_dims\s*\}\}\s*,?", re.IGNORECASE) +_WHERE_SLOT_RE = re.compile(r"(?P\bWHERE\s+)?\{\{\s*where\s*\}\}", re.IGNORECASE) +_AND_WHERE_SLOT_RE = re.compile(r"(?:\bAND\s+)?\{\{\s*and_where\s*\}\}", re.IGNORECASE) +_GROUP_BY_SLOT_RE = re.compile(r"(?:\bGROUP\s+BY\s+)?\{\{\s*group_by\s*\}\}", re.IGNORECASE) +_ORDER_BY_SLOT_RE = re.compile(r"(?:\bORDER\s+BY\s+)?\{\{\s*order_by\s*\}\}", re.IGNORECASE) +_DATE_FROM_SLOT_RE = re.compile(r"\{\{\s*date_from\s*\}\}", re.IGNORECASE) +_DATE_TO_SLOT_RE = re.compile(r"\{\{\s*date_to\s*\}\}", re.IGNORECASE) +_PERIOD_DAYS_SLOT_RE = re.compile(r"\{\{\s*period_days\s*\}\}", re.IGNORECASE) +_LIMIT_SLOT_RE = re.compile(r"\{\{\s*limit\s*\}\}", re.IGNORECASE) +_TRAILING_LIMIT_RE = re.compile(r"\s+\bLIMIT\s+(\d+)\s*$", re.IGNORECASE) +_HAS_ORACLE_ROW_LIMIT_RE = re.compile( + r"(?:\bFETCH\s+(?:FIRST|NEXT)\s+\d+\s+ROWS?\s+ONLY\s*$|\bROWNUM\s*<=?\s*\d+\s*$)", + re.IGNORECASE, +) + + +class MetricInvalid(ValueError): + """指标定义或查询参数不合法。""" + + def __init__(self, message: str, *, field: str = "") -> None: + super().__init__(message) + self.message = message + self.field = field + + +class MetricConflict(ValueError): + """指标唯一键冲突或乐观锁版本冲突。""" + + def __init__(self, message: str, *, field: str = "") -> None: + super().__init__(message) + self.message = message + self.field = field + + +class MetricNotFound(LookupError): + """指标或关联数据源不存在。""" + + +def _iso(val: Any) -> str | None: + if val is None or not hasattr(val, "isoformat"): + return None + if getattr(val, "tzinfo", None) is None: + val = val.replace(tzinfo=timezone.utc) + return val.isoformat() + + +def normalize_metric_key(raw: Any, *, field: str = "key") -> str: + val = str(raw or "").strip().lower() + if not _KEY_RE.match(val): + raise MetricInvalid("指标标识（key）须以小写字母开头，仅含小写字母、数字和下划线（最长 64 字符）", field=field) + return val + + +def _clean_aliases(raw: Any, *, field: str = "aliases") -> list[str]: + if raw in (None, ""): + return [] + if isinstance(raw, str): + items = [p.strip() for p in re.split(r"[,，、;\n]+", raw) if p.strip()] + elif isinstance(raw, list): + items = [str(p or "").strip() for p in raw if str(p or "").strip()] + else: + raise MetricInvalid(f"字段「{field}」需要是字符串列表", field=field) + seen: set[str] = set() + out: list[str] = [] + for item in items: + low = item.lower() + if low not in seen: + seen.add(low) + out.append(item[:80]) + return out[:24] + + +def _validate_column_identifier( + col: str, + *, + field: str, + masked_cols: set[str] | None = None, + allow_qualified: bool = False, +) -> str: + clean = str(col or "").strip() + if not clean: + raise MetricInvalid(f"字段「{field}」不能为空", field=field) + if len(clean) > 128 or any(ch in clean for ch in (";", "'", '"', "`", "\n", "\r", "\x00", "(", ")")): + raise MetricInvalid(f"列名「{clean}」包含非法字符", field=field) + if "." in clean and not allow_qualified: + raise MetricInvalid(f"单表模式下字段「{field}」请直接填写物理列名，不要带表别名前缀（{clean}）", field=field) + parts = [p.strip() for p in clean.split(".")] + if any(not p for p in parts) or len(parts) > 2: + raise MetricInvalid(f"列名「{clean}」格式不合法", field=field) + leaf_col = parts[-1] + if masked_cols and (name_key(clean) in masked_cols or name_key(leaf_col) in masked_cols): + raise MetricInvalid(f"列「{clean}」属于数据源敏感遮罩列，不能用于指标定义", field=field) + return clean + + +def normalize_dimension( + raw: Any, + *, + index: int = 0, + masked_cols: set[str] | None = None, + allow_qualified: bool = False, +) -> dict[str, Any]: + at = f"dimensions[{index}]" + if not isinstance(raw, Mapping): + raise MetricInvalid("指标维度需要是对象", field=at) + col = _validate_column_identifier( + str(raw.get("column") or ""), + field=f"{at}.column", + masked_cols=masked_cols, + allow_qualified=allow_qualified, + ) + raw_key = str(raw.get("key") or "").strip().lower() + if not raw_key: + leaf = col.rsplit(".", 1)[-1] + slug = re.sub(r"[^a-z0-9_]+", "_", leaf.lower()).strip("_") + raw_key = slug if slug and slug[0].isalpha() else f"dim_{index + 1}" + if not _KEY_RE.match(raw_key): + raise MetricInvalid(f"维度标识「{raw_key}」须以小写字母开头，仅含小写字母、数字和下划线", field=f"{at}.key") + label = str(raw.get("label") or raw_key).strip() or raw_key + entity_key = str(raw.get("entity_key") or "").strip().lower() + if entity_key and not _KEY_RE.match(entity_key): + raise MetricInvalid(f"关联实体标识「{entity_key}」格式不合法", field=f"{at}.entity_key") + return { + "key": raw_key, + "label": label[:80], + "column": col, + "entity_key": entity_key, + } + + +def infer_aggregation_type(sql_expression: str) -> str: + """从聚合 SQL 表达式推断展示用聚合类型标签（sum / count_distinct / count / avg / max / min / ratio / custom）。""" + expr = str(sql_expression or "").strip() + if not expr: + return "custom" + upper = expr.upper() + if "/" in upper and ("NULLIF" in upper or upper.startswith("ROUND(") or upper.count("(") >= 2): + if not upper.startswith(("SUM(", "COUNT(", "AVG(", "MAX(", "MIN(")): + return "ratio" + if re.match(r"^COUNT\s*\(\s*DISTINCT\b", upper): + return "count_distinct" + for prefix, tag in ( + ("SUM(", "sum"), + ("COUNT(", "count"), + ("AVG(", "avg"), + ("MAX(", "max"), + ("MIN(", "min"), + ): + if upper.startswith(prefix) or re.match(rf"^{prefix[:-1]}\s*\(", upper): + return tag + if "/" in upper: + return "ratio" + return "custom" + + +def validate_sql_fragment( + fragment: str, + *, + dialect: str | None, + field: str, + is_filter: bool = False, + allow_empty: bool = False, + masked_cols: set[str] | None = None, + known_cols: Mapping[str, str] | set[str] | None = None, + table_name: str = "", +) -> str: + """校验聚合表达式或过滤条件为单层安全只读表达式，不含注释、分号、写操作、子查询、不存在列或敏感遮罩列。""" + clean = str(fragment or "").strip() + if not clean: + if allow_empty: + return "" + raise MetricInvalid(f"字段「{field}」不能为空", field=field) + if len(clean) > 2000: + raise MetricInvalid(f"字段「{field}」长度不能超过 2000 字符", field=field) + if ";" in clean or "--" in clean or "/*" in clean or "*/" in clean: + raise MetricInvalid(f"字段「{field}」不能包含分号或 SQL 注释", field=field) + first_word = clean.split(None, 1)[0].lower() if clean.split() else "" + if first_word in _ALWAYS_DENIED or _HIDDEN_WRITE.search(clean) or _SIDE_EFFECT_CALL.search(clean): + raise MetricInvalid(f"字段「{field}」包含不允许的关键字或有副作用的函数", field=field) + + probe_sql = f"SELECT 1 FROM _metric_tbl WHERE ({clean})" if is_filter else f"SELECT ({clean}) AS _val FROM _metric_tbl" + read_dialect = dialect or "mysql" + try: + parsed = sqlglot.parse_one(probe_sql, read=read_dialect) + except Exception as exc: # noqa: BLE001 + # 回退默认方言再试一次，兼容通用 SQL 写法 + try: + parsed = sqlglot.parse_one(probe_sql) + except Exception as exc2: # noqa: BLE001 + raise MetricInvalid(f"字段「{field}」不是合法的 SQL 表达式（{exc2}）", field=field) from exc2 + + # 禁止嵌套 SELECT / 子查询 / UNION（保持指标底表血缘清晰且可受控加 WHERE 与 GROUP BY） + sub_selects = list(parsed.find_all(exp.Select)) + if len(sub_selects) > 1 or list(parsed.find_all(exp.Union)): + raise MetricInvalid(f"字段「{field}」不允许包含嵌套 SELECT 子查询，请使用聚合表达式（如 SUM(CASE WHEN cond THEN val END)）", field=field) + + for col_node in parsed.find_all(exp.Column): + cname = str(col_node.name or "").strip() + if not cname or cname == "*": + continue + if read_dialect == "oracle" and cname.lower() in _ORACLE_PSEUDOCOLUMNS: + continue + ckey = name_key(cname) + if masked_cols and ckey in masked_cols: + raise MetricInvalid(f"字段「{field}」引用了数据源敏感遮罩列「{cname}」，不允许用于指标定义", field=field) + if known_cols and ckey not in known_cols: + tbl_desc = f"表「{table_name}」" if table_name else "目标物理表" + raise MetricInvalid(f"字段「{field}」引用了{tbl_desc}中不存在的列「{cname}」", field=field) + + return clean + + +def metric_to_dict( + row: SemanticMetric, + *, + source_name: str = "", + restricted: bool = False, +) -> dict[str, Any]: + dims = list(row.dimensions or []) + if restricted: + dims = [ + {"key": d.get("key"), "label": d.get("label"), "entity_key": d.get("entity_key") or ""} + for d in dims + if isinstance(d, Mapping) + ] + calc_mode = str(getattr(row, "calc_mode", None) or "expression").strip().lower() or "expression" + sql_tpl = str(getattr(row, "sql_template", None) or "").strip() + base: dict[str, Any] = { + "id": row.id, + "key": row.key, + "name": row.name, + "aliases": list(row.aliases or []), + "description": row.description or "", + "unit": row.unit or "", + "source_id": row.source_id, + "source_name": source_name, + "calc_mode": calc_mode, + "aggregation_type": "template" if calc_mode == "sql_template" else infer_aggregation_type(row.sql_expression), + "dimensions": dims, + "default_time_grain": row.default_time_grain or "day", + "has_date_column": bool(str(row.date_column or "").strip()), + "status": row.status or "enabled", + "source": row.source or "human", + "version": int(row.version or 1), + "created_by": row.created_by, + "updated_by": row.updated_by, + "created_at": _iso(row.created_at), + "updated_at": _iso(row.updated_at), + } + if not restricted: + base.update( + table_name=row.table_name, + sql_expression=row.sql_expression, + sql_template=sql_tpl, + date_column=row.date_column or "", + filter_sql=row.filter_sql or "", + ) + return base + + +async def _resolve_source(session: AsyncSession, source_id_or_name: str) -> DataSource: + clean = str(source_id_or_name or "").strip() + if not clean: + raise MetricInvalid("请选择指标归属的数据源（source_id）", field="source_id") + ds = await session.get(DataSource, clean) + if ds is None: + ds = (await session.execute(select(DataSource).where(DataSource.name == clean))).scalar_one_or_none() + if ds is None: + raise MetricNotFound(f"找不到数据源「{clean}」") + return ds + + +def _resolve_table_and_columns(ds: DataSource, raw_table: str) -> tuple[str, dict[str, str]]: + """返回对齐后的表名与该表的 {name_key(col): col_type} 字典（未探查时仅做基本标识符校验）。""" + clean_table = str(raw_table or "").strip() + if not clean_table: + raise MetricInvalid("请填写物理表名（table_name）", field="table_name") + if any(ch in clean_table for ch in (";", "'", '"', "`", "\n", "\r", "(", ")")): + raise MetricInvalid(f"表名「{clean_table}」包含非法字符", field="table_name") + cache = ds.schema_cache if isinstance(ds.schema_cache, Mapping) else {} + tables = cache.get("tables") if isinstance(cache, Mapping) else None + if isinstance(tables, Mapping) and tables: + resolved = catalog.resolve_table_name(cache, clean_table) + if resolved is None: + raise MetricInvalid(f"数据源「{ds.name}」的已探查表结构中不存在表「{clean_table}」", field="table_name") + tinfo = tables.get(resolved) or {} + cols: dict[str, str] = {} + for c in tinfo.get("columns") or []: + if isinstance(c, Mapping) and c.get("name"): + cols[name_key(str(c["name"]))] = str(c.get("type") or "").lower() + return resolved, cols + return clean_table, {} + + +def _validate_and_analyze_sql_template( + raw_template: str, + *, + ds: DataSource, + dialect: str | None, + masked_set: set[str], + metric_key: str, + raw_table: str, + sql_expression: str, + date_column: str, + filter_sql: str, + dims: list[dict[str, Any]], + grain: str, +) -> tuple[str, str, str]: + """校验多表 / 参数化 SQL 模板（支持 {{select_dims}}、{{and_where}}、{{period_days}} 等安全槽位）， + 返回 (clean_template, primary_table_name, effective_sql_expression)。 + """ + clean_tpl = str(raw_template or "").strip().rstrip(";").strip() + if not clean_tpl: + raise MetricInvalid("多表或参数化模板模式下，请填写「SQL 模板」内容", field="sql_template") + if len(clean_tpl) > 8000: + raise MetricInvalid("「SQL 模板」长度不能超过 8000 字符", field="sql_template") + if ";" in clean_tpl or "--" in clean_tpl or "/*" in clean_tpl or "*/" in clean_tpl: + raise MetricInvalid("「SQL 模板」不能包含多语句分号或 SQL 注释", field="sql_template") + if _HIDDEN_WRITE.search(clean_tpl) or _SIDE_EFFECT_CALL.search(clean_tpl): + raise MetricInvalid("「SQL 模板」包含不允许的写操作或有副作用的函数", field="sql_template") + + slots_found = [m.group(1).strip().lower() for m in _SLOT_TOKEN_RE.finditer(clean_tpl)] + unknown_slots = sorted({s for s in slots_found if s not in ALLOWED_TEMPLATE_SLOTS}) + if unknown_slots: + allowed_txt = ", ".join(f"{{{{{s}}}}}" for s in sorted(ALLOWED_TEMPLATE_SLOTS)) + raise MetricInvalid( + f"「SQL 模板」使用了不支持的占位符：{', '.join(unknown_slots)}；仅支持：{allowed_txt}", + field="sql_template", + ) + + cache = ds.schema_cache if isinstance(ds.schema_cache, Mapping) else {} + tables = cache.get("tables") if isinstance(cache, Mapping) else None + has_schema = isinstance(tables, Mapping) and bool(tables) + + # 先预编译两种典型形态（无筛选汇总 + 带日期/维度切片），双重验证语法与只读安全守卫 + draft_metric = { + "key": metric_key, + "name": metric_key, + "calc_mode": "sql_template", + "table_name": raw_table or "_anchor", + "sql_expression": sql_expression, + "sql_template": clean_tpl, + "date_column": date_column, + "filter_sql": filter_sql, + "dimensions": dims, + "default_time_grain": grain, + } + test_variants: list[dict[str, Any]] = [ + {"date_from": None, "date_to": None, "group_by_date": False, "group_by": []}, + ] + if date_column or dims: + test_variants.append( + { + "date_from": "2026-01-01" if date_column else None, + "date_to": "2026-01-07" if date_column else None, + "group_by_date": bool(date_column), + "group_by": [dims[0]["key"]] if dims else [], + } + ) + + referenced_physical_tables: list[str] = [] + all_valid_cols: set[str] = set() + read_dialect = dialect or "mysql" + + for v_idx, variant in enumerate(test_variants): + compiled_sql, _ = compile_metric_sql( + draft_metric, + source=ds, + entities=(), + date_from=variant["date_from"], + date_to=variant["date_to"], + group_by_date=variant["group_by_date"], + group_by=variant["group_by"], + limit=50, + ) + try: + checked_stmt = guard_check(compiled_sql, readonly=True, dialect=dialect, source_name=ds.name) + except SqlRejected as exc: + raise MetricInvalid(f"「SQL 模板」未通过只读安全守卫：{exc}", field="sql_template") from exc + verb = first_verb(checked_stmt, dialect=dialect) + if verb not in ("select", "with"): + raise MetricInvalid("「SQL 模板」必须是只读的 SELECT 或带 WITH 子句的查询语句", field="sql_template") + + try: + parsed = sqlglot.parse_one(checked_stmt, read=read_dialect) + except Exception: # noqa: BLE001 + try: + parsed = sqlglot.parse_one(checked_stmt) + except Exception as exc2: # noqa: BLE001 + mode_label = "汇总态" if v_idx == 0 else "按日期或维度分组态" + raise MetricInvalid(f"「SQL 模板」在{mode_label}下编译出的 SQL 语法有误：{exc2}", field="sql_template") from exc2 + + cte_names = {name_key(str(cte.alias_or_name or "")) for cte in parsed.find_all(exp.CTE) if cte.alias_or_name} + cte_names.add(name_key("_metric_src")) + cte_names.add(name_key("metric_src")) + if read_dialect == "oracle": + cte_names.add(name_key("dual")) + + if v_idx == 0: + for tbl_node in parsed.find_all(exp.Table): + t_raw = str(tbl_node.name or "").strip() + if not t_raw or name_key(t_raw) in cte_names: + continue + db_part = str(tbl_node.db or "").strip() + qualified = f"{db_part}.{t_raw}" if db_part else t_raw + if has_schema: + resolved = catalog.resolve_table_name(cache, qualified) or catalog.resolve_table_name(cache, t_raw) + if resolved is None: + raise MetricInvalid( + f"「SQL 模板」引用了数据源「{ds.name}」中不存在的物理表「{qualified}」", + field="sql_template", + ) + if resolved not in referenced_physical_tables: + referenced_physical_tables.append(resolved) + tinfo = (tables or {}).get(resolved) or {} + for c in tinfo.get("columns") or []: + if isinstance(c, Mapping) and c.get("name"): + all_valid_cols.add(name_key(str(c["name"]))) + else: + if qualified not in referenced_physical_tables: + referenced_physical_tables.append(qualified) + + derived_aliases = { + name_key(str(al.alias or "")) + for al in parsed.find_all(exp.Alias) + if str(al.alias or "").strip() + } + derived_aliases.update(name_key(str(d.get("key") or "")) for d in dims if d.get("key")) + for col_node in parsed.find_all(exp.Column): + cname = str(col_node.name or "").strip() + if not cname or cname == "*": + continue + if read_dialect == "oracle" and cname.lower() in _ORACLE_PSEUDOCOLUMNS: + continue + ckey = name_key(cname) + if masked_set and ckey in masked_set: + raise MetricInvalid( + f"「SQL 模板」引用了数据源敏感遮罩列「{cname}」，不允许用于指标定义", + field="sql_template", + ) + if has_schema and all_valid_cols and ckey not in all_valid_cols and ckey not in derived_aliases: + if cname.lower() not in ("period", "stat_date", metric_key.lower()): + raise MetricInvalid( + f"「SQL 模板」引用了所涉物理表中不存在的列「{cname}」", + field="sql_template", + ) + + if raw_table.strip(): + primary_table, _ = _resolve_table_and_columns(ds, raw_table) + elif referenced_physical_tables: + primary_table = referenced_physical_tables[0] + else: + raise MetricInvalid("「SQL 模板」至少需要引用一张物理业务表", field="sql_template") + + effective_expr = sql_expression.strip() + if not effective_expr: + joined_hint = " + ".join(referenced_physical_tables[:3]) + if len(referenced_physical_tables) > 3: + joined_hint += f" 等 {len(referenced_physical_tables)} 表" + effective_expr = f"[多表模板] {joined_hint or primary_table}" + return clean_tpl, primary_table, effective_expr + + +async def _normalize_metric_payload( + session: AsyncSession, + payload: Mapping[str, Any], + *, + existing: SemanticMetric | None = None, +) -> tuple[DataSource, dict[str, Any]]: + source_ref = payload.get("source_id") if "source_id" in payload else (existing.source_id if existing else "") + ds = await _resolve_source(session, str(source_ref or "")) + masked_set = {name_key(c) for c in masked_columns(ds.options)} + dialect = dialect_of(ds.kind) + + key = normalize_metric_key( + payload.get("key") if "key" in payload else (existing.key if existing else ""), + field="key", + ) + name = str(payload.get("name") if "name" in payload else (existing.name if existing else "")).strip() + if not name: + raise MetricInvalid("指标名称（name）不能为空", field="name") + if len(name) > 200: + raise MetricInvalid("指标名称不能超过 200 字符", field="name") + + raw_tpl = ( + payload.get("sql_template") + if "sql_template" in payload + else (getattr(existing, "sql_template", "") if existing else "") + ) + raw_mode = ( + payload.get("calc_mode") + if "calc_mode" in payload + else (getattr(existing, "calc_mode", "expression") if existing else "") + ) + calc_mode = str(raw_mode or "").strip().lower() + if not calc_mode: + calc_mode = "sql_template" if str(raw_tpl or "").strip() else "expression" + if calc_mode not in CALC_MODES: + raise MetricInvalid(f"计算模式「{calc_mode}」不支持，可选：{', '.join(CALC_MODES)}", field="calc_mode") + + raw_table = str( + payload.get("table_name") if "table_name" in payload else (existing.table_name if existing else "") + ).strip() + raw_expr = str( + payload.get("sql_expression") if "sql_expression" in payload else (existing.sql_expression if existing else "") + ).strip() + raw_date = str( + payload.get("date_column") if "date_column" in payload else (existing.date_column if existing else "") + ).strip() + raw_filter = str( + payload.get("filter_sql") if "filter_sql" in payload else (existing.filter_sql if existing else "") + ).strip() + + grain = str( + payload.get("default_time_grain") + if "default_time_grain" in payload + else (existing.default_time_grain if existing else "day") + ).strip().lower() or "day" + if grain not in TIME_GRAINS: + raise MetricInvalid(f"默认时间粒度「{grain}」不支持，可选：{', '.join(TIME_GRAINS)}", field="default_time_grain") + + raw_dims = payload.get("dimensions") if "dimensions" in payload else (existing.dimensions if existing else []) + if raw_dims is None: + raw_dims = [] + if not isinstance(raw_dims, list): + raise MetricInvalid("维度列表（dimensions）需要是数组", field="dimensions") + + is_tpl = calc_mode == "sql_template" + dims: list[dict[str, Any]] = [] + seen_dim_keys: set[str] = set() + for idx, item in enumerate(raw_dims): + norm = normalize_dimension(item, index=idx, masked_cols=masked_set, allow_qualified=is_tpl) + if norm["key"] in seen_dim_keys: + raise MetricInvalid(f"维度标识「{norm['key']}」重复", field=f"dimensions[{idx}].key") + seen_dim_keys.add(norm["key"]) + dims.append(norm) + + if not is_tpl: + table_name, known_cols = _resolve_table_and_columns(ds, raw_table) + sql_expression = validate_sql_fragment( + raw_expr, + dialect=dialect, + field="sql_expression", + is_filter=False, + allow_empty=False, + masked_cols=masked_set, + known_cols=known_cols, + table_name=table_name, + ) + date_column = "" + if raw_date: + date_column = _validate_column_identifier(raw_date, field="date_column", masked_cols=masked_set) + if known_cols and name_key(date_column) not in known_cols: + raise MetricInvalid(f"表「{table_name}」中不存在时间列「{date_column}」", field="date_column") + filter_sql = validate_sql_fragment( + raw_filter, + dialect=dialect, + field="filter_sql", + is_filter=True, + allow_empty=True, + masked_cols=masked_set, + known_cols=known_cols, + table_name=table_name, + ) + for idx, norm in enumerate(dims): + if known_cols and name_key(norm["column"]) not in known_cols: + raise MetricInvalid(f"表「{table_name}」中不存在维度列「{norm['column']}」", field=f"dimensions[{idx}].column") + sql_template = "" + else: + date_column = "" + if raw_date: + date_column = _validate_column_identifier( + raw_date, field="date_column", masked_cols=masked_set, allow_qualified=True + ) + filter_sql = validate_sql_fragment( + raw_filter, + dialect=dialect, + field="filter_sql", + is_filter=True, + allow_empty=True, + masked_cols=masked_set, + known_cols=None, + table_name=raw_table, + ) + # 若用户在无占位符的基础多表查询之上另填了外层聚合表达式，也校验其聚合安全性（跳过以 [多表模板] 开头的自动摘要） + clean_outer_expr = "" + if raw_expr and not raw_expr.startswith("[多表模板]") and not _SLOT_TOKEN_RE.search(str(raw_tpl or "")): + clean_outer_expr = validate_sql_fragment( + raw_expr, + dialect=dialect, + field="sql_expression", + is_filter=False, + allow_empty=True, + masked_cols=masked_set, + known_cols=None, + table_name=raw_table, + ) + sql_template, table_name, sql_expression = _validate_and_analyze_sql_template( + str(raw_tpl or ""), + ds=ds, + dialect=dialect, + masked_set=masked_set, + metric_key=key, + raw_table=raw_table, + sql_expression=clean_outer_expr, + date_column=date_column, + filter_sql=filter_sql, + dims=dims, + grain=grain, + ) + + aliases = _clean_aliases( + payload.get("aliases") if "aliases" in payload else (existing.aliases if existing else []), + field="aliases", + ) + description = str( + payload.get("description") if "description" in payload else (existing.description if existing else "") + ).strip()[:2000] + unit = str(payload.get("unit") if "unit" in payload else (existing.unit if existing else "")).strip()[:40] + + status = str( + payload.get("status") if "status" in payload else (existing.status if existing else "enabled") + ).strip().lower() or "enabled" + if status not in METRIC_STATUSES: + raise MetricInvalid(f"指标状态「{status}」不支持，可选：{', '.join(METRIC_STATUSES)}", field="status") + + source_origin = str( + payload.get("source") if "source" in payload else (existing.source if existing else "human") + ).strip().lower() or "human" + if source_origin not in METRIC_SOURCES: + source_origin = "human" + + return ds, { + "key": key, + "name": name, + "aliases": aliases, + "description": description, + "unit": unit, + "source_id": ds.id, + "calc_mode": calc_mode, + "table_name": table_name, + "sql_expression": sql_expression, + "sql_template": sql_template, + "date_column": date_column, + "filter_sql": filter_sql, + "dimensions": dims, + "default_time_grain": grain, + "status": status, + "source": source_origin, + } + + +async def list_metrics( + session: AsyncSession, + *, + source_id: str | None = None, + status: str | None = None, + q: str | None = None, +) -> list[SemanticMetric]: + stmt = select(SemanticMetric).order_by(SemanticMetric.updated_at.desc(), SemanticMetric.key.asc()) + if source_id: + stmt = stmt.where(SemanticMetric.source_id == source_id.strip()) + if status: + stmt = stmt.where(SemanticMetric.status == status.strip()) + rows = list((await session.execute(stmt)).scalars().all()) + if q and q.strip(): + needle = q.strip().lower() + rows = [ + r + for r in rows + if needle in (r.key or "").lower() + or needle in (r.name or "").lower() + or needle in (r.description or "").lower() + or needle in (r.table_name or "").lower() + or any(needle in str(a).lower() for a in (r.aliases or [])) + ] + return rows + + +async def get_metric(session: AsyncSession, id_or_key: str) -> SemanticMetric: + clean = str(id_or_key or "").strip() + if not clean: + raise MetricNotFound("未指定指标标识") + row = await session.get(SemanticMetric, clean) + if row is None: + row = ( + await session.execute( + select(SemanticMetric).where(or_(SemanticMetric.key == clean.lower(), SemanticMetric.name == clean)) + ) + ).scalars().first() + if row is None: + raise MetricNotFound(f"找不到语义指标「{clean}」") + return row + + +async def create_metric( + session: AsyncSession, + payload: Mapping[str, Any], + *, + actor: str | None = None, +) -> tuple[SemanticMetric, DataSource]: + ds, fields = await _normalize_metric_payload(session, payload, existing=None) + dup = (await session.execute(select(SemanticMetric).where(SemanticMetric.key == fields["key"]))).scalar_one_or_none() + if dup is not None: + raise MetricConflict(f"指标标识「{fields['key']}」已存在，请换用其他标识或直接编辑现有指标", field="key") + row = SemanticMetric( + **fields, + version=1, + created_by=actor, + updated_by=actor, + ) + session.add(row) + await session.flush() + return row, ds + + +async def update_metric( + session: AsyncSession, + metric_id: str, + payload: Mapping[str, Any], + *, + actor: str | None = None, +) -> tuple[SemanticMetric, DataSource]: + row = await get_metric(session, metric_id) + expected_version = payload.get("expected_version") + if expected_version is not None and int(expected_version) != int(row.version or 1): + raise MetricConflict("该指标已被其他人修改，请刷新后重试", field="version") + ds, fields = await _normalize_metric_payload(session, payload, existing=row) + if fields["key"] != row.key: + dup = ( + await session.execute(select(SemanticMetric).where(SemanticMetric.key == fields["key"], SemanticMetric.id != row.id)) + ).scalar_one_or_none() + if dup is not None: + raise MetricConflict(f"指标标识「{fields['key']}」已被其他指标占用", field="key") + for k, v in fields.items(): + setattr(row, k, v) + row.version = int(row.version or 1) + 1 + row.updated_by = actor + await session.flush() + return row, ds + + +async def delete_metric(session: AsyncSession, metric_id: str) -> SemanticMetric: + row = await get_metric(session, metric_id) + await session.delete(row) + await session.flush() + return row + + +# -------------------------------------------------------------------------- +# 确定性 SQL 编译与业务实体编码翻译 +# -------------------------------------------------------------------------- + + +@lru_cache(maxsize=1) +def _oracle_reserved_words() -> frozenset[str]: + try: + return frozenset(str(w).lower() for w in oracle_dialect.dialect().identifier_preparer._reserved_words) + except Exception: # noqa: BLE001 + return frozenset({"date", "level", "number", "uid", "user", "comment", "group", "order", "size", "mode", "share"}) + + +def _quote_single_part(part: str, kind: str | None, *, force_quote: bool = False, is_alias: bool = False) -> str: + clean = str(part or "").strip() + if not clean: + return "" + k = (kind or "").strip().lower() + if k == "oracle": + if force_quote: + return '"' + clean.replace('"', '""') + '"' + if _ORACLE_SIMPLE_IDENT_RE.match(clean): + if clean.lower() in _oracle_reserved_words(): + val = clean if is_alias else clean.upper() + return '"' + val.replace('"', '""') + '"' + return clean + return '"' + clean.replace('"', '""') + '"' + q = "`" if k in ("mysql", "mariadb") else '"' + return f"{q}{clean.replace(q, q + q)}{q}" + + +def _quote_ident(ident: str, kind: str | None, *, is_alias: bool = False) -> str: + parts = [p.strip() for p in str(ident or "").split(".") if p.strip()] + return ".".join(_quote_single_part(p, kind, is_alias=is_alias and idx == len(parts) - 1) for idx, p in enumerate(parts)) + + +def _format_table_ref(source: DataSource, table_name: str) -> str: + clean = str(table_name or "").strip() + if not clean: + return "" + cache = source.schema_cache if isinstance(source.schema_cache, Mapping) else {} + tables = cache.get("tables") if isinstance(cache, Mapping) else None + if isinstance(tables, Mapping) and tables: + resolved = catalog.resolve_table_name(cache, clean) + if resolved and resolved in tables: + meta = tables.get(resolved) or {} + force_t = bool(meta.get("quote_name") is True) + if "." in resolved: + schema_part, tbl_part = resolved.split(".", 1) + else: + schema_part = str(meta.get("schema") or "").strip() + tbl_part = resolved + tbl_sql = _quote_single_part(tbl_part, source.kind, force_quote=force_t) + if schema_part: + return f"{_quote_single_part(schema_part, source.kind)}.{tbl_sql}" + return tbl_sql + return _quote_ident(clean, source.kind) + + +def _lookup_column_meta(source: DataSource, table_name: str, column_name: str) -> Mapping[str, Any] | None: + cache = source.schema_cache if isinstance(source.schema_cache, Mapping) else {} + tables = cache.get("tables") if isinstance(cache, Mapping) else None + if not isinstance(tables, Mapping) or not tables: + return None + target = name_key(str(column_name or "").rsplit(".", 1)[-1]) + if not target: + return None + if table_name: + resolved = catalog.resolve_table_name(cache, table_name) or table_name + tinfo = tables.get(resolved) or {} + for col in tinfo.get("columns") or []: + if isinstance(col, Mapping) and name_key(str(col.get("name") or "")) == target: + return col + for tinfo in tables.values(): + if not isinstance(tinfo, Mapping): + continue + for col in tinfo.get("columns") or []: + if isinstance(col, Mapping) and name_key(str(col.get("name") or "")) == target: + return col + return None + + +def _format_col_ref(source: DataSource, table_name: str, col_expr: str) -> str: + parts = [p.strip() for p in str(col_expr or "").split(".") if p.strip()] + if not parts: + return "" + col_meta = _lookup_column_meta(source, table_name, parts[-1]) + force_col = bool(col_meta and col_meta.get("quote_name") is True) + col_name = str(col_meta["name"]) if (force_col and col_meta and col_meta.get("name")) else parts[-1] + if len(parts) == 1: + return _quote_single_part(col_name, source.kind, force_quote=force_col) + prefix = ".".join(_quote_single_part(p, source.kind) for p in parts[:-1]) + return f"{prefix}.{_quote_single_part(col_name, source.kind, force_quote=force_col)}" + + +def _sql_literal(val: Any) -> str: + if isinstance(val, bool): + return "1" if val else "0" + if isinstance(val, (int, float)): + return str(val) + text = str(val if val is not None else "") + escaped = text.replace("'", "''") + return f"'{escaped}'" + + +def _column_temporal_kind(source: DataSource, table_name: str, column_name: str) -> str: + """识别业务时间列的底层存储形态：text | datetime_text | date | datetime。 + 注意：Oracle 的 DATE 类型自带时分秒（YYYY-MM-DD HH24:MI:SS），必须按 datetime 处理。 + """ + k = (source.kind or "").strip().lower() + col_meta = _lookup_column_meta(source, table_name, column_name) + ctype = str((col_meta or {}).get("type") or "").lower() + if k == "sqlite": + if "timestamp" in ctype or "datetime" in ctype: + return "datetime_text" + return "text" + if ctype: + if any(tok in ctype for tok in ("char", "text", "clob", "string")): + return "text" + if k == "oracle": + if "date" in ctype or "timestamp" in ctype: + return "datetime" + else: + if "timestamp" in ctype or "datetime" in ctype: + return "datetime" + if "date" in ctype: + return "date" + if k == "oracle": + return "datetime" + if k in ("postgres", "postgresql", "mysql", "mariadb"): + return "date" + return "text" + + +def _date_expr_for_grain(quoted_col: str, grain: str, kind: str | None, *, temporal_kind: str = "") -> str: + k = (kind or "").strip().lower() + if k == "oracle": + if temporal_kind == "text": + if grain == "month": + return f"SUBSTR({quoted_col}, 1, 7)" + if grain == "hour": + return f"SUBSTR({quoted_col}, 1, 13) || ':00'" + return f"SUBSTR({quoted_col}, 1, 10)" + if grain == "month": + return f"TO_CHAR({quoted_col}, 'YYYY-MM')" + if grain == "hour": + return f"TO_CHAR({quoted_col}, 'YYYY-MM-DD HH24') || ':00'" + return f"TO_CHAR({quoted_col}, 'YYYY-MM-DD')" + if k in ("postgres", "postgresql"): + if temporal_kind == "text": + if grain == "month": + return f"SUBSTR({quoted_col}, 1, 7)" + if grain == "hour": + return f"SUBSTR({quoted_col}, 1, 13) || ':00'" + return f"SUBSTR({quoted_col}, 1, 10)" + if grain == "month": + return f"TO_CHAR({quoted_col}, 'YYYY-MM')" + if grain == "hour": + return f"TO_CHAR({quoted_col}, 'YYYY-MM-DD HH24') || ':00'" + return f"CAST({quoted_col} AS DATE)" + if k in ("mysql", "mariadb"): + if grain == "month": + return f"DATE_FORMAT({quoted_col}, '%Y-%m')" + if grain == "hour": + return f"DATE_FORMAT({quoted_col}, '%Y-%m-%d %H:00')" + return f"DATE({quoted_col})" + if grain == "month": + return f"SUBSTR(CAST({quoted_col} AS TEXT), 1, 7)" + if grain == "hour": + return f"SUBSTR(CAST({quoted_col} AS TEXT), 1, 13) || ':00'" + return f"SUBSTR(CAST({quoted_col} AS TEXT), 1, 10)" + + +def _format_date_bound_sql(bound: str, *, kind: str | None, temporal_kind: str) -> str: + k = (kind or "").strip().lower() + if temporal_kind in ("text", "datetime_text") or k == "sqlite": + return _sql_literal(bound) + if k == "oracle": + if _ISO_DATE_RE.match(bound): + return f"DATE '{bound}'" + norm_ts = bound.replace("T", " ") + if len(norm_ts) == 16: + norm_ts += ":00" + return f"TO_DATE({_sql_literal(norm_ts)}, 'YYYY-MM-DD HH24:MI:SS')" + return _sql_literal(bound) + + +def _validate_date_bound(raw: str, *, field: str) -> str: + clean = str(raw or "").strip() + if not clean: + return "" + if not re.match(r"^\d{4}-\d{2}-\d{2}(?:[ T]\d{2}:\d{2}(?::\d{2})?)?$", clean): + raise MetricInvalid(f"日期参数「{field}={clean}」格式无效，请使用 YYYY-MM-DD", field=field) + return clean + + +def _compute_period_days_sql( + *, + source: DataSource, + table_name: str, + d_from: str, + d_to: str, + group_by_date: bool, + grain: str, + date_col: str, + kind: str | None, +) -> str: + if group_by_date and grain == "day": + return "1" + if not group_by_date and d_from and d_to: + try: + d1 = date.fromisoformat(d_from[:10]) + d2 = date.fromisoformat(d_to[:10]) + return str(max(1, (d2 - d1).days + 1)) + except ValueError: + pass + if date_col: + t_kind = _column_temporal_kind(source, table_name, date_col) + q_col = _format_col_ref(source, table_name, date_col) + day_expr = _date_expr_for_grain(q_col, "day", kind, temporal_kind=t_kind) + return f"COALESCE(NULLIF(COUNT(DISTINCT {day_expr}), 0), 1)" + return "1" + + +def _split_top_level_with_clause(sql: str) -> tuple[str, str]: + """若 SQL 以顶层 WITH 子句开头，拆分为 (with_prefix, main_select)；否则返回 ('', sql)。 + 用于 Oracle 套 ROWNUM 或外层子查询包装时将 WITH 保持在最外层，避免 ORA-32034。 + """ + stripped = sql.strip() + if not re.match(r"^WITH\b", stripped, re.IGNORECASE): + return "", stripped + n = len(stripped) + i = 0 + depth = 0 + while i < n: + ch = stripped[i] + if ch == "'": + i += 1 + while i < n: + if stripped[i] == "'": + if i + 1 < n and stripped[i + 1] == "'": + i += 2 + continue + i += 1 + break + i += 1 + continue + if ch in ('"', "`"): + quote_ch = ch + i += 1 + while i < n: + if stripped[i] == quote_ch: + if i + 1 < n and stripped[i + 1] == quote_ch: + i += 2 + continue + i += 1 + break + i += 1 + continue + if ch == "(": + depth += 1 + i += 1 + continue + if ch == ")": + if depth > 0: + depth -= 1 + if depth == 0: + j = i + 1 + while j < n and stripped[j].isspace(): + j += 1 + if j < n and stripped[j] == ",": + i = j + 1 + continue + if re.match(r"^(?:SELECT\b|\(\s*SELECT\b)", stripped[j:], re.IGNORECASE): + return stripped[: i + 1].rstrip(), stripped[j:].lstrip() + i += 1 + continue + i += 1 + return "", stripped + + +def _apply_row_limit(sql: str, limit: int, kind: str | None, *, force_if_missing: bool = True) -> str: + clean = sql.strip() + safe_n = max(1, int(limit)) + k = (kind or "").strip().lower() + if k == "oracle": + # Oracle 不支持 LIMIT N（会报 ORA-00933）；统一剥离并转换为兼容 Oracle 11g/19c 的顶层 WITH + ROWNUM 包裹 + m_lim = _TRAILING_LIMIT_RE.search(clean) + if m_lim: + safe_n = max(1, int(m_lim.group(1))) + clean = clean[: m_lim.start()].rstrip() + force_if_missing = True + if not force_if_missing or _HAS_ORACLE_ROW_LIMIT_RE.search(clean): + return clean + with_prefix, main_select = _split_top_level_with_clause(clean) + wrapped = f"SELECT * FROM ({main_select}) WHERE ROWNUM <= {safe_n}" + return f"{with_prefix} {wrapped}" if with_prefix else wrapped + + if not force_if_missing: + return clean + if _TRAILING_LIMIT_RE.search(clean) or _HAS_ORACLE_ROW_LIMIT_RE.search(clean): + return clean + return f"{clean} LIMIT {safe_n}" + + +def compile_metric_sql( + metric: Mapping[str, Any] | SemanticMetric, + *, + source: DataSource, + entities: Sequence[Any] = (), + date_from: str | None = None, + date_to: str | None = None, + group_by_date: bool = False, + time_grain: str | None = None, + group_by: Sequence[str] | None = None, + filters: Mapping[str, Any] | None = None, + limit: int = 200, +) -> tuple[str, list[dict[str, Any]]]: + """把语义指标卡片（单表表达式或多表参数化 SQL 模板）与动态查询条件编译为单条确定性只读 SELECT 语句。 + + 返回 (sql, entity_alignments)。 + """ + m = ( + { + "key": metric.key, + "name": metric.name, + "calc_mode": getattr(metric, "calc_mode", None) or "expression", + "table_name": metric.table_name, + "sql_expression": metric.sql_expression, + "sql_template": getattr(metric, "sql_template", None) or "", + "date_column": metric.date_column or "", + "filter_sql": metric.filter_sql or "", + "dimensions": list(metric.dimensions or []), + "default_time_grain": metric.default_time_grain or "day", + } + if isinstance(metric, SemanticMetric) + else dict(metric) + ) + kind = source.kind + calc_mode = str(m.get("calc_mode") or ("sql_template" if m.get("sql_template") else "expression")).strip().lower() + table_name = str(m.get("table_name") or "").strip() + sql_expr = str(m.get("sql_expression") or "").strip() + sql_template = str(m.get("sql_template") or "").strip().rstrip(";").strip() + date_col = str(m.get("date_column") or "").strip() + filter_sql = str(m.get("filter_sql") or "").strip() + dims = [d for d in (m.get("dimensions") or []) if isinstance(d, Mapping)] + + # 构建维度查找索引（支持按 key / label / column / bare_column / entity_key 引用） + dim_lookup: dict[str, Mapping[str, Any]] = {} + for d in dims: + raw_col = str(d.get("column") or "") + bare_col = raw_col.rsplit(".", 1)[-1] + for tok in (d.get("key"), d.get("label"), raw_col, bare_col, d.get("entity_key")): + s = str(tok or "").strip().lower() + if s and s not in dim_lookup: + dim_lookup[s] = d + + # 构建实体索引（key -> CanonicalEntity，以及 col -> CanonicalEntity） + entity_by_key: dict[str, Any] = {} + entity_by_col: dict[str, Any] = {} + for ent in entities or []: + entity_by_key[str(ent.key).lower()] = ent + for b in confirmed_bindings(ent): + if name_key(str(b.get("source_name") or "")) == name_key(source.name): + b_tbl = name_key(str(b.get("table_name") or "").rsplit(".", 1)[-1]) + c_key = name_key(str(b.get("column_name") or "")) + if not table_name or b_tbl == name_key(table_name.rsplit(".", 1)[-1]) or calc_mode == "sql_template": + entity_by_col.setdefault(c_key, ent) + + dim_selects: list[str] = [] + group_exprs: list[str] = [] + order_parts: list[str] = [] + tpl_order_parts: list[str] = [] + + grain = str(time_grain or m.get("default_time_grain") or "day").strip().lower() + if grain not in TIME_GRAINS or grain == "none": + grain = "day" + + date_temporal_kind = _column_temporal_kind(source, table_name, date_col) if date_col else "text" + + if group_by_date: + if not date_col: + raise MetricInvalid(f"指标「{m.get('key')}」未配置业务时间列（date_column），无法按日期分组", field="group_by_date") + q_date = _format_col_ref(source, table_name, date_col) + # 当粒度是 day 且列本身是不含时分秒的纯 DATE 类型（非 Oracle DATE / 非 TIMESTAMP）时直接按原列分组 + if grain == "day" and date_temporal_kind == "date": + bare_date_lower = date_col.rsplit(".", 1)[-1].lower() + date_alias = _quote_ident("stat_date" if bare_date_lower != "stat_date" else date_col.rsplit(".", 1)[-1], kind, is_alias=True) + dim_selects.append(f"{q_date} AS {date_alias}" if bare_date_lower != "stat_date" else q_date) + group_exprs.append(q_date) + order_parts.append("1 ASC") + tpl_order_parts.append(f"{q_date} ASC") + else: + d_expr = _date_expr_for_grain(q_date, grain, kind, temporal_kind=date_temporal_kind) + dim_selects.append(f"{d_expr} AS {_quote_ident('period', kind, is_alias=True)}") + group_exprs.append(d_expr) + order_parts.append("1 ASC") + tpl_order_parts.append(f"{d_expr} ASC") + + selected_dims: list[Mapping[str, Any]] = [] + for raw_g in group_by or []: + g_tok = str(raw_g or "").strip().lower() + if not g_tok: + continue + matched_dim = dim_lookup.get(g_tok) + if matched_dim is None: + avail = "、".join(f"{d.get('key')}({d.get('label')})" for d in dims) or "无已声明维度" + raise MetricInvalid(f"指标「{m.get('key')}」不支持按「{raw_g}」分组；可用维度：{avail}", field="group_by") + if matched_dim not in selected_dims: + selected_dims.append(matched_dim) + + for d in selected_dims: + raw_d_col = str(d["column"]) + q_col = _format_col_ref(source, table_name, raw_d_col) + d_key = str(d["key"]) + if raw_d_col.lower() == d_key.lower(): + dim_selects.append(q_col) + else: + dim_selects.append(f"{q_col} AS {_quote_ident(d_key, kind, is_alias=True)}") + group_exprs.append(q_col) + + where_parts: list[str] = [] + if filter_sql: + where_parts.append(f"({filter_sql})") + + d_from = _validate_date_bound(date_from or "", field="date_from") + d_to = _validate_date_bound(date_to or "", field="date_to") + if (d_from or d_to) and not date_col: + raise MetricInvalid(f"指标「{m.get('key')}」未配置业务时间列，不支持按日期区间筛选", field="date_from") + + if d_from: + q_date = _format_col_ref(source, table_name, date_col) + where_parts.append(f"{q_date} >= {_format_date_bound_sql(d_from, kind=kind, temporal_kind=date_temporal_kind)}") + if d_to: + q_date = _format_col_ref(source, table_name, date_col) + if _ISO_DATE_RE.match(d_to) and date_temporal_kind in ("datetime", "datetime_text"): + next_day = (date.fromisoformat(d_to) + timedelta(days=1)).isoformat() + where_parts.append(f"{q_date} < {_format_date_bound_sql(next_day, kind=kind, temporal_kind=date_temporal_kind)}") + else: + where_parts.append(f"{q_date} <= {_format_date_bound_sql(d_to, kind=kind, temporal_kind=date_temporal_kind)}") + + alignments: list[dict[str, Any]] = [] + for raw_fk, raw_fv in (filters or {}).items(): + if raw_fv is None or raw_fv == "" or raw_fv == []: + continue + fk_tok = str(raw_fk or "").strip().lower() + matched_dim = dim_lookup.get(fk_tok) + if matched_dim is None: + avail = "、".join(f"{d.get('key')}({d.get('label')})" for d in dims) or "无已声明过滤维度" + raise MetricInvalid(f"指标「{m.get('key')}」不支持按「{raw_fk}」过滤；可用维度：{avail}", field="filters") + col_name = str(matched_dim["column"]) + bare_col_key = name_key(col_name.rsplit(".", 1)[-1]) + ent = ( + entity_by_key.get(str(matched_dim.get("entity_key") or "").lower()) + or entity_by_col.get(name_key(col_name)) + or entity_by_col.get(bare_col_key) + ) + values_list = list(raw_fv) if isinstance(raw_fv, (list, tuple, set)) else [raw_fv] + resolved_vals: list[Any] = [] + for v in values_list: + if ent is not None: + translated = translate_entity_value(ent, v, to_source=source.name) + if translated is not None: + target_val = ( + str(translated.get("target_value") or "").strip() + if isinstance(translated, Mapping) + else str(translated).strip() + ) + if target_val: + if target_val != str(v): + alignments.append( + { + "dimension": matched_dim["key"], + "entity_key": ent.key, + "entity_label": ent.label, + "input": str(v), + "input_value": v, + "translated": [target_val], + "resolved_code": target_val, + "canonical_value": translated.get("canonical", "") if isinstance(translated, Mapping) else "", + "canonical_label": translated.get("label", "") if isinstance(translated, Mapping) else "", + } + ) + resolved_vals.append(target_val) + continue + resolved_vals.append(v) + q_col = _format_col_ref(source, table_name, col_name) + if len(resolved_vals) == 1: + where_parts.append(f"{q_col} = {_sql_literal(resolved_vals[0])}") + else: + in_list = ", ".join(_sql_literal(x) for x in resolved_vals) + where_parts.append(f"{q_col} IN ({in_list})") + + safe_limit = max(1, min(int(limit or 200), 1000)) + metric_col_alias = _quote_ident(str(m.get("key") or "metric_value"), kind, is_alias=True) + + if calc_mode == "sql_template" and sql_template: + slots_used = {tok.group(1).strip().lower() for tok in _SLOT_TOKEN_RE.finditer(sql_template)} + uses_outer_wrapper = ( + not slots_used + and bool(sql_expr) + and not sql_expr.startswith("[多表模板]") + ) + if uses_outer_wrapper: + with_prefix, main_select = _split_top_level_with_clause(sql_template) + select_parts = [*dim_selects, f"({sql_expr}) AS {metric_col_alias}"] + # 子查询别名不写 AS，兼容 Oracle（Oracle 表或子查询别名写 AS 会报 ORA-00933） + sql = f"SELECT {', '.join(select_parts)} FROM ({main_select}) metric_src" + if with_prefix: + sql = f"{with_prefix} {sql}" + if where_parts: + sql += " WHERE " + " AND ".join(where_parts) + if group_exprs: + sql += " GROUP BY " + ", ".join(group_exprs) + if not order_parts: + order_parts.append(f"{len(select_parts)} DESC") + if order_parts: + sql += " ORDER BY " + ", ".join(order_parts) + sql = _apply_row_limit(sql, safe_limit, kind, force_if_missing=bool(group_exprs)) + return sql, alignments + + if group_exprs and not tpl_order_parts: + tpl_order_parts.append(f"{len(dim_selects) + 1} DESC") + + period_days_sql = _compute_period_days_sql( + source=source, + table_name=table_name, + d_from=d_from, + d_to=d_to, + group_by_date=group_by_date, + grain=grain, + date_col=date_col, + kind=kind, + ) + compiled = _SELECT_DIMS_SLOT_RE.sub( + (f"{', '.join(dim_selects)}, " if dim_selects else ""), + sql_template, + ) + compiled = _WHERE_SLOT_RE.sub( + (f"WHERE {' AND '.join(where_parts)}" if where_parts else ""), + compiled, + ) + compiled = _AND_WHERE_SLOT_RE.sub( + (f"AND ({' AND '.join(where_parts)})" if where_parts else ""), + compiled, + ) + compiled = _GROUP_BY_SLOT_RE.sub( + (f"GROUP BY {', '.join(group_exprs)}" if group_exprs else ""), + compiled, + ) + compiled = _ORDER_BY_SLOT_RE.sub( + (f"ORDER BY {', '.join(tpl_order_parts)}" if tpl_order_parts else ""), + compiled, + ) + compiled = _DATE_FROM_SLOT_RE.sub(_sql_literal(d_from) if d_from else "NULL", compiled) + compiled = _DATE_TO_SLOT_RE.sub(_sql_literal(d_to) if d_to else "NULL", compiled) + compiled = _PERIOD_DAYS_SLOT_RE.sub(period_days_sql, compiled) + compiled = _LIMIT_SLOT_RE.sub(str(safe_limit), compiled) + compiled = re.sub(r"[ \t]+\n", "\n", compiled).strip() + compiled = _apply_row_limit( + compiled, + safe_limit, + kind, + force_if_missing=bool(group_exprs and "limit" not in slots_used), + ) + return compiled, alignments + + select_parts = [*dim_selects, f"({sql_expr}) AS {metric_col_alias}"] + sql = f"SELECT {', '.join(select_parts)} FROM {_format_table_ref(source, table_name)}" + if where_parts: + sql += " WHERE " + " AND ".join(where_parts) + if group_exprs: + sql += " GROUP BY " + ", ".join(group_exprs) + if not order_parts: + order_parts.append(f"{len(select_parts)} DESC") + if order_parts: + sql += " ORDER BY " + ", ".join(order_parts) + sql = _apply_row_limit(sql, safe_limit, kind, force_if_missing=bool(group_exprs)) + return sql, alignments + + +async def probe_latest_metric_date( + source: DataSource, + *, + table_name: str, + date_column: str, + filter_sql: str = "", +) -> str | None: + """当某日查询无结果或聚合值为 NULL 时，快速探查底表最新可用业务日期（限时 3s）。""" + if not date_column or not table_name: + return None + bare_date = date_column.rsplit(".", 1)[-1] + q_date = _format_col_ref(source, table_name, bare_date) + q_table = _format_table_ref(source, table_name) + sql = f"SELECT MAX({q_date}) AS latest_date FROM {q_table}" + if filter_sql and "." not in filter_sql: + sql += f" WHERE ({filter_sql})" + try: + res = await run_query(source, sql, limits=QueryLimits(timeout_seconds=3, max_rows=1)) + if res.rows and res.rows[0] and res.rows[0][0] is not None: + return str(res.rows[0][0]) + except Exception: # noqa: BLE001 + return None + return None + + +def is_empty_metric_result(rows: Sequence[Sequence[Any]], *, has_groups: bool) -> bool: + if not rows: + return True + if not has_groups and len(rows) == 1 and all(cell is None for cell in rows[0]): + return True + return False + + +async def preview_metric( + session: AsyncSession, + *, + metric_id: str | None = None, + definition: Mapping[str, Any] | None = None, + date_from: str | None = None, + date_to: str | None = None, + group_by_date: bool = False, + time_grain: str | None = None, + group_by: Sequence[str] | None = None, + filters: Mapping[str, Any] | None = None, + limit: int = 50, +) -> dict[str, Any]: + """在线试算一条已保存或未保存草稿状态的指标定义（搭建者工作台预览用）。""" + if metric_id: + row = await get_metric(session, metric_id) + ds = await _resolve_source(session, row.source_id) + normalized = metric_to_dict(row, source_name=ds.name) + elif definition is not None: + ds, normalized = await _normalize_metric_payload(session, definition, existing=None) + else: + raise MetricInvalid("请提供要试算的 metric_id 或 definition") + + entities = await list_entities(session) + sql, alignments = compile_metric_sql( + normalized, + source=ds, + entities=entities, + date_from=date_from, + date_to=date_to, + group_by_date=group_by_date, + time_grain=time_grain, + group_by=group_by, + filters=filters, + limit=limit, + ) + started = time.perf_counter() + result = await run_query(ds, sql, limits=QueryLimits(timeout_seconds=15, max_rows=max(1, min(int(limit or 50), 200)))) + elapsed_ms = int((time.perf_counter() - started) * 1000) + payload = result.to_payload() + + has_groups = bool(group_by_date or group_by) + latest_date: str | None = None + if is_empty_metric_result(result.rows, has_groups=has_groups) and normalized.get("date_column"): + latest_date = await probe_latest_metric_date( + ds, + table_name=str(normalized["table_name"]), + date_column=str(normalized["date_column"]), + filter_sql=str(normalized.get("filter_sql") or ""), + ) + + return { + "sql": sql, + "source_name": ds.name, + "columns": payload["columns"], + "rows": payload["rows"], + "row_count": payload["row_count"], + "truncated": payload["truncated"], + "elapsed_ms": elapsed_ms, + "entity_alignments": alignments, + "latest_available_date": latest_date, + } + + +# -------------------------------------------------------------------------- +# AI 辅助从数据目录与业务文档批量提取指标卡片 +# -------------------------------------------------------------------------- + +SUGGEST_SYSTEM_PROMPT = """你是 AgentLab 的「语义指标建模专家」。请根据给定的数据源物理表结构、数据目录（中文名、粒度、业务日期、有效过滤条件、列角色与符号映射）以及可选的业务指标文档，提取高价值、可直接查询的「独立语义指标卡片（SemanticMetric）」。 + +硬性纪律： +1. 只输出一个合法 JSON 对象，格式为 {"metrics": [...]}，不要输出任何额外解释文字。 +2. 表名（table_name）、时间列（date_column）、维度列（dimensions[].column）及表达式或模板引用的所有物理表名与列名，必须 100% 逐字存在于输入提供的表结构中，严禁臆造不存在的表或列。若文档里某指标注明「暂无数据源」，不要生成该指标。 +3. 标有「[遮罩列]」的敏感列严禁出现在 sql_expression、sql_template、filter_sql 或 dimensions 中。 +4. 支持两种计算模式（calc_mode）： + - 单表模式（"calc_mode": "expression"，默认优先使用）：指标可由单张物理表聚合得到时使用。sql_expression 写单表聚合表达式（如 SUM(total_sys)、ROUND(SUM(qualified_qty) * 100.0 / NULLIF(SUM(total_qty), 0), 2)），严禁写完整 SELECT 或子查询；sql_template 置空串 ""。注意文档提示的存储陷阱（如某比率列已存为百分数 99.65 时，切勿再乘 100 或重复除 100）。 + - 多表 / 参数化模板模式（"calc_mode": "sql_template"）：当口径依赖跨表 JOIN、WITH CTE 预聚合、兜底系数（如 COALESCE(mj.mjxs, 0.5)）或动态统计周期天数（如 TEEP 自然日历秒数 = {{period_days}} * 86400 * COUNT(DISTINCT s.line_code)）时使用！ + · sql_template 写单条只读 SELECT 或 WITH ... SELECT 语句，不含分号与 SQL 注释； + · 仅允许使用以下安全槽位：{{select_dims}}（放在 SELECT 开头用于动态展开分组列）、{{and_where}}（放在 WHERE 1=1 之后承接日期与维度筛选）、{{where}}、{{group_by}}、{{order_by}}、{{period_days}}、{{date_from}}、{{date_to}}、{{limit}}； + · 多表模板下，date_column 与 dimensions[].column 必须带主表别名前缀（如 "s.stat_date"、"s.line_code"），table_name 填主事实表名；若表旁标注了「SQL 全名：」，在 sql_template 的 FROM / JOIN 中务必使用该带 schema 前缀的全名； + · 严格遵循目标数据源方言语法：例如当方言为 oracle 时，严禁写 LIMIT（行数限制由系统自动套外层 ROWNUM）、严禁写 CAST(... AS TEXT)（转字符用 TO_CHAR、转数值用 TO_NUMBER）、表或子查询别名不要写 AS（写 FROM tbl t）。 +5. 若数据目录注明某度量列为「存量（stock，不可跨期直接求和）」，优先使用 AVG(...) 或在 description 中明确标注「存量指标，多日查询请开启按日分组查看趋势」。 +6. 若物理列旁标注了「已绑定标准实体：」，在对应 dimension 的 entity_key 字段填入该实体键。 +7. 避免与「已存在的指标标识」重复；提取文档或目录中最核心的 4～16 个业务指标（单表与多表复杂指标均可覆盖）。 + +单表指标示例： +{ + "key": "zy_daily_footfall_sys", + "name": "张园日客流（系统去重）", + "aliases": ["客流", "总客流"], + "description": "统计每日系统去重客流人次", + "unit": "人次", + "calc_mode": "expression", + "table_name": "t_zy_daily_stat", + "sql_expression": "SUM(total_sys)", + "sql_template": "", + "date_column": "stat_date", + "filter_sql": "", + "dimensions": [{"key": "zone", "label": "区域", "column": "zone_code", "entity_key": ""}] +} + +多表 / 参数化 SQL 模板指标示例： +{ + "key": "oee_multi_table", + "name": "设备综合效率 OEE（跨表精算）", + "aliases": ["OEE", "综合效率"], + "description": "联合班次日报、不合格品记录与模具系数表精算 OEE；含判异红线说明", + "unit": "%", + "calc_mode": "sql_template", + "table_name": "mes_shift_daily", + "sql_expression": "", + "sql_template": "WITH mj AS (SELECT prod_code, AVG(mjxs) AS mjxs FROM mes_jzhmjxs GROUP BY prod_code), sc AS (SELECT line_code, stat_date, SUM(scrap_qty) AS bad_qty FROM mes_bhgsj GROUP BY line_code, stat_date) SELECT {{select_dims}} ROUND(SUM(s.good_qty * COALESCE(mj.mjxs, 0.5)) * 100.0 / NULLIF(SUM(s.good_qty + COALESCE(sc.bad_qty, 0)), 0), 2) AS oee_multi_table FROM mes_shift_daily s LEFT JOIN mj ON mj.prod_code = s.prod_code LEFT JOIN sc ON sc.line_code = s.line_code AND sc.stat_date = s.stat_date WHERE 1=1 {{and_where}} {{group_by}} {{order_by}}", + "date_column": "s.stat_date", + "filter_sql": "", + "dimensions": [{"key": "line_code", "label": "产线", "column": "s.line_code", "entity_key": ""}] +} +""" + + +def _parse_suggested_metrics(raw_text: str) -> list[dict[str, Any]]: + text = (raw_text or "").strip() + if not text: + return [] + m = _FENCE_RE.search(text) + candidate = m.group(1).strip() if m else text + for snippet in (candidate, text): + start = snippet.find("{") + end = snippet.rfind("}") + if start >= 0 and end > start: + try: + obj = json.loads(snippet[start : end + 1]) + if isinstance(obj, Mapping) and isinstance(obj.get("metrics"), list): + return [x for x in obj["metrics"] if isinstance(x, Mapping)] + except json.JSONDecodeError: + continue + return [] + + +async def suggest_metrics_from_source( + session: AsyncSession, + *, + source_id: str, + table_names: Sequence[str] | None = None, + document_ids: Sequence[str] | None = None, + note: str = "", +) -> dict[str, Any]: + """基于数据源目录、标准业务实体绑定与可选知识库文档，智能生成已通过语法与物理列核验的候选指标卡片。""" + ds = await _resolve_source(session, source_id) + cache = ds.schema_cache if isinstance(ds.schema_cache, Mapping) else {} + tables = cache.get("tables") if isinstance(cache, Mapping) else None + if not isinstance(tables, Mapping) or not tables: + raise MetricInvalid(f"数据源「{ds.name}」尚未探查表结构，请先在数据源设置中探查结构", field="source_id") + + entries = await catalog.read_catalog(session, ds.id) + masked_set = {name_key(c) for c in masked_columns(ds.options)} + entities = await list_entities(session) + existing = await list_metrics(session, source_id=ds.id) + existing_keys = {m.key for m in existing} + + # (table_short_key, col_key) -> entity_key + col_entity: dict[tuple[str, str], str] = {} + for ent in entities: + for b in confirmed_bindings(ent): + if name_key(str(b.get("source_name") or "")) == name_key(ds.name): + t_short = name_key(str(b.get("table_name") or "").rsplit(".", 1)[-1]) + c_key = name_key(str(b.get("column_name") or "")) + col_entity[(t_short, c_key)] = ent.key + + selected_tables: list[str] = [] + if table_names: + for raw_t in table_names: + resolved = catalog.resolve_table_name(cache, str(raw_t)) + if resolved and resolved not in selected_tables: + selected_tables.append(resolved) + else: + # 优先排有数据目录说明或名称含 stat/daily/fact/order/summary/detail 的业务表（最多 30 张控制上下文） + def _table_priority(tname: str) -> tuple[int, str]: + has_cat = 0 if (tname in entries and catalog.visible_notes(entries[tname].notes)) else 1 + return (has_cat, tname) + + selected_tables = sorted(tables.keys(), key=_table_priority)[:30] + + lines: list[str] = [f"数据源：{ds.name}（方言：{ds.kind}）"] + if (ds.kind or "").strip().lower() == "oracle": + lines.append("方言注意：Oracle 严禁写 LIMIT、严禁写 CAST(x AS TEXT)、表或子查询别名不写 AS；动态周期天数请直接用 {{period_days}} 槽位。") + if existing_keys: + lines.append(f"已存在的指标标识（请勿重复）：{', '.join(sorted(existing_keys))}") + lines.append("\n【物理表与数据目录】") + for tname in selected_tables: + tinfo = tables.get(tname) or {} + notes = catalog.visible_notes(entries[tname].notes) if tname in entries else {} + t_label = (notes.get("label") or {}).get("value") if isinstance(notes.get("label"), Mapping) else "" + t_grain = (notes.get("grain") or {}).get("value") if isinstance(notes.get("grain"), Mapping) else "" + b_date_raw = (notes.get("business_date") or {}).get("value") if isinstance(notes.get("business_date"), Mapping) else "" + if isinstance(b_date_raw, Mapping): + b_col = str(b_date_raw.get("column") or "").strip() + b_rule = str(b_date_raw.get("rule") or "").strip() + b_date = f"{b_col}（{b_rule}）" if (b_col and b_rule) else (b_col or b_rule) + else: + b_date = str(b_date_raw or "").strip() + v_filt = (notes.get("valid_filter") or {}).get("value") if isinstance(notes.get("valid_filter"), Mapping) else "" + head_bits = [f"表 {tname}"] + qualified_t = str(tinfo.get("qualified") or "").strip() + if qualified_t and qualified_t != tname: + head_bits.append(f"SQL 全名：{qualified_t}") + if t_label: + head_bits.append(f"中文名：{t_label}") + if t_grain: + head_bits.append(f"粒度：{t_grain}") + if b_date: + head_bits.append(f"业务日期列：{b_date}") + if v_filt: + head_bits.append(f"有效过滤：{v_filt}") + lines.append("- " + " ｜ ".join(head_bits)) + col_notes = notes.get("columns") if isinstance(notes.get("columns"), Mapping) else {} + t_short = name_key(tname.rsplit(".", 1)[-1]) + col_descs: list[str] = [] + for col in (tinfo.get("columns") or [])[:45]: + if not isinstance(col, Mapping) or not col.get("name"): + continue + cname = str(col["name"]) + ctype = str(col.get("type") or "") + if name_key(cname) in masked_set: + col_descs.append(f" · {cname} ({ctype}) [遮罩列]") + continue + cn = col_notes.get(cname) if isinstance(col_notes, Mapping) else None + c_label = (cn.get("label") or {}).get("value") if isinstance(cn, Mapping) and isinstance(cn.get("label"), Mapping) else "" + c_unit = (cn.get("unit") or {}).get("value") if isinstance(cn, Mapping) and isinstance(cn.get("unit"), Mapping) else "" + c_measure = (cn.get("measure") or {}).get("value") if isinstance(cn, Mapping) and isinstance(cn.get("measure"), Mapping) else "" + ent_key = col_entity.get((t_short, name_key(cname)), "") + extra_bits = [b for b in ( + c_label, + f"单位：{c_unit}" if c_unit else "", + f"度量：{c_measure}" if c_measure else "", + f"已绑定标准实体：{ent_key}" if ent_key else "", + ) if b] + suffix = f" — {'，'.join(extra_bits)}" if extra_bits else "" + col_descs.append(f" · {cname} ({ctype}){suffix}") + lines.extend(col_descs) + + doc_titles: list[str] = [] + if document_ids: + docs = list( + (await session.execute(select(Document).where(Document.id.in_([str(d) for d in document_ids])))).scalars().all() + ) + if docs: + lines.append("\n【参考业务口径文档】") + budget = 16_000 + for doc in docs: + body = str(doc.content or "").strip() + if not body or budget <= 0: + continue + doc_label = str(doc.title or doc.source or doc.id) + doc_titles.append(doc_label) + snippet = body[: min(len(body), budget)] + budget -= len(snippet) + lines.append(f"--- 文档《{doc_label}》 ---\n{snippet}") + + if note and note.strip(): + lines.append(f"\n【用户补充要求】\n{note.strip()[:1000]}") + + model, model_id = await catalog_assistant.resolve_model(session) + msgs = [ + SystemMessage(content=SUGGEST_SYSTEM_PROMPT), + HumanMessage(content="\n".join(lines)), + ] + reply = await model.ainvoke(msgs) + raw_candidates = _parse_suggested_metrics(message_text(reply)) + + accepted: list[dict[str, Any]] = [] + skipped: list[dict[str, str]] = [] + seen_batch_keys: set[str] = set(existing_keys) + for raw in raw_candidates: + candidate = {**raw, "source_id": ds.id, "source": "ai", "status": "enabled"} + raw_k = str(candidate.get("key") or "").strip().lower() + if raw_k in seen_batch_keys: + skipped.append({"key": raw_k, "reason": "与已有指标或同批候选标识重复"}) + continue + try: + _, norm = await _normalize_metric_payload(session, candidate, existing=None) + seen_batch_keys.add(norm["key"]) + accepted.append({**norm, "source_name": ds.name}) + except Exception as exc: # noqa: BLE001 + skipped.append({"key": raw_k or str(candidate.get("name") or "?"), "reason": str(exc)}) + + return { + "source_id": ds.id, + "source_name": ds.name, + "model_id": model_id, + "documents_used": doc_titles, + "candidates": accepted, + "skipped": skipped, + } diff --git a/backend/app/db/base.py b/backend/app/db/base.py index fee6da95..3240545b 100644 --- a/backend/app/db/base.py +++ b/backend/app/db/base.py @@ -493,6 +493,9 @@ _COLUMN_MIGRATIONS: list[tuple[str, str, str]] = [ # 增量重导入的持久对齐键（K11）：存量工作项回填空串——空串不参与按对齐键的匹配， # 不会被误当成「同一项」串上旧决定（宁可不匹配，也不按位置猜） ("caliber_import_items", "align_key", "VARCHAR(64) DEFAULT ''"), + # 独立语义指标库升级：支持多表 JOIN / CTE 参数化 SQL 模板口径，存量卡片默认保持单表聚合（expression） + ("semantic_metrics", "calc_mode", "VARCHAR(24) DEFAULT 'expression'"), + ("semantic_metrics", "sql_template", "TEXT DEFAULT ''"), ] #: 迁移补上的列要建的索引：(索引名, 表, 列或多列)。create_all 不碰已有的表，表上新列的索引也就不会建。 diff --git a/backend/app/db/models.py b/backend/app/db/models.py index 161d814e..5147775f 100644 --- a/backend/app/db/models.py +++ b/backend/app/db/models.py @@ -1432,3 +1432,6 @@ from app.db import models_cluster # noqa: E402,F401 # 指标本体穿透（W1-a）：标准业务实体与跨源映射 from app.db import models_ontology # noqa: E402,F401 from app.db.models_ontology import CanonicalEntity # noqa: E402,F401 +# 独立语义指标库（即存即用指标卡片）与文档拆解后台任务：与受管工作流口径并存 +from app.db import models_metrics # noqa: E402,F401 +from app.db.models_metrics import SemanticDecomposeJob, SemanticMetric # noqa: E402,F401 diff --git a/backend/app/db/models_metrics.py b/backend/app/db/models_metrics.py new file mode 100644 index 00000000..68423b85 --- /dev/null +++ b/backend/app/db/models_metrics.py @@ -0,0 +1,104 @@ +"""独立语义指标库（SemanticMetric）与口径文档拆解后台任务（SemanticDecomposeJob）持久化模型。 + +由 app/db/models.py 末尾整模块导入，create_all 自动建表，存量表新列由 app/db/base.py:_COLUMN_MIGRATIONS 补齐（兼容 SQLite 与 PostgreSQL）。 + +设计目的： +- 解耦原先强绑定在「多节点工作流 + 单行单元格抽取 + 多重发布门禁」上的繁重口径模式； +- 同时支持「单表聚合表达式（expression）」与「多表 JOIN / CTE 参数化 SQL 模板（sql_template）」两种确定性计算模式； +- status="enabled" 即存即生效，直接供问数入口语义预命中（chat/grounding.py）与内置工具 metric_query 按任意日期区间或业务维度动态聚合取数； +- SemanticDecomposeJob 支持后台异步执行「从目录与文档智能提取指标」与「文档一键拆解工作流（字段目录补丁 + 指标口径 + 标准实体 + 知识库归档）」，实时持久化阶段进度与待审阅结果。 +""" +from __future__ import annotations + +from datetime import datetime +from typing import Any + +from sqlalchemy import Boolean, Integer, String, Text +from sqlalchemy.orm import Mapped, mapped_column + +from app.db.base import Base, TimestampMixin, UTCDateTime, new_id + + +class SemanticMetric(Base, TimestampMixin): + """独立语义指标卡片定义（即存即用、支持单表聚合与多表参数化 SQL 模板）。""" + + __tablename__ = "semantic_metrics" + + id: Mapped[str] = mapped_column(String(32), primary_key=True, default=new_id) + #: 机器稳定标识（小写字母开头，小写字母/数字/下划线，最长 64 字符，如 zy_daily_footfall_sys） + key: Mapped[str] = mapped_column(String(64), unique=True, index=True) + #: 指标中文名称（如「张园日客流（系统去重）」） + name: Mapped[str] = mapped_column(String(200)) + #: 业务别名 / 同义词列表（如 ["客流", "总客流", "日客流量"]） + aliases: Mapped[list[Any]] = mapped_column(default=list) + #: 口径业务说明与注意事项（如统计口径、去重规则、T+1 延迟说明） + description: Mapped[str] = mapped_column(Text, default="") + #: 展示单位（如「人次」「元」「%」） + unit: Mapped[str] = mapped_column(String(40), default="") + #: 归属数据源 id（data_sources.id） + source_id: Mapped[str] = mapped_column(String(32), index=True) + #: 计算模式：expression（单表聚合表达式） | sql_template（多表 JOIN / CTE 参数化 SQL 模板） + calc_mode: Mapped[str] = mapped_column(String(24), default="expression") + #: 主物理底表名（expression 模式下为查询底表；sql_template 模式下为主锚点表） + table_name: Mapped[str] = mapped_column(String(255)) + #: 只读聚合 SQL 表达式（expression 模式必填；sql_template 模式可作外层聚合表达式或展示摘要） + sql_expression: Mapped[str] = mapped_column(Text) + #: 多表 / 参数化只读 SQL 模板（sql_template 模式使用，支持 {{select_dims}}、{{and_where}}、{{group_by}}、{{period_days}} 等槽位） + sql_template: Mapped[str] = mapped_column(Text, default="") + #: 业务时间列名（可选，如 stat_date 或多表别名 s.stat_date；为空表示无时间序列维度） + date_column: Mapped[str] = mapped_column(String(128), default="") + #: 默认有效记录过滤条件（可选 WHERE 子表达式，如 status = 1 AND is_deleted = 0） + filter_sql: Mapped[str] = mapped_column(Text, default="") + #: 允许按维度下钻与过滤的列定义：[{key, label, column, entity_key?}] + dimensions: Mapped[list[Any]] = mapped_column(default=list) + #: 默认时间聚合粒度：day | month | hour | none + default_time_grain: Mapped[str] = mapped_column(String(24), default="day") + #: 状态：enabled（启用，直接参与问数预命中与查询） | draft（草稿） | disabled（停用） + status: Mapped[str] = mapped_column(String(24), default="enabled", index=True) + #: 来源：human（人工创建） | ai（目录/文档智能提取） + source: Mapped[str] = mapped_column(String(24), default="human") + #: 乐观锁版本号（从 1 起，每次修改加 1） + version: Mapped[int] = mapped_column(Integer, default=1) + created_by: Mapped[str | None] = mapped_column(String(100), default=None) + updated_by: Mapped[str | None] = mapped_column(String(100), default=None) + + +class SemanticDecomposeJob(Base, TimestampMixin): + """后台口径文档一键拆解与智能提取指标任务（支持实时阶段进度、后台运行与审阅一键应用）。""" + + __tablename__ = "semantic_decompose_jobs" + + id: Mapped[str] = mapped_column(String(32), primary_key=True, default=new_id) + #: queued | running | succeeded | failed | cancelled + status: Mapped[str] = mapped_column(String(24), default="queued", index=True) + #: 目标数据源 id + source_id: Mapped[str] = mapped_column(String(32), index=True) + source_name: Mapped[str] = mapped_column(String(120), default="") + #: 拆解范围列表：["catalog", "metrics", "entities"] + scopes: Mapped[list[Any]] = mapped_column(default=list) + #: 关联的知识库文档 id 列表 + document_ids: Mapped[list[Any]] = mapped_column(default=list) + #: 上传附件 id 列表（entity_docs 暂存解析文本） + attachment_ids: Mapped[list[Any]] = mapped_column(default=list) + #: 直接粘贴的业务文档标题与正文 + doc_title: Mapped[str] = mapped_column(String(240), default="") + doc_text: Mapped[str] = mapped_column(Text, default="") + #: 是否同步将粘贴/上传的文档存入知识库（保留判异阈值与边界契约供问数检索） + save_to_kb: Mapped[bool] = mapped_column(Boolean, default=True) + #: 用户补充要求 + note: Mapped[str] = mapped_column(Text, default="") + #: 当前阶段标识与文案：preparing | syncing_kb | extracting_catalog | extracting_metrics | extracting_entities | done + phase: Mapped[str] = mapped_column(String(48), default="preparing") + phase_label: Mapped[str] = mapped_column(String(200), default="正在准备数据源与文档上下文") + #: 0 ~ 100 百分比进度 + progress: Mapped[int] = mapped_column(Integer, default=0) + #: 分步轨迹：[{key, label, status, detail, elapsed_ms}] + steps: Mapped[list[Any]] = mapped_column(default=list) + #: 结构化拆解产物：{kb_document, catalog_patches, metric_candidates, metric_skipped, entity_operations, guardrail_notes, applied} + result: Mapped[dict[str, Any]] = mapped_column(default=dict) + #: 简要汇总计数：{catalog_count, metric_count, entity_count, kb_synced} + summary: Mapped[dict[str, Any]] = mapped_column(default=dict) + error: Mapped[str | None] = mapped_column(Text, default=None) + started_by: Mapped[str | None] = mapped_column(String(100), default=None) + started_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None) + finished_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None) diff --git a/backend/app/main.py b/backend/app/main.py index ac4ae1d3..47823f3c 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -33,6 +33,8 @@ from app.api import chat_turns as chat_turns_api from app.api import caliber_import as caliber_import_api # 指标本体穿透（W1-a）：标准业务实体与跨口径指标关系网 from app.api import ontology as ontology_api +# 独立语义指标库（即存即用指标卡片与在线试算） +from app.api import semantic_metrics as semantic_metrics_api from app.api.coded import CodedHTTPException, coded_handler from app.api.deps import chat_scope, evidence_scope, private_run_scope, require_role, role_by_method, run_scope from app.auth.keys import key_gate @@ -143,7 +145,7 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: finally: # 先停调度，再停运行：停运行的途中不该再有新运行被发起。起草、剖析任务停下来标成中断，下次启动不会再续 await scheduler.stop() - from app.data import catalog_jobs, catalog_profile_jobs, entity_discovery, introspect_jobs + from app.data import catalog_jobs, catalog_profile_jobs, entity_discovery, introspect_jobs, semantic_decompose from app.chat import job as chat_job from app.eval import runner as eval_runner, snapshots as eval_snapshots @@ -155,6 +157,7 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: await catalog_profile_jobs.manager.shutdown() await introspect_jobs.manager.shutdown() await entity_discovery.shutdown() + await semantic_decompose.shutdown() # 问数据的一轮停下不改那几行（下次启动、或者别的进程照「属主不在了」的规矩接手），也要在运行之前停 await chat_job.shutdown() # 评测要在运行之前停：它停下时会取消自己发起的作答、算期望的运行，再标中断 @@ -249,7 +252,7 @@ def _register_reapers() -> None: from app.blueprints import publish_plan from app.chat import job as chat_job from app.cluster import reaper - from app.data import catalog_jobs, catalog_profile_jobs, entity_discovery, introspect_jobs + from app.data import catalog_jobs, catalog_profile_jobs, entity_discovery, introspect_jobs, semantic_decompose from app.eval import runner as eval_runner, snapshots as eval_snapshots from app.process import actions, tasks from app.ledger import jobs as draft_jobs, reimport as import_jobs @@ -266,6 +269,7 @@ def _register_reapers() -> None: ("数据剖析任务", lambda live: catalog_profile_jobs.startup(live=live)), ("探查任务", lambda live: introspect_jobs.startup(live=live)), ("候选实体扫描任务", lambda live: entity_discovery.startup(live=live)), + ("口径拆解任务", lambda live: semantic_decompose.startup(live=live)), ("影子起草", lambda live: eval_snapshots.startup(live=live)), ("评测任务", lambda live: eval_runner.startup(live=live)), ("问数据的一轮", _chat_reaper(chat_job)), @@ -436,6 +440,8 @@ for router, guard in ( (caliber_import_api.router, _BUILDER), # 指标本体穿透（W1-a）：标准业务实体与指标本体图谱（搭建者及以上） (ontology_api.router, _BUILDER), + # 独立语义指标库（即存即用指标卡片）：同口径与本体工作台权限（搭建者及以上） + (semantic_metrics_api.router, _BUILDER), # 下一轮第 2 波：评测只给搭建者；访问密钥登录即可，接口里再按本人 / 管理员判 (evals_api.router, _BUILDER), # 口径账本里按答案出具、对账的（第 3 波）：登录即可，接口里按运行和应用授权判 diff --git a/backend/app/tools/metrics.py b/backend/app/tools/metrics.py new file mode 100644 index 00000000..2e5f0710 --- /dev/null +++ b/backend/app/tools/metrics.py @@ -0,0 +1,172 @@ +"""独立语义指标库内置只读工具：metric_query。 + +设计原则： +1. 只执行 status="enabled" 的已启用指标卡片（SemanticMetric），确定性编译为单条只读 SELECT 语句。 +2. 底层委托对应数据源的 `db_query__<源名>` 工具执行：自动继承上传快照版本锁定（data_versions）、 + 普通使用者结果遮罩（MASK_RESULTS_OPTION）、只读守卫（READ_ONLY_OPTION）、数据目录 SQL 检查（SqlChecker） + 以及内容寻址查询快照落盘（query_snapshot），运行证据链与 `db_query__` 完全一致。 +3. 支持按日期区间（date_from / date_to）、时间序列趋势（group_by_date / time_grain）、业务维度下钻（group_by） + 与标准业务实体值过滤（filters，自动把业务别名翻译为目标源底层编码）。 +4. 当单日查询返回空或全 NULL 且配置了业务时间列时，在同一次调用结果内附带底表最新可用日期 `latest_available_date`， + 消除 Agent 多轮探查开销。 +""" +from __future__ import annotations + +import json +from typing import Any + +from pydantic import BaseModel, Field, model_validator + +from app.data.entities import list_entities +from app.data.semantic_metrics import ( + MetricInvalid, + MetricNotFound, + compile_metric_sql, + get_metric, + is_empty_metric_result, + probe_latest_metric_date, +) +from app.db.base import SessionLocal +from app.db.models import DataSource +from app.tools.datasource import QUERY_PREFIX +from app.tools.registry import ToolContext, ToolGuide, Versions, build_tools, register + +METRIC_QUERY = "metric_query" + +METRIC_GUIDE = ToolGuide( + id="semantic_metric.method@1", + text=( + "【语义指标查询纪律】\n" + "当用户提问命中已启用的独立语义指标（metric_key）时，优先调用 `metric_query` 获取标准口径结果：\n" + "- 查某一天传 `date_from` 与 `date_to` 为同一日期；查趋势或近 N 日明细传区间并设 `group_by_date=true`；\n" + "- 按业务维度拆分传 `group_by`（填指标声明的维度 key）；按实体或维度筛选传 `filters`（可直接传业务名称，系统自动翻译底层编码）；\n" + "- 若返回结果附带 `latest_available_date`（例如当日 T+1 尚未入库），可据此说明当日暂无数据或按需再查最新日期。" + ), +) + + +class MetricQueryArgs(BaseModel): + """按已启用的语义指标卡片（SemanticMetric）执行确定性聚合取数。""" + + metric: str = Field(description="要查询的语义指标 key、指标 id 或唯一指标名称") + date_from: str | None = Field(default=None, description="可选：起始业务日期 YYYY-MM-DD（闭区间）") + date_to: str | None = Field(default=None, description="可选：结束业务日期 YYYY-MM-DD（闭区间）") + group_by_date: bool = Field(default=False, description="是否按时间序列分组输出趋势（如逐日/逐月）") + time_grain: str | None = Field( + default=None, + description="可选：时间分组粒度 day | month | hour（配合 group_by_date=true 使用，默认按指标设定）", + ) + group_by: list[str] = Field( + default_factory=list, + description="可选：按指标声明的维度 key 分组下钻，例如 ['area', 'store']", + ) + filters: dict[str, Any] = Field( + default_factory=dict, + description="可选：按指标声明的维度过滤，例如 {'area': '西区'}（关联了标准业务实体的维度会自动翻译物理编码）", + ) + limit: int = Field(default=200, ge=1, le=1000, description="分组查询最多返回行数，默认 200") + + @model_validator(mode="before") + @classmethod + def _accept_metric_key_aliases(cls, data: Any) -> Any: + if isinstance(data, dict) and "metric" not in data: + for alt in ("metric_key", "key", "metric_id"): + if alt in data and data[alt] is not None: + copied = dict(data) + copied["metric"] = copied.pop(alt) + return copied + return data + + +@register( + name=METRIC_QUERY, + description="按已启用的独立语义指标卡片查询标准口径数值、时间趋势或维度拆解明细（自动应用有效记录过滤与标准业务实体编码转换）", + category="data", + args_schema=MetricQueryArgs, + dangerous=False, + needs_context=True, + guide=METRIC_GUIDE, +) +async def run_metric_query( + ctx: ToolContext, + metric: str, + date_from: str | None = None, + date_to: str | None = None, + group_by_date: bool = False, + time_grain: str | None = None, + group_by: list[str] | None = None, + filters: dict[str, Any] | None = None, + limit: int = 200, +) -> str: + async with SessionLocal() as session: + try: + row = await get_metric(session, metric) + except MetricNotFound as exc: + return f"查询失败：{exc}" + if row.status != "enabled": + return f"查询失败：指标「{row.name}（{row.key}）」当前状态为 {row.status}，未启用" + ds = await session.get(DataSource, row.source_id) + if ds is None or not ds.enabled: + return f"查询失败：指标「{row.name}」关联的数据源不可用或已停用" + + entities = await list_entities(session) + try: + sql, alignments = compile_metric_sql( + row, + source=ds, + entities=entities, + date_from=date_from, + date_to=date_to, + group_by_date=group_by_date, + time_grain=time_grain, + group_by=group_by or [], + filters=filters or {}, + limit=limit, + ) + except MetricInvalid as exc: + return f"查询失败：{exc}" + + sub_ctx = ctx + if sub_ctx.data_versions is Versions.UNSET and not sub_ctx.run_id: + # 工具试用等非运行上下文默认读当前版本 + from dataclasses import replace + + sub_ctx = replace(ctx, data_versions=Versions.CURRENT) + + query_tool_name = f"{QUERY_PREFIX}{ds.name}" + built = await build_tools([query_tool_name], sub_ctx, session=session) + if not built: + return f"查询失败：无法初始化数据源「{ds.name}」的查询工具" + + raw_res = await built[0].ainvoke({"sql": sql, "limit": limit}) + raw_str = raw_res if isinstance(raw_res, str) else json.dumps(raw_res, ensure_ascii=False, default=str) + if not raw_str.lstrip().startswith("{"): + return raw_str + + try: + payload = json.loads(raw_str) + except ValueError: + return raw_str + + payload["metric"] = { + "id": row.id, + "key": row.key, + "name": row.name, + "unit": row.unit or "", + } + if alignments: + payload["entity_alignments"] = alignments + + has_groups = bool(group_by_date or group_by) + rows_data = payload.get("rows") if isinstance(payload.get("rows"), list) else [] + if is_empty_metric_result(rows_data, has_groups=has_groups) and row.date_column: + latest_date = await probe_latest_metric_date( + ds, + table_name=row.table_name, + date_column=row.date_column, + filter_sql=row.filter_sql or "", + ) + if latest_date: + payload["latest_available_date"] = latest_date + + return json.dumps(payload, ensure_ascii=False, default=str) diff --git a/backend/app/tools/registry.py b/backend/app/tools/registry.py index 1e353571..929c819d 100644 --- a/backend/app/tools/registry.py +++ b/backend/app/tools/registry.py @@ -157,6 +157,7 @@ def all_specs() -> dict[str, ToolSpec]: import app.tools.builtin # noqa: F401 触发内置工具注册 import app.ledger.querycore # noqa: F401 触发 caliber_query（K10 的父 Run 工具入口）注册 import app.tools.ontology # noqa: F401 触发 ontology_neighbors / source_health 注册 + import app.tools.metrics # noqa: F401 触发 metric_query（独立语义指标查询）注册 return dict(_REGISTRY) diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index 3da34a26..3d11d180 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -42,6 +42,7 @@ import { blueprints, files, personal, templates } from './blueprints' // 问数据的一轮在服务端跑（编排完善第 1 波）：发起、看、停，实时流在 chatTurns.ts 的 streamChatTurn import { chatTurns } from './chatTurns' import { ontology } from './ontology' +import { semanticMetrics } from './semanticMetrics' // 请求层搬到了 http.ts（见那边的说明）。原先从这里导出的照旧从这里导出，调用方不用改； // 请求函数也一并转出，给不在 api/ 下、又想直接发请求的地方用。api/ 下的命名空间文件请直接从 http.ts 导入 @@ -1009,6 +1010,8 @@ export const api = { chatTurns, // 业务本体：标准业务实体与跨口径指标拓扑 ontology, + // 独立语义指标库（SemanticMetric）：即存即生效指标卡片、在线试算与智能提取 + semanticMetrics, } /** diff --git a/frontend/src/api/semanticMetrics.ts b/frontend/src/api/semanticMetrics.ts new file mode 100644 index 00000000..53b187d5 --- /dev/null +++ b/frontend/src/api/semanticMetrics.ts @@ -0,0 +1,283 @@ +/** + * 独立语义指标库（SemanticMetric）与后台文档一键拆解工作流 REST 客户端（服务端 backend/app/api/semantic_metrics.py）。 + * + * 与工作流受管口径（caliberLedger）互补： + * - 单表模式（expression）：绑定「数据源 + 物理表 + 单表聚合 SQL 表达式」； + * - 多表 / 参数化模式（sql_template）：支持跨表 JOIN、WITH CTE 预聚合与 {{select_dims}}、{{and_where}}、{{period_days}} 等安全槽位； + * - 后台一键拆解任务（decomposeJobs）：喂入业务文档后在后台分阶段拆解「字段目录补丁 + 单表/多表指标口径 + 标准业务实体 + 知识库归档」。 + */ +import { del, get, post, put, qs, upload, type RequestOptions, type UploadOptions } from './http' + +export type MetricCalcMode = 'expression' | 'sql_template' + +export interface SemanticMetricDimension { + key: string + label: string + column: string + entity_key?: string +} + +export interface SemanticMetric { + id: string + key: string + name: string + aliases: string[] + description: string + unit: string + source_id: string + source_name?: string + calc_mode?: MetricCalcMode | string + table_name: string + sql_expression: string + sql_template?: string + aggregation_type: string + date_column: string + filter_sql: string + dimensions: SemanticMetricDimension[] + default_time_grain: string + status: 'enabled' | 'disabled' | 'draft' | string + source: 'human' | 'ai' | 'doc' | string + version: number + created_by?: string | null + updated_by?: string | null + created_at?: string | null + updated_at?: string | null +} + +export interface SemanticMetricInput { + key: string + name: string + aliases?: string[] + description?: string + unit?: string + source_id: string + calc_mode?: MetricCalcMode | string + table_name?: string + sql_expression?: string + sql_template?: string + date_column?: string + filter_sql?: string + dimensions?: SemanticMetricDimension[] + default_time_grain?: string + status?: string + source?: string + expected_version?: number + exists_in_db?: boolean +} + +export interface SemanticMetricPreviewIn { + metric_id?: string + definition?: Record + date_from?: string | null + date_to?: string | null + group_by_date?: boolean + time_grain?: string | null + group_by?: string[] + filters?: Record + limit?: number +} + +export interface SemanticMetricPreviewOut { + sql: string + source_name: string + columns: string[] + rows: unknown[][] + row_count: number + truncated: boolean + elapsed_ms: number + entity_alignments: Record[] + latest_available_date?: string | null +} + +export interface SemanticMetricSuggestOut { + source_id: string + source_name: string + model_id?: string | null + documents_used: string[] + candidates: SemanticMetricInput[] + skipped: { key: string; reason: string }[] +} + +export interface DecomposeAttachmentOut { + attachment_id: string + title: string + mime: string + char_count: number + chunk_count: number + saved_to_kb: boolean + document_id?: string | null + collection?: string | null +} + +export interface SemanticDecomposeStep { + key: string + label: string + status: 'pending' | 'running' | 'succeeded' | 'skipped' | 'failed' | string + detail: string + count: number + elapsed_ms: number +} + +export interface SemanticCatalogChangeItem { + id: string + table: string + path: string + slot_label: string + before: unknown + after: unknown + value: unknown + reason: string + state: string +} + +export interface SemanticCatalogPatchTable { + table: string + if_version: number + changes: SemanticCatalogChangeItem[] +} + +export interface SemanticEntityChange { + id: string + op: string + entity_key: string + entity_label: string + summary: string + before: unknown + after: unknown + reason: string + op_payload: Record +} + +export interface SemanticGuardrailNote { + title: string + rule: string + category: string +} + +export interface SemanticDecomposeAppliedRecord { + applied_at: string + applied_by?: string | null + catalog_applied_count: number + catalog_errors: string[] + metrics_created_count: number + metrics_updated_count: number + metric_errors: { key: string; reason: string }[] + entities_applied_count: number + entity_errors: string[] +} + +export interface SemanticDecomposeResult { + model_id?: string | null + kb_documents?: { id: string; title: string; created_now: boolean }[] + guardrail_notes?: SemanticGuardrailNote[] + catalog_patches?: SemanticCatalogPatchTable[] + catalog_skipped?: string[] + metric_candidates?: SemanticMetricInput[] + metric_skipped?: { key: string; reason: string }[] + entity_changes?: SemanticEntityChange[] + entity_skipped?: string[] + applied?: SemanticDecomposeAppliedRecord | null +} + +export interface SemanticDecomposeJob { + id: string + status: 'queued' | 'running' | 'succeeded' | 'failed' | 'cancelled' | string + source_id: string + source_name: string + scopes: string[] + document_ids: string[] + attachment_ids: string[] + doc_title: string + has_doc_text: boolean + save_to_kb: boolean + note: string + phase: string + phase_label: string + progress: number + steps: SemanticDecomposeStep[] + summary: Record + result?: SemanticDecomposeResult + error?: string | null + started_by?: string | null + started_at?: string | null + finished_at?: string | null + created_at?: string | null + updated_at?: string | null +} + +export interface SemanticDecomposeCreateIn { + source_id: string + scopes?: string[] + table_names?: string[] + document_ids?: string[] + attachment_ids?: string[] + doc_title?: string + doc_text?: string + save_to_kb?: boolean + note?: string +} + +export interface SemanticDecomposeApplyIn { + catalog_patches?: { table: string; changes: { path: string; value: unknown; reason: string }[] }[] + metrics?: SemanticMetricInput[] + entity_operations?: Record[] + overwrite_existing_metrics?: boolean +} + +export const semanticMetrics = { + list: (params?: { source_id?: string; status?: string; q?: string }, opts?: RequestOptions) => + get<{ items: SemanticMetric[]; total: number }>( + `/semantic-metrics${qs({ + source_id: params?.source_id, + status: params?.status, + q: params?.q, + })}`, + opts, + ), + get: (id: string, opts?: RequestOptions) => + get<{ metric: SemanticMetric }>(`/semantic-metrics/${encodeURIComponent(id)}`, opts), + create: (body: SemanticMetricInput) => + post<{ metric: SemanticMetric }>('/semantic-metrics', body), + batchCreate: (items: SemanticMetricInput[]) => + post<{ created: SemanticMetric[]; skipped: { key: string; reason: string }[] }>('/semantic-metrics/batch', { items }), + update: (id: string, body: Partial) => + put<{ metric: SemanticMetric }>(`/semantic-metrics/${encodeURIComponent(id)}`, body), + remove: (id: string) => + del(`/semantic-metrics/${encodeURIComponent(id)}`), + preview: (body: SemanticMetricPreviewIn) => + post('/semantic-metrics/preview', body), + suggest: (body: { source_id: string; table_names?: string[]; document_ids?: string[]; note?: string }) => + post('/semantic-metrics/suggest', body), + uploadAttachment: (file: File, opts?: { save_to_kb?: boolean; collection?: string } & UploadOptions) => { + const form = new FormData() + form.append('file', file) + if (opts?.save_to_kb !== undefined) form.append('save_to_kb', String(opts.save_to_kb)) + if (opts?.collection) form.append('collection', opts.collection) + return upload('/semantic-metrics/attachments', form, opts) + }, + decomposeJobs: { + active: (sourceId?: string, opts?: RequestOptions) => + get<{ active: SemanticDecomposeJob | null; latest: SemanticDecomposeJob | null }>( + `/semantic-metrics/decompose-jobs/active${qs({ source_id: sourceId })}`, + opts, + ), + list: (sourceId?: string, limit = 10, opts?: RequestOptions) => + get<{ items: SemanticDecomposeJob[] }>( + `/semantic-metrics/decompose-jobs${qs({ source_id: sourceId, limit })}`, + opts, + ), + create: (body: SemanticDecomposeCreateIn) => + post<{ job: SemanticDecomposeJob }>('/semantic-metrics/decompose-jobs', body), + get: (jobId: string, opts?: RequestOptions) => + get<{ job: SemanticDecomposeJob }>(`/semantic-metrics/decompose-jobs/${encodeURIComponent(jobId)}`, opts), + cancel: (jobId: string) => + post<{ job: SemanticDecomposeJob }>(`/semantic-metrics/decompose-jobs/${encodeURIComponent(jobId)}/cancel`, {}), + apply: (jobId: string, body: SemanticDecomposeApplyIn) => + post<{ + job_id: string + applied: SemanticDecomposeAppliedRecord + created_metrics: SemanticMetric[] + updated_metrics: SemanticMetric[] + }>(`/semantic-metrics/decompose-jobs/${encodeURIComponent(jobId)}/apply`, body), + }, +} diff --git a/frontend/src/pages/ledger/CaliberCatalog.tsx b/frontend/src/pages/ledger/CaliberCatalog.tsx index 8fe2412c..d19153b4 100644 --- a/frontend/src/pages/ledger/CaliberCatalog.tsx +++ b/frontend/src/pages/ledger/CaliberCatalog.tsx @@ -1,25 +1,32 @@ /** - * 口径目录（/ledger/calibers）：已发布口径的入口、指标、依赖和试跑。 + * 口径与指标目录（/ledger/calibers）： * - * 只读目录 + 参数表单 + 一次只读探索运行。口径的技术身份是「工作流@版本/口径卡」；目录给业务的人看的是名字、指标、 - * 入口字段和能不能用。试跑按固定版本派生只读图（服务端 ledger/derive.caliber_probe），不是正式出具：来源记 trial， - * 不进使用统计。SQL 不在页面上出现——服务端 descriptor 本来就不返回。 + * 1. 独立语义指标卡片（SemanticMetric）：即存即生效，直接绑定数据源、物理表、聚合 SQL 表达式、时间列与标准业务实体维度， + * 保存后自动接入问数页「快速直查」与 metric_query 工具；支持在线试算与从数据目录 / 业务文档批量提取。 + * 2. 工作流受管口径（CaliberDescriptor）：固定工作流版本@口径卡，用于多步编排与正式出具对账。 */ import { useEffect, useMemo, useState } from 'react' import { Link } from 'react-router-dom' -import { ChevronDown, Layers, Play, Plus, RefreshCw, Search } from 'lucide-react' +import { ChevronDown, Layers, Play, Plus, RefreshCw, Search, Sparkles, Zap } from 'lucide-react' import clsx from 'clsx' import { api } from '../../api/client' +import type { SemanticDecomposeJob, SemanticMetric } from '../../api/semanticMetrics' import { SchemaForm } from '../../components/SchemaForm' import { EmptyState, ErrorState, Notice, PageHeader, Skeleton, Spinner, toast } from '../../components/ui' import { LEDGER_TEXT } from '../../lib/terms' import { LEDGER_UI } from '../../lib/termsLedger' +import { useAtLeast } from '../../store/auth' +import { + DecomposeJobBanner, MetricEditorModal, MetricSuggestModal, SemanticMetricCardsSection, +} from './SemanticMetricStudio' import { TrialResult, TrialValidation } from './TrialResult' import type { FormSchema } from '../../typesApp' -import type { CaliberDescriptor, CaliberTrialOut } from '../../typesLedger' +import type { CaliberDescriptor, CaliberTrialOut, CanonicalEntity } from '../../typesLedger' const keyOf = (d: CaliberDescriptor) => `${d.workflow_id}@${d.version}/${d.card.node_id}` +type ViewFilter = 'all' | 'metrics' | 'calibers' + function defaultsOf(schema: FormSchema): Record { const out: Record = {} for (const [name, field] of Object.entries(schema.properties ?? {})) { @@ -29,7 +36,17 @@ function defaultsOf(schema: FormSchema): Record { } export function CaliberCatalog() { + const builder = useAtLeast('builder') const [items, setItems] = useState(null) + const [metrics, setMetrics] = useState([]) + const [entities, setEntities] = useState([]) + const [filter, setFilter] = useState('all') + const [editorOpen, setEditorOpen] = useState(false) + const [editingMetric, setEditingMetric] = useState(null) + const [suggestOpen, setSuggestOpen] = useState(false) + const [bannerJob, setBannerJob] = useState(null) + const [modalInitialJob, setModalInitialJob] = useState(null) + const [dismissedJobId, setDismissedJobId] = useState(null) const [error, setError] = useState(null) const [query, setQuery] = useState('') const [open, setOpen] = useState(null) @@ -37,6 +54,29 @@ export function CaliberCatalog() { const [trials, setTrials] = useState>({}) const [busy, setBusy] = useState(null) + const refreshDecomposeBanner = () => { + if (!builder) return + api.semanticMetrics.decomposeJobs.active().then( + ({ active, latest }) => { + setBannerJob(active ?? latest ?? null) + }, + () => {}, + ) + } + + const loadMetrics = () => { + if (!builder) return + api.semanticMetrics.list().then( + (res) => setMetrics(res.items ?? []), + () => setMetrics([]), + ) + api.ontology.listEntities().then( + (res) => setEntities(res.items ?? []), + () => setEntities([]), + ) + refreshDecomposeBanner() + } + const load = () => { setError(null) setItems(null) @@ -44,8 +84,44 @@ export function CaliberCatalog() { (body) => setItems(body.calibers), (e) => setError(e), ) + loadMetrics() + } + // eslint-disable-next-line react-hooks/exhaustive-deps + useEffect(load, [builder]) + + // 当有后台口径拆解任务正在排队或执行时，自动轮询最新阶段进度 + useEffect(() => { + if (!builder || !bannerJob || (bannerJob.status !== 'queued' && bannerJob.status !== 'running')) return + let alive = true + const timer = window.setInterval(() => { + api.semanticMetrics.decomposeJobs.active().then( + ({ active, latest }) => { + if (!alive) return + const next = active ?? latest ?? null + setBannerJob(next) + if (!active && latest?.status === 'succeeded') { + setDismissedJobId(null) + toast.ok('后台口径文档拆解已完成，点击横幅即可审阅并一键应用') + } + }, + () => {}, + ) + }, 1800) + return () => { + alive = false + window.clearInterval(timer) + } + }, [builder, bannerJob?.id, bannerJob?.status]) + + const handleCancelBannerJob = async (jobId: string) => { + try { + const { job: cancelled } = await api.semanticMetrics.decomposeJobs.cancel(jobId) + setBannerJob(cancelled) + toast.info('已取消后台拆解任务') + } catch (e) { + toast.error(e) + } } - useEffect(load, []) const shown = useMemo(() => { const q = query.trim().toLowerCase() @@ -54,6 +130,14 @@ export function CaliberCatalog() { ...d.metrics.map((m) => m.name ?? m.id)].some((s) => String(s ?? '').toLowerCase().includes(q))) }, [items, query]) + const shownMetricsCount = useMemo(() => { + const q = query.trim().toLowerCase() + if (!q) return metrics.length + return metrics.filter((m) => [ + m.name, m.key, m.table_name, m.source_name, m.description, ...(m.aliases ?? []), + ].some((s) => String(s ?? '').toLowerCase().includes(q))).length + }, [metrics, query]) + const toggle = (d: CaliberDescriptor) => { const key = keyOf(d) if (open === key) { setOpen(null); return } @@ -77,12 +161,33 @@ export function CaliberCatalog() { } } + const showMetrics = filter !== 'calibers' + const showCalibers = filter !== 'metrics' + return (
- } title="口径" subtitle="已发布口径的入口、指标、依赖与试运行" + } title="口径" subtitle="独立语义指标卡片（即存即生效）与工作流受管口径目录" actions={<> - + {builder && ( + <> + + + + )} + {LEDGER_UI.drafts.entry} - {expanded && ( -
-
-
-

指标

-
    - {d.metrics.map((m) => ( -
  • -
    - {m.name ?? m.id} - {m.unit || '—'} -
    - {m.definition?.text && ( -

    {m.definition.text}

    - )} - {!!m.params?.length && ( -

    - 参数：{m.params.map((p) => `${p.name}（${p.type}）`).join('、')} -

    - )} -
    • - ))} -
-
-
-

依赖

-

- 数据源：{d.dependencies.sources.join('、') || '—'}
- 表：{d.dependencies.tables.join('、') || '—'}
- {d.compose_reason ? `不能组合：${d.compose_reason}` : '可以按这一版组合'} -

- {d.issuance.reasons.length > 0 && ( -

- 出具：{d.issuance.reasons.join('；')} -

- )} -
-
-
-

入口字段

- {d.entry_schema.properties && Object.keys(d.entry_schema.properties).length > 0 ? ( - { - setValues((all) => ({ ...all, [key]: v })) - setTrials((all) => { - const next = { ...all } - delete next[key] - return next - }) - }} - idPrefix={`caliber-${key}`} /> - ) :

这份口径不需要参数。

} -
-
- - 打开工作流 - 试运行是只读探索运行，不是正式出具 -
- {result && ( -
- {result.status === 'succeeded' ? ( -
    - {result.metrics.map((m) => ( -
  • - {m.name ?? m.id} - {m.rendered ?? String(m.value ?? '—')}{m.unit ? ` ${m.unit}` : ''} -
    • - ))} -
- ) : ( - - {result.status === 'running' ? '还在运行：' : '试运行没有成功：'} - {result.error || result.status} - - )} - {result.run_id && ( -
- - {LEDGER_UI.drafts.trialRun} - -
- )} - {result.status === 'succeeded' && <>} -
- )} + {builder && bannerJob && bannerJob.id !== dismissedJobId && ( + { + setModalInitialJob(j) + setSuggestOpen(true) + }} + onCancel={(jid) => void handleCancelBannerJob(jid)} + onDismiss={() => setDismissedJobId(bannerJob.id)} + /> + )} +
+
+ + + +
+ +
+ + {showMetrics && ( + { setEditingMetric(m); setEditorOpen(true) }} + /> + )} + + {filter === 'metrics' && shownMetricsCount === 0 && ( + } + title="还没有独立语义指标卡片" + body={query ? '换个关键词试试' : '点击右上角「新建指标卡片」或「从目录与文档提取」，保存后无需发布工作流即可在问数快速直查中直接命中'} + /> + )} + + {showCalibers && ( + error ? + : !items ? + : !shown.length ? ( + shownMetricsCount === 0 ? ( + } title="还没有可用的口径" + body={query ? '换个关键词试试' : '可新建即存即用的指标卡片，或在工作流里发布带口径卡的版本'} /> + ) : null + ) : ( +
+ {showMetrics && shownMetricsCount > 0 && ( +
+ 工作流受管口径（共 {shown.length} 项）
)} - + {shown.map((d) => { + const key = keyOf(d) + const expanded = open === key + const result = trials[key] + return ( +
+ + {expanded && ( +
+
+
+

指标

+
    + {d.metrics.map((m) => ( +
  • +
    + {m.name ?? m.id} + {m.unit || '—'} +
    + {m.definition?.text && ( +

    {m.definition.text}

    + )} + {!!m.params?.length && ( +

    + 参数：{m.params.map((p) => `${p.name}（${p.type}）`).join('、')} +

    + )} +
    • + ))} +
+
+
+

依赖

+

+ 数据源：{d.dependencies.sources.join('、') || '—'}
+ 表：{d.dependencies.tables.join('、') || '—'}
+ {d.compose_reason ? `不能组合：${d.compose_reason}` : '可以按这一版组合'} +

+ {d.issuance.reasons.length > 0 && ( +

+ 出具：{d.issuance.reasons.join('；')} +

+ )} +
+
+
+

入口字段

+ {d.entry_schema.properties && Object.keys(d.entry_schema.properties).length > 0 ? ( + { + setValues((all) => ({ ...all, [key]: v })) + setTrials((all) => { + const next = { ...all } + delete next[key] + return next + }) + }} + idPrefix={`caliber-${key}`} /> + ) :

这份口径不需要参数。

} +
+
+ + 打开工作流 + 试运行是只读探索运行，不是正式出具 +
+ {result && ( +
+ {result.status === 'succeeded' ? ( +
    + {result.metrics.map((m) => ( +
  • + {m.name ?? m.id} + {m.rendered ?? String(m.value ?? '—')}{m.unit ? ` ${m.unit}` : ''} +
    • + ))} +
+ ) : ( + + {result.status === 'running' ? '还在运行：' : '试运行没有成功：'} + {result.error || result.status} + + )} + {result.run_id && ( +
+ + {LEDGER_UI.drafts.trialRun} + +
+ )} + {result.status === 'succeeded' && <>} +
+ )} +
+ )} +
+ ) + })} +
) - })} + )}
+ + {builder && ( + <> + { setEditorOpen(false); setEditingMetric(null) }} + onSaved={loadMetrics} + /> + setSuggestOpen(false)} + onSaved={loadMetrics} + onJobUpdated={(nextJob) => { + setBannerJob(nextJob) + setModalInitialJob(nextJob) + if (nextJob) setDismissedJobId(null) + }} + /> + + )} ) } diff --git a/frontend/src/pages/ledger/SemanticMetricStudio.tsx b/frontend/src/pages/ledger/SemanticMetricStudio.tsx new file mode 100644 index 00000000..3efbe88e --- /dev/null +++ b/frontend/src/pages/ledger/SemanticMetricStudio.tsx @@ -0,0 +1,1737 @@ +/** + * 独立语义指标工作台与口径文档一键拆解中心（SemanticMetricStudio）： + * + * 1. 即存即用指标卡片编辑器（MetricEditorModal）：同时支持「单表聚合表达式」与「多表 / 参数化 SQL 模板」两种模式； + * 2. 后台文档一键拆解与智能提取工作台（MetricSuggestModal）：喂入业务口径文档（粘贴 Markdown、上传附件、选择知识库文档）， + * 在服务端后台分阶段拆解「字段目录补丁 + 单表与多表指标口径 + 标准业务实体映射 + 知识库归档」，支持实时进度、关闭弹窗后台继续执行、行内试算与一键应用； + * 3. 语义指标卡片列表与在线试算面板（SemanticMetricCardsSection）。 + */ +import { useEffect, useMemo, useRef, useState } from 'react' +import { + AlertTriangle, Check, CheckCircle2, ChevronDown, Clock, Code2, Database, Edit3, + FileText, Layers, Play, Plus, Power, RotateCcw, ShieldAlert, Sparkles, Tag, Trash2, Upload, X, Zap, +} from 'lucide-react' +import clsx from 'clsx' +import { api } from '../../api/client' +import type { + DecomposeAttachmentOut, + MetricCalcMode, + SemanticDecomposeJob, + SemanticMetric, + SemanticMetricDimension, + SemanticMetricInput, + SemanticMetricPreviewOut, +} from '../../api/semanticMetrics' +import { confirmDialog, CopyButton, Field, Modal, Notice, Spinner, toast } from '../../components/ui' +import { useAtLeast } from '../../store/auth' +import { useDatasources } from '../../store/catalog' +import type { CanonicalEntity } from '../../typesLedger' +import type { KbDocument } from '../../types' + +const TEMPLATE_SLOT_CHIPS: { token: string; tip: string }[] = [ + { token: '{{select_dims}}', tip: '动态展开分组列（放在 SELECT 紧后，无分组时自动省去）' }, + { token: '{{and_where}}', tip: '在 WHERE 1=1 后动态追加日期区间与维度过滤条件' }, + { token: '{{where}}', tip: '完整 WHERE 子句占位符（无过滤条件时自动省去 WHERE）' }, + { token: '{{group_by}}', tip: '按所选时间粒度与维度动态生成 GROUP BY 子句' }, + { token: '{{order_by}}', tip: '动态生成趋势升序或指标降序 ORDER BY 子句' }, + { token: '{{period_days}}', tip: '动态统计周期自然天数（如 TEEP 日历秒数 = {{period_days}} * 86400）' }, + { token: '{{date_from}}', tip: '查询起始日期字面量（未传时为 NULL）' }, + { token: '{{date_to}}', tip: '查询结束日期字面量（未传时为 NULL）' }, +] + +function splitAliases(raw: string): string[] { + return raw + .split(/[，,、;\n]+/) + .map((s) => s.trim()) + .filter(Boolean) +} + +function formatCatalogValue(val: unknown): string { + if (val === null || val === undefined || val === '') return '（未填写）' + if (typeof val === 'string' || typeof val === 'number' || typeof val === 'boolean') return String(val) + if (typeof val === 'object') { + const obj = val as Record + if ('column' in obj) { + const col = String(obj.column ?? '') + const rule = String(obj.rule ?? '') + return rule ? `${col}（${rule}）` : col + } + return Object.entries(obj) + .map(([k, v]) => `${k}=${String(v ?? '')}`) + .join('，') + } + return String(val) +} + +function PreviewResultPanel({ preview }: { preview: SemanticMetricPreviewOut }) { + return ( +
+
+ + 数据源：{preview.source_name} · 返回 {preview.row_count} 行 · 耗时 {preview.elapsed_ms} ms + + +
+ 
+        {preview.sql}
+
+ {preview.latest_available_date && ( + + 当前筛选条件下结果为空；探查到该表最新可用业务日期为 {preview.latest_available_date}。 + + )} + {preview.entity_alignments?.length > 0 && ( +
+ 标准实体编码自动转换： + {preview.entity_alignments.map((a, idx) => ( + + {String(a.dimension ?? '')}：{String(a.input ?? '')} → {Array.isArray(a.translated) ? a.translated.join(' / ') : ''} + + ))} +
+ )} + {preview.columns.length > 0 && preview.rows.length > 0 ? ( +
+ + + + {preview.columns.map((c) => ( + + ))} + + + + {preview.rows.slice(0, 30).map((r, ri) => ( + + {r.map((cell, ci) => ( + + ))} + + ))} + +
{c}
+ {cell === null || cell === undefined ? NULL : String(cell)} +
+
+ ) : ( +
查询执行成功，当前条件下暂无数据行。
+ )} +
+ ) +} + +interface TableColumnMeta { + name: string + type: string + label?: string +} + +export function MetricEditorModal({ + open, + metric, + entities, + onClose, + onSaved, +}: { + open: boolean + metric: SemanticMetric | null + entities: CanonicalEntity[] + onClose: () => void + onSaved: () => void +}) { + const { list: allSources } = useDatasources() + const sources = useMemo(() => allSources.filter((s) => s.enabled !== false), [allSources]) + + const [name, setName] = useState('') + const [key, setKey] = useState('') + const [unit, setUnit] = useState('') + const [aliasesText, setAliasesText] = useState('') + const [description, setDescription] = useState('') + const [sourceId, setSourceId] = useState('') + const [calcMode, setCalcMode] = useState('expression') + const [tableName, setTableName] = useState('') + const [sqlExpression, setSqlExpression] = useState('') + const [sqlTemplate, setSqlTemplate] = useState('') + const [dateColumn, setDateColumn] = useState('') + const [filterSql, setFilterSql] = useState('') + const [defaultTimeGrain, setDefaultTimeGrain] = useState('day') + const [dimensions, setDimensions] = useState([]) + + const [tables, setTables] = useState([]) + const [columns, setColumns] = useState([]) + const [catalogHint, setCatalogHint] = useState('') + const [saving, setSaving] = useState(false) + const [previewing, setPreviewing] = useState(false) + const [preview, setPreview] = useState(null) + const tplRef = useRef(null) + + useEffect(() => { + if (!open) return + setPreview(null) + if (metric) { + setName(metric.name) + setKey(metric.key) + setUnit(metric.unit || '') + setAliasesText((metric.aliases ?? []).join('、')) + setDescription(metric.description || '') + setSourceId(metric.source_id) + setCalcMode(metric.calc_mode === 'sql_template' ? 'sql_template' : 'expression') + setTableName(metric.table_name || '') + setSqlExpression(metric.calc_mode === 'sql_template' && metric.sql_expression?.startsWith('[多表模板]') ? '' : (metric.sql_expression || '')) + setSqlTemplate(metric.sql_template || '') + setDateColumn(metric.date_column || '') + setFilterSql(metric.filter_sql || '') + setDefaultTimeGrain(metric.default_time_grain || 'day') + setDimensions(metric.dimensions ?? []) + } else { + setName('') + setKey('') + setUnit('') + setAliasesText('') + setDescription('') + setSourceId(sources[0]?.id ?? '') + setCalcMode('expression') + setTableName('') + setSqlExpression('') + setSqlTemplate('') + setDateColumn('') + setFilterSql('') + setDefaultTimeGrain('day') + setDimensions([]) + } + }, [open, metric, sources]) + + useEffect(() => { + if (!open || !sourceId) { + setTables([]) + return + } + let alive = true + api.datasources.schema(sourceId).then( + (res) => { + if (!alive) return + const nextTables: string[] = Array.isArray(res?.tables) ? res.tables : [] + setTables(nextTables) + if (!metric && nextTables.length > 0 && !tableName) { + setTableName(nextTables[0]) + } + }, + () => { if (alive) setTables([]) }, + ) + return () => { alive = false } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [open, sourceId]) + + useEffect(() => { + if (!open || !sourceId || !tableName) { + setColumns([]) + setCatalogHint('') + return + } + let alive = true + api.dataCatalog.get(sourceId, tableName).then( + (detail: any) => { + if (!alive) return + const rawCols = Array.isArray(detail?.structure?.columns) ? detail.structure.columns : [] + const colNotes = detail?.notes?.columns ?? {} + const mapped: TableColumnMeta[] = rawCols.map((c: any) => ({ + name: String(c.name ?? ''), + type: String(c.type ?? ''), + label: typeof colNotes?.[c.name]?.label?.value === 'string' ? colNotes[c.name].label.value : undefined, + })).filter((c: TableColumnMeta) => c.name) + setColumns(mapped) + const bDateRaw = detail?.notes?.business_date?.value + const bDate = typeof bDateRaw === 'string' + ? bDateRaw + : (bDateRaw && typeof bDateRaw === 'object' && typeof bDateRaw.column === 'string' ? bDateRaw.column : '') + const vFilter = typeof detail?.notes?.valid_filter?.value === 'string' ? detail.notes.valid_filter.value : '' + const tLabel = typeof detail?.notes?.label?.value === 'string' ? detail.notes.label.value : '' + const hints = [ + tLabel ? `目录名：${tLabel}` : '', + bDate ? `推荐业务时间列：${bDate}` : '', + vFilter ? `推荐有效过滤：${vFilter}` : '', + ].filter(Boolean) + setCatalogHint(hints.join(' · ')) + if (!metric && calcMode === 'expression') { + if (bDate) setDateColumn((prev) => prev || bDate) + if (vFilter) setFilterSql((prev) => prev || vFilter) + } + }, + () => { + if (alive) { + setColumns([]) + setCatalogHint('') + } + }, + ) + return () => { alive = false } + }, [open, sourceId, tableName, metric, calcMode]) + + const isTemplate = calcMode === 'sql_template' + + const insertSlotToken = (token: string) => { + const el = tplRef.current + if (!el) { + setSqlTemplate((prev) => (prev ? `${prev} ${token}` : token)) + return + } + const start = el.selectionStart ?? sqlTemplate.length + const end = el.selectionEnd ?? sqlTemplate.length + const next = `${sqlTemplate.slice(0, start)}${token}${sqlTemplate.slice(end)}` + setSqlTemplate(next) + requestAnimationFrame(() => { + el.focus() + const pos = start + token.length + el.setSelectionRange(pos, pos) + }) + } + + const fillTemplateSkeleton = () => { + const anchor = tableName || tables[0] || 'fact_table' + const metricAlias = key.trim() || 'metric_value' + const sampleDate = columns[0]?.name || 'stat_date' + setSqlTemplate( + `WITH aux AS (\n SELECT id, AVG(coeff) AS coeff\n FROM ${tables[1] || anchor}\n GROUP BY id\n)\nSELECT\n {{select_dims}}\n ROUND(SUM(s.${columns[1]?.name || 'amount'}) * 1.0, 2) AS ${metricAlias}\nFROM ${anchor} s\nWHERE 1=1 {{and_where}}\n{{group_by}}\n{{order_by}}`, + ) + if (!dateColumn) setDateColumn(`s.${sampleDate}`) + } + + const buildPayload = (): SemanticMetricInput => ({ + key: key.trim(), + name: name.trim(), + unit: unit.trim(), + aliases: splitAliases(aliasesText), + description: description.trim(), + source_id: sourceId, + calc_mode: calcMode, + table_name: tableName.trim(), + sql_expression: isTemplate ? sqlExpression.trim() : sqlExpression.trim(), + sql_template: isTemplate ? sqlTemplate.trim() : '', + date_column: dateColumn.trim(), + filter_sql: filterSql.trim(), + default_time_grain: defaultTimeGrain, + dimensions: dimensions + .filter((d) => d.column.trim()) + .map((d) => ({ + key: (d.key || d.column.split('.').pop() || d.column).trim(), + label: (d.label || d.key || d.column).trim(), + column: d.column.trim(), + entity_key: (d.entity_key || '').trim(), + })), + ...(metric ? { expected_version: metric.version } : {}), + }) + + const canRunOrSave = Boolean( + sourceId && (isTemplate ? sqlTemplate.trim() : (tableName.trim() && sqlExpression.trim())), + ) + + const runPreview = async () => { + setPreviewing(true) + try { + const payload = buildPayload() + if (!payload.key) payload.key = 'draft_metric' + if (!payload.name) payload.name = '草稿试算指标' + const res = await api.semanticMetrics.preview({ + definition: payload as unknown as Record, + limit: 20, + }) + setPreview(res) + toast.ok('在线试算成功') + } catch (e) { + toast.error(e) + } finally { + setPreviewing(false) + } + } + + const save = async () => { + setSaving(true) + try { + const payload = buildPayload() + if (metric) { + await api.semanticMetrics.update(metric.id, payload) + toast.ok(`已更新指标「${payload.name}」`) + } else { + await api.semanticMetrics.create(payload) + toast.ok(`已创建并启用指标「${payload.name}」`) + } + onSaved() + onClose() + } catch (e) { + toast.error(e) + } finally { + setSaving(false) + } + } + + return ( + + +
+ + +
+ + } + > +
+
+ 指标计算模式 +
+ + +
+
+ +
+ + setName(e.target.value)} + placeholder="例如：设备综合效率 OEE（跨表精算）" /> + + + setKey(e.target.value)} + placeholder="例如：oee_multi_table" /> + +
+ +
+ + + + + {tables.length > 0 ? ( + + ) : ( + setTableName(e.target.value)} + placeholder="物理表名" /> + )} + + + setUnit(e.target.value)} + placeholder="例如：% / 秒 / 人次 / 元" /> + +
+ + {catalogHint && ( +
+ 数据目录提示：{catalogHint} +
+ )} + + {!isTemplate ? ( + + setSqlExpression(e.target.value)} + placeholder="例如：SUM(total_sys) 或 ROUND(SUM(good_qty) * 100.0 / NULLIF(SUM(total_qty), 0), 2)" /> + {columns.length > 0 && ( +
+ {columns.map((c) => ( + + ))} +
+ )} + + ) : ( + +
+
+ {TEMPLATE_SLOT_CHIPS.map((slot) => ( + + ))} +
+ +
+