独立语义指标库与文档一键拆解:支持多表参数化 SQL 模板、后台进度与 Oracle 方言

- 新增独立语义指标模型(SemanticMetric)与 metric_query 内置工具,同时支持单表聚合表达式(expression)与跨表 JOIN / CTE 参数化模板(sql_template)
- 新增后台口径文档一键拆解任务(SemanticDecomposeJob),支持实时阶段进度、转后台执行、中途取消,串联知识库红线归档、盲列数据目录补丁、指标试算与标准实体绑定四层资产一键落库
- 完善多数据库方言确定性编译(重点补齐 Oracle 顶层 WITH + ROWNUM 限流、标识符引号规整、DATE 字面量与 TO_CHAR 时间粒度转换)
This commit is contained in:
ThearchyHelios
2026-11-12 12:58:24 +08:00
parent e15e94f93e
commit f144ef16d2
14 changed files with 6078 additions and 145 deletions
+418
View File
@@ -0,0 +1,418 @@
"""独立语义指标库(SemanticMetric)与后台口径文档一键拆解工作流 REST 接口(搭建者及以上权限)。
- GET /api/semantic-metrics:列出语义指标卡片(支持按 source_id、status、q 过滤)
- POST /api/semantic-metrics:创建单条语义指标卡片(支持单表 expression 与多表参数化 sql_template,保存即刻生效)
- POST /api/semantic-metrics/batch:批量保存勾选的候选语义指标卡片
- PUT /api/semantic-metrics/{metric_id}:编辑语义指标卡片(支持乐观锁 expected_version)
- DELETE /api/semantic-metrics/{metric_id}:删除语义指标卡片
- POST /api/semantic-metrics/preview:在线试算(支持已落库 metric_id 或草稿 definition 实时预览 SQL 与结果表)
- POST /api/semantic-metrics/suggest:同步提取候选指标卡片(兼容旧调用)
- POST /api/semantic-metrics/attachments:上传业务口径文档或编码对照表附件(.md/.txt/.pdf/.docx/.xlsx/.csv)
- POST /api/semantic-metrics/decompose-jobs:启动后台文档一键拆解 / 智能提取指标任务(实时阶段进度,关闭弹窗继续执行)
- GET /api/semantic-metrics/decompose-jobs/active:查询正在运行与最近一次拆解任务(供口径台账页顶部横幅轮询)
- GET /api/semantic-metrics/decompose-jobs/{job_id}:查询单个拆解任务的实时进度、分步耗时与待审阅结果
- POST /api/semantic-metrics/decompose-jobs/{job_id}/cancel:取消排队或执行中的拆解任务
- POST /api/semantic-metrics/decompose-jobs/{job_id}/apply:一键应用审阅勾选的「字段目录补丁 + 指标口径 + 标准业务实体」
"""
from __future__ import annotations
from typing import Any
from fastapi import APIRouter, BackgroundTasks, Depends, File, Form, Header, HTTPException, Query, UploadFile
from pydantic import BaseModel, Field
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.api.coded import CodedHTTPException
from app.api.deps import actor_of
from app.data import entity_docs
from app.data.semantic_decompose import (
DECOMPOSE_BUSY_CODE,
DECOMPOSE_FINISHED_CODE,
DECOMPOSE_NOT_FOUND_CODE,
DecomposeBusy,
DecomposeFinished,
DecomposeNotFound,
apply_decompose_job,
cancel_decompose_job,
create_decompose_job,
get_active_or_latest_job,
get_job,
job_to_dict,
list_jobs,
)
from app.data.semantic_metrics import (
MetricConflict,
MetricInvalid,
MetricNotFound,
create_metric,
delete_metric,
get_metric,
list_metrics,
metric_to_dict,
preview_metric,
suggest_metrics_from_source,
update_metric,
)
from app.db.base import get_session
from app.db.models import DataSource
router = APIRouter(prefix="/api/semantic-metrics", tags=["semantic-metrics"])
class MetricDimensionIn(BaseModel):
key: str = ""
label: str = ""
column: str = Field(min_length=1)
entity_key: str = ""
class MetricCreateIn(BaseModel):
key: str = Field(min_length=1, max_length=64)
name: str = Field(min_length=1, max_length=200)
aliases: list[str] = Field(default_factory=list)
description: str = ""
unit: str = ""
source_id: str = Field(min_length=1)
calc_mode: str = "expression"
table_name: str = ""
sql_expression: str = ""
sql_template: str = ""
date_column: str = ""
filter_sql: str = ""
dimensions: list[MetricDimensionIn] = Field(default_factory=list)
default_time_grain: str = "day"
status: str = "enabled"
source: str = "human"
class MetricUpdateIn(BaseModel):
key: str | None = None
name: str | None = None
aliases: list[str] | None = None
description: str | None = None
unit: str | None = None
source_id: str | None = None
calc_mode: str | None = None
table_name: str | None = None
sql_expression: str | None = None
sql_template: str | None = None
date_column: str | None = None
filter_sql: str | None = None
dimensions: list[MetricDimensionIn] | None = None
default_time_grain: str | None = None
status: str | None = None
expected_version: int | None = None
class MetricBatchCreateIn(BaseModel):
items: list[MetricCreateIn] = Field(min_length=1, max_length=50)
class MetricPreviewIn(BaseModel):
metric_id: str | None = None
definition: dict[str, Any] | None = None
date_from: str | None = None
date_to: str | None = None
group_by_date: bool = False
time_grain: str | None = None
group_by: list[str] = Field(default_factory=list)
filters: dict[str, Any] = Field(default_factory=dict)
limit: int = Field(default=50, ge=1, le=200)
class MetricSuggestIn(BaseModel):
source_id: str = Field(min_length=1)
table_names: list[str] | None = None
document_ids: list[str] | None = None
note: str = ""
class DecomposeJobCreateIn(BaseModel):
source_id: str = Field(min_length=1)
scopes: list[str] | None = None
table_names: list[str] | None = None
document_ids: list[str] = Field(default_factory=list)
attachment_ids: list[str] = Field(default_factory=list)
doc_title: str = ""
doc_text: str = ""
save_to_kb: bool = True
note: str = ""
class DecomposeJobApplyIn(BaseModel):
catalog_patches: list[dict[str, Any]] | None = None
metrics: list[dict[str, Any]] | None = None
entity_operations: list[dict[str, Any]] | None = None
overwrite_existing_metrics: bool = True
def _raise_metric_error(exc: Exception) -> HTTPException:
if isinstance(exc, DecomposeNotFound):
return CodedHTTPException(404, str(exc), DECOMPOSE_NOT_FOUND_CODE)
if isinstance(exc, DecomposeBusy):
return CodedHTTPException(409, str(exc), DECOMPOSE_BUSY_CODE)
if isinstance(exc, DecomposeFinished):
return CodedHTTPException(409, str(exc), DECOMPOSE_FINISHED_CODE)
if isinstance(exc, MetricNotFound):
return CodedHTTPException(404, str(exc), "metric_not_found")
if isinstance(exc, MetricConflict):
return CodedHTTPException(409, str(exc), "metric_conflict")
if isinstance(exc, MetricInvalid):
return CodedHTTPException(422, str(exc), "metric_invalid")
return HTTPException(400, str(exc))
async def _source_names_map(session: AsyncSession, source_ids: set[str]) -> dict[str, str]:
if not source_ids:
return {}
rows = (await session.execute(select(DataSource.id, DataSource.name).where(DataSource.id.in_(source_ids)))).all()
return {str(sid): str(sname) for sid, sname in rows}
@router.get("")
async def api_list_metrics(
source_id: str | None = Query(default=None),
status: str | None = Query(default=None),
q: str | None = Query(default=None),
session: AsyncSession = Depends(get_session),
) -> dict[str, Any]:
rows = await list_metrics(session, source_id=source_id, status=status, q=q)
src_names = await _source_names_map(session, {r.source_id for r in rows if r.source_id})
return {
"items": [metric_to_dict(r, source_name=src_names.get(r.source_id, "")) for r in rows],
"total": len(rows),
}
@router.post("/attachments")
async def api_upload_decompose_attachment(
background: BackgroundTasks,
file: UploadFile = File(...),
save_to_kb: bool = Form(default=False),
collection: str = Form(default="default"),
session: AsyncSession = Depends(get_session),
) -> dict[str, Any]:
try:
raw_bytes = await entity_docs.read_upload_limited(file)
res = await entity_docs.ingest_entity_attachment(
session,
raw_bytes=raw_bytes,
filename=file.filename or "上传文档",
mime=file.content_type or "",
save_to_kb=save_to_kb,
collection=collection,
background=background,
)
await session.commit()
return res
except entity_docs.AttachmentTooLarge as exc:
raise CodedHTTPException(413, str(exc), "attachment_too_large") from exc
except entity_docs.AttachmentInvalid as exc:
raise CodedHTTPException(422, str(exc), "attachment_invalid") from exc
@router.get("/decompose-jobs/active")
async def api_active_decompose_job(
source_id: str | None = Query(default=None),
session: AsyncSession = Depends(get_session),
) -> dict[str, Any]:
return await get_active_or_latest_job(session, source_id=source_id)
@router.get("/decompose-jobs")
async def api_list_decompose_jobs(
source_id: str | None = Query(default=None),
limit: int = Query(default=10, ge=1, le=30),
session: AsyncSession = Depends(get_session),
) -> dict[str, Any]:
items = await list_jobs(session, source_id=source_id, limit=limit)
return {"items": items}
@router.post("/decompose-jobs", status_code=202)
async def api_create_decompose_job(
payload: DecomposeJobCreateIn,
session: AsyncSession = Depends(get_session),
x_actor: str | None = Header(default=None),
) -> dict[str, Any]:
try:
job = await create_decompose_job(
session,
source_id=payload.source_id,
scopes=payload.scopes,
table_names=payload.table_names,
document_ids=payload.document_ids,
attachment_ids=payload.attachment_ids,
doc_title=payload.doc_title,
doc_text=payload.doc_text,
save_to_kb=payload.save_to_kb,
note=payload.note,
actor=actor_of(x_actor),
)
return {"job": job_to_dict(job, include_result=False)}
except Exception as exc: # noqa: BLE001
await session.rollback()
raise _raise_metric_error(exc) from exc
@router.get("/decompose-jobs/{job_id}")
async def api_get_decompose_job(
job_id: str,
session: AsyncSession = Depends(get_session),
) -> dict[str, Any]:
try:
job = await get_job(session, job_id)
return {"job": job_to_dict(job, include_result=True)}
except Exception as exc: # noqa: BLE001
raise _raise_metric_error(exc) from exc
@router.post("/decompose-jobs/{job_id}/cancel")
async def api_cancel_decompose_job(
job_id: str,
session: AsyncSession = Depends(get_session),
) -> dict[str, Any]:
try:
job = await cancel_decompose_job(session, job_id)
return {"job": job_to_dict(job, include_result=False)}
except Exception as exc: # noqa: BLE001
await session.rollback()
raise _raise_metric_error(exc) from exc
@router.post("/decompose-jobs/{job_id}/apply")
async def api_apply_decompose_job(
job_id: str,
payload: DecomposeJobApplyIn,
session: AsyncSession = Depends(get_session),
x_actor: str | None = Header(default=None),
) -> dict[str, Any]:
try:
return await apply_decompose_job(
session,
job_id,
catalog_patches=payload.catalog_patches,
metrics=payload.metrics,
entity_operations=payload.entity_operations,
overwrite_existing_metrics=payload.overwrite_existing_metrics,
actor=actor_of(x_actor),
)
except Exception as exc: # noqa: BLE001
await session.rollback()
raise _raise_metric_error(exc) from exc
@router.get("/{metric_id}")
async def api_get_metric(metric_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]:
try:
row = await get_metric(session, metric_id)
except Exception as exc: # noqa: BLE001
raise _raise_metric_error(exc) from exc
src_names = await _source_names_map(session, {row.source_id} if row.source_id else set())
return {"metric": metric_to_dict(row, source_name=src_names.get(row.source_id, ""))}
@router.post("", status_code=201)
async def api_create_metric(
payload: MetricCreateIn,
session: AsyncSession = Depends(get_session),
x_actor: str | None = Header(default=None),
) -> dict[str, Any]:
try:
row, ds = await create_metric(session, payload.model_dump(), actor=actor_of(x_actor))
await session.commit()
await session.refresh(row)
except Exception as exc: # noqa: BLE001
await session.rollback()
raise _raise_metric_error(exc) from exc
return {"metric": metric_to_dict(row, source_name=ds.name)}
@router.post("/batch", status_code=201)
async def api_batch_create_metrics(
payload: MetricBatchCreateIn,
session: AsyncSession = Depends(get_session),
x_actor: str | None = Header(default=None),
) -> dict[str, Any]:
actor = actor_of(x_actor)
created: list[dict[str, Any]] = []
skipped: list[dict[str, str]] = []
for item in payload.items:
data = item.model_dump()
try:
async with session.begin_nested():
row, ds = await create_metric(session, data, actor=actor)
created.append(metric_to_dict(row, source_name=ds.name))
except Exception as exc: # noqa: BLE001
skipped.append({"key": item.key, "reason": str(exc)})
await session.commit()
return {"created": created, "skipped": skipped}
@router.put("/{metric_id}")
async def api_update_metric(
metric_id: str,
payload: MetricUpdateIn,
session: AsyncSession = Depends(get_session),
x_actor: str | None = Header(default=None),
) -> dict[str, Any]:
data = {k: v for k, v in payload.model_dump().items() if v is not None}
try:
row, ds = await update_metric(session, metric_id, data, actor=actor_of(x_actor))
await session.commit()
await session.refresh(row)
except Exception as exc: # noqa: BLE001
await session.rollback()
raise _raise_metric_error(exc) from exc
return {"metric": metric_to_dict(row, source_name=ds.name)}
@router.delete("/{metric_id}")
async def api_delete_metric(metric_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]:
try:
row = await delete_metric(session, metric_id)
await session.commit()
except Exception as exc: # noqa: BLE001
await session.rollback()
raise _raise_metric_error(exc) from exc
return {"ok": True, "deleted_id": row.id, "deleted_key": row.key}
@router.post("/preview")
async def api_preview_metric(
payload: MetricPreviewIn,
session: AsyncSession = Depends(get_session),
) -> dict[str, Any]:
try:
return await preview_metric(
session,
metric_id=payload.metric_id,
definition=payload.definition,
date_from=payload.date_from,
date_to=payload.date_to,
group_by_date=payload.group_by_date,
time_grain=payload.time_grain,
group_by=payload.group_by,
filters=payload.filters,
limit=payload.limit,
)
except Exception as exc: # noqa: BLE001
raise _raise_metric_error(exc) from exc
@router.post("/suggest")
async def api_suggest_metrics(
payload: MetricSuggestIn,
session: AsyncSession = Depends(get_session),
) -> dict[str, Any]:
try:
return await suggest_metrics_from_source(
session,
source_id=payload.source_id,
table_names=payload.table_names,
document_ids=payload.document_ids,
note=payload.note,
)
except Exception as exc: # noqa: BLE001
raise _raise_metric_error(exc) from exc
+4 -3
View File
@@ -41,18 +41,19 @@ from typing import Any
from app.analysis.contract import TOOL_NAMES
from app.ledger.querycore import CALIBER_QUERY
from app.tools.metrics import METRIC_QUERY
from app.tools.ontology import ONTOLOGY_TOOLS
#: 问数据运行里 user 角色能用的节点类型(engine/schema.py 的 NodeType 取值)
CHAT_NODE_TYPES = frozenset({"input", "llm", "agent", "report", "output", "retrieve", "transform", "branch",
"metrics", "merge", "analysis", "tool"})
#: Agent 能绑的只读内置工具(tools/builtin/):分析工具也在内(contract.TOOL_NAMES,不查库),以及口径查询
#: (querycore.CALIBER_QUERY,只读、按资产授权收窄——见文件开头)
#: (querycore.CALIBER_QUERY,只读、按资产授权收窄——见文件开头)、独立语义指标查询(METRIC_QUERY,只读、经数据源遮罩)
#: 和本体邻居/源健康探针(ONTOLOGY_TOOLS,只读、按授权与遮罩收窄)。工具名只从登记处取常量,不在这里另写一个字符串
CHAT_BUILTIN_TOOLS = frozenset({"calculator", "current_time", "json_query", "kb_search", "memory_recall",
CALIBER_QUERY, *ONTOLOGY_TOOLS, *TOOL_NAMES.values()})
CALIBER_QUERY, METRIC_QUERY, *ONTOLOGY_TOOLS, *TOOL_NAMES.values()})
#: 上面那几个内置工具给人看的说法(拒绝时的报错、提示词里都写它)
CHAT_BUILTIN_LABEL = "计算、当前时间、JSON 取值、知识库检索、记忆读取、分析、口径查询、本体邻居、数据源健康"
CHAT_BUILTIN_LABEL = "计算、当前时间、JSON 取值、知识库检索、记忆读取、分析、口径查询、指标查询、本体邻居、数据源健康"
#: 运行时改记忆范围、知识库的配置键(engine/nodes 的 ctx.cfg("memory_scope") / ctx.cfg("collection"),图级 defaults 同样生效)
SCOPE_KEYS = ("memory_scope", "collection")
SCOPE_REFUSED = "没有权限:使用者在问数据中不能指定记忆范围和知识库,使用的是管理员设好的默认值"
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+3
View File
@@ -493,6 +493,9 @@ _COLUMN_MIGRATIONS: list[tuple[str, str, str]] = [
# 增量重导入的持久对齐键(K11):存量工作项回填空串——空串不参与按对齐键的匹配,
# 不会被误当成「同一项」串上旧决定(宁可不匹配,也不按位置猜)
("caliber_import_items", "align_key", "VARCHAR(64) DEFAULT ''"),
# 独立语义指标库升级:支持多表 JOIN / CTE 参数化 SQL 模板口径,存量卡片默认保持单表聚合(expression)
("semantic_metrics", "calc_mode", "VARCHAR(24) DEFAULT 'expression'"),
("semantic_metrics", "sql_template", "TEXT DEFAULT ''"),
]
#: 迁移补上的列要建的索引:(索引名, 表, 列或多列)。create_all 不碰已有的表,表上新列的索引也就不会建。
+3
View File
@@ -1432,3 +1432,6 @@ from app.db import models_cluster # noqa: E402,F401
# 指标本体穿透(W1-a):标准业务实体与跨源映射
from app.db import models_ontology # noqa: E402,F401
from app.db.models_ontology import CanonicalEntity # noqa: E402,F401
# 独立语义指标库(即存即用指标卡片)与文档拆解后台任务:与受管工作流口径并存
from app.db import models_metrics # noqa: E402,F401
from app.db.models_metrics import SemanticDecomposeJob, SemanticMetric # noqa: E402,F401
+104
View File
@@ -0,0 +1,104 @@
"""独立语义指标库(SemanticMetric)与口径文档拆解后台任务(SemanticDecomposeJob)持久化模型。
由 app/db/models.py 末尾整模块导入,create_all 自动建表,存量表新列由 app/db/base.py:_COLUMN_MIGRATIONS 补齐(兼容 SQLite 与 PostgreSQL)。
设计目的:
- 解耦原先强绑定在「多节点工作流 + 单行单元格抽取 + 多重发布门禁」上的繁重口径模式;
- 同时支持「单表聚合表达式(expression)」与「多表 JOIN / CTE 参数化 SQL 模板(sql_template)」两种确定性计算模式;
- status="enabled" 即存即生效,直接供问数入口语义预命中(chat/grounding.py)与内置工具 metric_query 按任意日期区间或业务维度动态聚合取数;
- SemanticDecomposeJob 支持后台异步执行「从目录与文档智能提取指标」与「文档一键拆解工作流(字段目录补丁 + 指标口径 + 标准实体 + 知识库归档)」,实时持久化阶段进度与待审阅结果。
"""
from __future__ import annotations
from datetime import datetime
from typing import Any
from sqlalchemy import Boolean, Integer, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from app.db.base import Base, TimestampMixin, UTCDateTime, new_id
class SemanticMetric(Base, TimestampMixin):
"""独立语义指标卡片定义(即存即用、支持单表聚合与多表参数化 SQL 模板)。"""
__tablename__ = "semantic_metrics"
id: Mapped[str] = mapped_column(String(32), primary_key=True, default=new_id)
#: 机器稳定标识(小写字母开头,小写字母/数字/下划线,最长 64 字符,如 zy_daily_footfall_sys)
key: Mapped[str] = mapped_column(String(64), unique=True, index=True)
#: 指标中文名称(如「张园日客流(系统去重)」)
name: Mapped[str] = mapped_column(String(200))
#: 业务别名 / 同义词列表(如 ["客流", "总客流", "日客流量"])
aliases: Mapped[list[Any]] = mapped_column(default=list)
#: 口径业务说明与注意事项(如统计口径、去重规则、T+1 延迟说明)
description: Mapped[str] = mapped_column(Text, default="")
#: 展示单位(如「人次」「元」「%」)
unit: Mapped[str] = mapped_column(String(40), default="")
#: 归属数据源 id(data_sources.id)
source_id: Mapped[str] = mapped_column(String(32), index=True)
#: 计算模式:expression(单表聚合表达式) | sql_template(多表 JOIN / CTE 参数化 SQL 模板)
calc_mode: Mapped[str] = mapped_column(String(24), default="expression")
#: 主物理底表名(expression 模式下为查询底表;sql_template 模式下为主锚点表)
table_name: Mapped[str] = mapped_column(String(255))
#: 只读聚合 SQL 表达式(expression 模式必填;sql_template 模式可作外层聚合表达式或展示摘要)
sql_expression: Mapped[str] = mapped_column(Text)
#: 多表 / 参数化只读 SQL 模板(sql_template 模式使用,支持 {{select_dims}}、{{and_where}}、{{group_by}}、{{period_days}} 等槽位)
sql_template: Mapped[str] = mapped_column(Text, default="")
#: 业务时间列名(可选,如 stat_date 或多表别名 s.stat_date;为空表示无时间序列维度)
date_column: Mapped[str] = mapped_column(String(128), default="")
#: 默认有效记录过滤条件(可选 WHERE 子表达式,如 status = 1 AND is_deleted = 0)
filter_sql: Mapped[str] = mapped_column(Text, default="")
#: 允许按维度下钻与过滤的列定义:[{key, label, column, entity_key?}]
dimensions: Mapped[list[Any]] = mapped_column(default=list)
#: 默认时间聚合粒度:day | month | hour | none
default_time_grain: Mapped[str] = mapped_column(String(24), default="day")
#: 状态:enabled(启用,直接参与问数预命中与查询) | draft(草稿) | disabled(停用)
status: Mapped[str] = mapped_column(String(24), default="enabled", index=True)
#: 来源:human(人工创建) | ai(目录/文档智能提取)
source: Mapped[str] = mapped_column(String(24), default="human")
#: 乐观锁版本号(从 1 起,每次修改加 1)
version: Mapped[int] = mapped_column(Integer, default=1)
created_by: Mapped[str | None] = mapped_column(String(100), default=None)
updated_by: Mapped[str | None] = mapped_column(String(100), default=None)
class SemanticDecomposeJob(Base, TimestampMixin):
"""后台口径文档一键拆解与智能提取指标任务(支持实时阶段进度、后台运行与审阅一键应用)。"""
__tablename__ = "semantic_decompose_jobs"
id: Mapped[str] = mapped_column(String(32), primary_key=True, default=new_id)
#: queued | running | succeeded | failed | cancelled
status: Mapped[str] = mapped_column(String(24), default="queued", index=True)
#: 目标数据源 id
source_id: Mapped[str] = mapped_column(String(32), index=True)
source_name: Mapped[str] = mapped_column(String(120), default="")
#: 拆解范围列表:["catalog", "metrics", "entities"]
scopes: Mapped[list[Any]] = mapped_column(default=list)
#: 关联的知识库文档 id 列表
document_ids: Mapped[list[Any]] = mapped_column(default=list)
#: 上传附件 id 列表(entity_docs 暂存解析文本)
attachment_ids: Mapped[list[Any]] = mapped_column(default=list)
#: 直接粘贴的业务文档标题与正文
doc_title: Mapped[str] = mapped_column(String(240), default="")
doc_text: Mapped[str] = mapped_column(Text, default="")
#: 是否同步将粘贴/上传的文档存入知识库(保留判异阈值与边界契约供问数检索)
save_to_kb: Mapped[bool] = mapped_column(Boolean, default=True)
#: 用户补充要求
note: Mapped[str] = mapped_column(Text, default="")
#: 当前阶段标识与文案:preparing | syncing_kb | extracting_catalog | extracting_metrics | extracting_entities | done
phase: Mapped[str] = mapped_column(String(48), default="preparing")
phase_label: Mapped[str] = mapped_column(String(200), default="正在准备数据源与文档上下文")
#: 0 ~ 100 百分比进度
progress: Mapped[int] = mapped_column(Integer, default=0)
#: 分步轨迹:[{key, label, status, detail, elapsed_ms}]
steps: Mapped[list[Any]] = mapped_column(default=list)
#: 结构化拆解产物:{kb_document, catalog_patches, metric_candidates, metric_skipped, entity_operations, guardrail_notes, applied}
result: Mapped[dict[str, Any]] = mapped_column(default=dict)
#: 简要汇总计数:{catalog_count, metric_count, entity_count, kb_synced}
summary: Mapped[dict[str, Any]] = mapped_column(default=dict)
error: Mapped[str | None] = mapped_column(Text, default=None)
started_by: Mapped[str | None] = mapped_column(String(100), default=None)
started_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None)
finished_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None)
+8 -2
View File
@@ -33,6 +33,8 @@ from app.api import chat_turns as chat_turns_api
from app.api import caliber_import as caliber_import_api
# 指标本体穿透(W1-a):标准业务实体与跨口径指标关系网
from app.api import ontology as ontology_api
# 独立语义指标库(即存即用指标卡片与在线试算)
from app.api import semantic_metrics as semantic_metrics_api
from app.api.coded import CodedHTTPException, coded_handler
from app.api.deps import chat_scope, evidence_scope, private_run_scope, require_role, role_by_method, run_scope
from app.auth.keys import key_gate
@@ -143,7 +145,7 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]:
finally:
# 先停调度,再停运行:停运行的途中不该再有新运行被发起。起草、剖析任务停下来标成中断,下次启动不会再续
await scheduler.stop()
from app.data import catalog_jobs, catalog_profile_jobs, entity_discovery, introspect_jobs
from app.data import catalog_jobs, catalog_profile_jobs, entity_discovery, introspect_jobs, semantic_decompose
from app.chat import job as chat_job
from app.eval import runner as eval_runner, snapshots as eval_snapshots
@@ -155,6 +157,7 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]:
await catalog_profile_jobs.manager.shutdown()
await introspect_jobs.manager.shutdown()
await entity_discovery.shutdown()
await semantic_decompose.shutdown()
# 问数据的一轮停下不改那几行(下次启动、或者别的进程照「属主不在了」的规矩接手),也要在运行之前停
await chat_job.shutdown()
# 评测要在运行之前停:它停下时会取消自己发起的作答、算期望的运行,再标中断
@@ -249,7 +252,7 @@ def _register_reapers() -> None:
from app.blueprints import publish_plan
from app.chat import job as chat_job
from app.cluster import reaper
from app.data import catalog_jobs, catalog_profile_jobs, entity_discovery, introspect_jobs
from app.data import catalog_jobs, catalog_profile_jobs, entity_discovery, introspect_jobs, semantic_decompose
from app.eval import runner as eval_runner, snapshots as eval_snapshots
from app.process import actions, tasks
from app.ledger import jobs as draft_jobs, reimport as import_jobs
@@ -266,6 +269,7 @@ def _register_reapers() -> None:
("数据剖析任务", lambda live: catalog_profile_jobs.startup(live=live)),
("探查任务", lambda live: introspect_jobs.startup(live=live)),
("候选实体扫描任务", lambda live: entity_discovery.startup(live=live)),
("口径拆解任务", lambda live: semantic_decompose.startup(live=live)),
("影子起草", lambda live: eval_snapshots.startup(live=live)),
("评测任务", lambda live: eval_runner.startup(live=live)),
("问数据的一轮", _chat_reaper(chat_job)),
@@ -436,6 +440,8 @@ for router, guard in (
(caliber_import_api.router, _BUILDER),
# 指标本体穿透(W1-a):标准业务实体与指标本体图谱(搭建者及以上)
(ontology_api.router, _BUILDER),
# 独立语义指标库(即存即用指标卡片):同口径与本体工作台权限(搭建者及以上)
(semantic_metrics_api.router, _BUILDER),
# 下一轮第 2 波:评测只给搭建者;访问密钥登录即可,接口里再按本人 / 管理员判
(evals_api.router, _BUILDER),
# 口径账本里按答案出具、对账的(第 3 波):登录即可,接口里按运行和应用授权判
+172
View File
@@ -0,0 +1,172 @@
"""独立语义指标库内置只读工具:metric_query。
设计原则:
1. 只执行 status="enabled" 的已启用指标卡片(SemanticMetric),确定性编译为单条只读 SELECT 语句。
2. 底层委托对应数据源的 `db_query__<源名>` 工具执行:自动继承上传快照版本锁定(data_versions)、
普通使用者结果遮罩(MASK_RESULTS_OPTION)、只读守卫(READ_ONLY_OPTION)、数据目录 SQL 检查(SqlChecker)
以及内容寻址查询快照落盘(query_snapshot),运行证据链与 `db_query__` 完全一致。
3. 支持按日期区间(date_from / date_to)、时间序列趋势(group_by_date / time_grain)、业务维度下钻(group_by)
与标准业务实体值过滤(filters,自动把业务别名翻译为目标源底层编码)。
4. 当单日查询返回空或全 NULL 且配置了业务时间列时,在同一次调用结果内附带底表最新可用日期 `latest_available_date`,
消除 Agent 多轮探查开销。
"""
from __future__ import annotations
import json
from typing import Any
from pydantic import BaseModel, Field, model_validator
from app.data.entities import list_entities
from app.data.semantic_metrics import (
MetricInvalid,
MetricNotFound,
compile_metric_sql,
get_metric,
is_empty_metric_result,
probe_latest_metric_date,
)
from app.db.base import SessionLocal
from app.db.models import DataSource
from app.tools.datasource import QUERY_PREFIX
from app.tools.registry import ToolContext, ToolGuide, Versions, build_tools, register
METRIC_QUERY = "metric_query"
METRIC_GUIDE = ToolGuide(
id="semantic_metric.method@1",
text=(
"【语义指标查询纪律】\n"
"当用户提问命中已启用的独立语义指标(metric_key)时,优先调用 `metric_query` 获取标准口径结果:\n"
"- 查某一天传 `date_from` 与 `date_to` 为同一日期;查趋势或近 N 日明细传区间并设 `group_by_date=true`;\n"
"- 按业务维度拆分传 `group_by`(填指标声明的维度 key);按实体或维度筛选传 `filters`(可直接传业务名称,系统自动翻译底层编码);\n"
"- 若返回结果附带 `latest_available_date`(例如当日 T+1 尚未入库),可据此说明当日暂无数据或按需再查最新日期。"
),
)
class MetricQueryArgs(BaseModel):
"""按已启用的语义指标卡片(SemanticMetric)执行确定性聚合取数。"""
metric: str = Field(description="要查询的语义指标 key、指标 id 或唯一指标名称")
date_from: str | None = Field(default=None, description="可选:起始业务日期 YYYY-MM-DD(闭区间)")
date_to: str | None = Field(default=None, description="可选:结束业务日期 YYYY-MM-DD(闭区间)")
group_by_date: bool = Field(default=False, description="是否按时间序列分组输出趋势(如逐日/逐月)")
time_grain: str | None = Field(
default=None,
description="可选:时间分组粒度 day | month | hour(配合 group_by_date=true 使用,默认按指标设定)",
)
group_by: list[str] = Field(
default_factory=list,
description="可选:按指标声明的维度 key 分组下钻,例如 ['area', 'store']",
)
filters: dict[str, Any] = Field(
default_factory=dict,
description="可选:按指标声明的维度过滤,例如 {'area': '西区'}(关联了标准业务实体的维度会自动翻译物理编码)",
)
limit: int = Field(default=200, ge=1, le=1000, description="分组查询最多返回行数,默认 200")
@model_validator(mode="before")
@classmethod
def _accept_metric_key_aliases(cls, data: Any) -> Any:
if isinstance(data, dict) and "metric" not in data:
for alt in ("metric_key", "key", "metric_id"):
if alt in data and data[alt] is not None:
copied = dict(data)
copied["metric"] = copied.pop(alt)
return copied
return data
@register(
name=METRIC_QUERY,
description="按已启用的独立语义指标卡片查询标准口径数值、时间趋势或维度拆解明细(自动应用有效记录过滤与标准业务实体编码转换)",
category="data",
args_schema=MetricQueryArgs,
dangerous=False,
needs_context=True,
guide=METRIC_GUIDE,
)
async def run_metric_query(
ctx: ToolContext,
metric: str,
date_from: str | None = None,
date_to: str | None = None,
group_by_date: bool = False,
time_grain: str | None = None,
group_by: list[str] | None = None,
filters: dict[str, Any] | None = None,
limit: int = 200,
) -> str:
async with SessionLocal() as session:
try:
row = await get_metric(session, metric)
except MetricNotFound as exc:
return f"查询失败:{exc}"
if row.status != "enabled":
return f"查询失败:指标「{row.name}({row.key})」当前状态为 {row.status},未启用"
ds = await session.get(DataSource, row.source_id)
if ds is None or not ds.enabled:
return f"查询失败:指标「{row.name}」关联的数据源不可用或已停用"
entities = await list_entities(session)
try:
sql, alignments = compile_metric_sql(
row,
source=ds,
entities=entities,
date_from=date_from,
date_to=date_to,
group_by_date=group_by_date,
time_grain=time_grain,
group_by=group_by or [],
filters=filters or {},
limit=limit,
)
except MetricInvalid as exc:
return f"查询失败:{exc}"
sub_ctx = ctx
if sub_ctx.data_versions is Versions.UNSET and not sub_ctx.run_id:
# 工具试用等非运行上下文默认读当前版本
from dataclasses import replace
sub_ctx = replace(ctx, data_versions=Versions.CURRENT)
query_tool_name = f"{QUERY_PREFIX}{ds.name}"
built = await build_tools([query_tool_name], sub_ctx, session=session)
if not built:
return f"查询失败:无法初始化数据源「{ds.name}」的查询工具"
raw_res = await built[0].ainvoke({"sql": sql, "limit": limit})
raw_str = raw_res if isinstance(raw_res, str) else json.dumps(raw_res, ensure_ascii=False, default=str)
if not raw_str.lstrip().startswith("{"):
return raw_str
try:
payload = json.loads(raw_str)
except ValueError:
return raw_str
payload["metric"] = {
"id": row.id,
"key": row.key,
"name": row.name,
"unit": row.unit or "",
}
if alignments:
payload["entity_alignments"] = alignments
has_groups = bool(group_by_date or group_by)
rows_data = payload.get("rows") if isinstance(payload.get("rows"), list) else []
if is_empty_metric_result(rows_data, has_groups=has_groups) and row.date_column:
latest_date = await probe_latest_metric_date(
ds,
table_name=row.table_name,
date_column=row.date_column,
filter_sql=row.filter_sql or "",
)
if latest_date:
payload["latest_available_date"] = latest_date
return json.dumps(payload, ensure_ascii=False, default=str)
+1
View File
@@ -157,6 +157,7 @@ def all_specs() -> dict[str, ToolSpec]:
import app.tools.builtin # noqa: F401 触发内置工具注册
import app.ledger.querycore # noqa: F401 触发 caliber_query(K10 的父 Run 工具入口)注册
import app.tools.ontology # noqa: F401 触发 ontology_neighbors / source_health 注册
import app.tools.metrics # noqa: F401 触发 metric_query(独立语义指标查询)注册
return dict(_REGISTRY)
+3
View File
@@ -42,6 +42,7 @@ import { blueprints, files, personal, templates } from './blueprints'
// 问数据的一轮在服务端跑(编排完善第 1 波):发起、看、停,实时流在 chatTurns.ts 的 streamChatTurn
import { chatTurns } from './chatTurns'
import { ontology } from './ontology'
import { semanticMetrics } from './semanticMetrics'
// 请求层搬到了 http.ts(见那边的说明)。原先从这里导出的照旧从这里导出,调用方不用改;
// 请求函数也一并转出,给不在 api/ 下、又想直接发请求的地方用。api/ 下的命名空间文件请直接从 http.ts 导入
@@ -1009,6 +1010,8 @@ export const api = {
chatTurns,
// 业务本体:标准业务实体与跨口径指标拓扑
ontology,
// 独立语义指标库(SemanticMetric):即存即生效指标卡片、在线试算与智能提取
semanticMetrics,
}
/**
+283
View File
@@ -0,0 +1,283 @@
/**
* 独立语义指标库(SemanticMetric)与后台文档一键拆解工作流 REST 客户端(服务端 backend/app/api/semantic_metrics.py)。
*
* 与工作流受管口径(caliberLedger)互补:
* - 单表模式(expression):绑定「数据源 + 物理表 + 单表聚合 SQL 表达式」;
* - 多表 / 参数化模式(sql_template):支持跨表 JOIN、WITH CTE 预聚合与 {{select_dims}}、{{and_where}}、{{period_days}} 等安全槽位;
* - 后台一键拆解任务(decomposeJobs):喂入业务文档后在后台分阶段拆解「字段目录补丁 + 单表/多表指标口径 + 标准业务实体 + 知识库归档」。
*/
import { del, get, post, put, qs, upload, type RequestOptions, type UploadOptions } from './http'
export type MetricCalcMode = 'expression' | 'sql_template'
export interface SemanticMetricDimension {
key: string
label: string
column: string
entity_key?: string
}
export interface SemanticMetric {
id: string
key: string
name: string
aliases: string[]
description: string
unit: string
source_id: string
source_name?: string
calc_mode?: MetricCalcMode | string
table_name: string
sql_expression: string
sql_template?: string
aggregation_type: string
date_column: string
filter_sql: string
dimensions: SemanticMetricDimension[]
default_time_grain: string
status: 'enabled' | 'disabled' | 'draft' | string
source: 'human' | 'ai' | 'doc' | string
version: number
created_by?: string | null
updated_by?: string | null
created_at?: string | null
updated_at?: string | null
}
export interface SemanticMetricInput {
key: string
name: string
aliases?: string[]
description?: string
unit?: string
source_id: string
calc_mode?: MetricCalcMode | string
table_name?: string
sql_expression?: string
sql_template?: string
date_column?: string
filter_sql?: string
dimensions?: SemanticMetricDimension[]
default_time_grain?: string
status?: string
source?: string
expected_version?: number
exists_in_db?: boolean
}
export interface SemanticMetricPreviewIn {
metric_id?: string
definition?: Record<string, unknown>
date_from?: string | null
date_to?: string | null
group_by_date?: boolean
time_grain?: string | null
group_by?: string[]
filters?: Record<string, unknown>
limit?: number
}
export interface SemanticMetricPreviewOut {
sql: string
source_name: string
columns: string[]
rows: unknown[][]
row_count: number
truncated: boolean
elapsed_ms: number
entity_alignments: Record<string, unknown>[]
latest_available_date?: string | null
}
export interface SemanticMetricSuggestOut {
source_id: string
source_name: string
model_id?: string | null
documents_used: string[]
candidates: SemanticMetricInput[]
skipped: { key: string; reason: string }[]
}
export interface DecomposeAttachmentOut {
attachment_id: string
title: string
mime: string
char_count: number
chunk_count: number
saved_to_kb: boolean
document_id?: string | null
collection?: string | null
}
export interface SemanticDecomposeStep {
key: string
label: string
status: 'pending' | 'running' | 'succeeded' | 'skipped' | 'failed' | string
detail: string
count: number
elapsed_ms: number
}
export interface SemanticCatalogChangeItem {
id: string
table: string
path: string
slot_label: string
before: unknown
after: unknown
value: unknown
reason: string
state: string
}
export interface SemanticCatalogPatchTable {
table: string
if_version: number
changes: SemanticCatalogChangeItem[]
}
export interface SemanticEntityChange {
id: string
op: string
entity_key: string
entity_label: string
summary: string
before: unknown
after: unknown
reason: string
op_payload: Record<string, unknown>
}
export interface SemanticGuardrailNote {
title: string
rule: string
category: string
}
export interface SemanticDecomposeAppliedRecord {
applied_at: string
applied_by?: string | null
catalog_applied_count: number
catalog_errors: string[]
metrics_created_count: number
metrics_updated_count: number
metric_errors: { key: string; reason: string }[]
entities_applied_count: number
entity_errors: string[]
}
export interface SemanticDecomposeResult {
model_id?: string | null
kb_documents?: { id: string; title: string; created_now: boolean }[]
guardrail_notes?: SemanticGuardrailNote[]
catalog_patches?: SemanticCatalogPatchTable[]
catalog_skipped?: string[]
metric_candidates?: SemanticMetricInput[]
metric_skipped?: { key: string; reason: string }[]
entity_changes?: SemanticEntityChange[]
entity_skipped?: string[]
applied?: SemanticDecomposeAppliedRecord | null
}
export interface SemanticDecomposeJob {
id: string
status: 'queued' | 'running' | 'succeeded' | 'failed' | 'cancelled' | string
source_id: string
source_name: string
scopes: string[]
document_ids: string[]
attachment_ids: string[]
doc_title: string
has_doc_text: boolean
save_to_kb: boolean
note: string
phase: string
phase_label: string
progress: number
steps: SemanticDecomposeStep[]
summary: Record<string, unknown>
result?: SemanticDecomposeResult
error?: string | null
started_by?: string | null
started_at?: string | null
finished_at?: string | null
created_at?: string | null
updated_at?: string | null
}
export interface SemanticDecomposeCreateIn {
source_id: string
scopes?: string[]
table_names?: string[]
document_ids?: string[]
attachment_ids?: string[]
doc_title?: string
doc_text?: string
save_to_kb?: boolean
note?: string
}
export interface SemanticDecomposeApplyIn {
catalog_patches?: { table: string; changes: { path: string; value: unknown; reason: string }[] }[]
metrics?: SemanticMetricInput[]
entity_operations?: Record<string, unknown>[]
overwrite_existing_metrics?: boolean
}
export const semanticMetrics = {
list: (params?: { source_id?: string; status?: string; q?: string }, opts?: RequestOptions) =>
get<{ items: SemanticMetric[]; total: number }>(
`/semantic-metrics${qs({
source_id: params?.source_id,
status: params?.status,
q: params?.q,
})}`,
opts,
),
get: (id: string, opts?: RequestOptions) =>
get<{ metric: SemanticMetric }>(`/semantic-metrics/${encodeURIComponent(id)}`, opts),
create: (body: SemanticMetricInput) =>
post<{ metric: SemanticMetric }>('/semantic-metrics', body),
batchCreate: (items: SemanticMetricInput[]) =>
post<{ created: SemanticMetric[]; skipped: { key: string; reason: string }[] }>('/semantic-metrics/batch', { items }),
update: (id: string, body: Partial<SemanticMetricInput>) =>
put<{ metric: SemanticMetric }>(`/semantic-metrics/${encodeURIComponent(id)}`, body),
remove: (id: string) =>
del(`/semantic-metrics/${encodeURIComponent(id)}`),
preview: (body: SemanticMetricPreviewIn) =>
post<SemanticMetricPreviewOut>('/semantic-metrics/preview', body),
suggest: (body: { source_id: string; table_names?: string[]; document_ids?: string[]; note?: string }) =>
post<SemanticMetricSuggestOut>('/semantic-metrics/suggest', body),
uploadAttachment: (file: File, opts?: { save_to_kb?: boolean; collection?: string } & UploadOptions) => {
const form = new FormData()
form.append('file', file)
if (opts?.save_to_kb !== undefined) form.append('save_to_kb', String(opts.save_to_kb))
if (opts?.collection) form.append('collection', opts.collection)
return upload<DecomposeAttachmentOut>('/semantic-metrics/attachments', form, opts)
},
decomposeJobs: {
active: (sourceId?: string, opts?: RequestOptions) =>
get<{ active: SemanticDecomposeJob | null; latest: SemanticDecomposeJob | null }>(
`/semantic-metrics/decompose-jobs/active${qs({ source_id: sourceId })}`,
opts,
),
list: (sourceId?: string, limit = 10, opts?: RequestOptions) =>
get<{ items: SemanticDecomposeJob[] }>(
`/semantic-metrics/decompose-jobs${qs({ source_id: sourceId, limit })}`,
opts,
),
create: (body: SemanticDecomposeCreateIn) =>
post<{ job: SemanticDecomposeJob }>('/semantic-metrics/decompose-jobs', body),
get: (jobId: string, opts?: RequestOptions) =>
get<{ job: SemanticDecomposeJob }>(`/semantic-metrics/decompose-jobs/${encodeURIComponent(jobId)}`, opts),
cancel: (jobId: string) =>
post<{ job: SemanticDecomposeJob }>(`/semantic-metrics/decompose-jobs/${encodeURIComponent(jobId)}/cancel`, {}),
apply: (jobId: string, body: SemanticDecomposeApplyIn) =>
post<{
job_id: string
applied: SemanticDecomposeAppliedRecord
created_metrics: SemanticMetric[]
updated_metrics: SemanticMetric[]
}>(`/semantic-metrics/decompose-jobs/${encodeURIComponent(jobId)}/apply`, body),
},
}
+335 -140
View File
@@ -1,25 +1,32 @@
/**
* 口径目录(/ledger/calibers):已发布口径的入口、指标、依赖和试跑。
* 口径与指标目录(/ledger/calibers):
*
* 只读目录 + 参数表单 + 一次只读探索运行。口径的技术身份是「工作流@版本/口径卡」;目录给业务的人看的是名字、指标、
* 入口字段和能不能用。试跑按固定版本派生只读图(服务端 ledger/derive.caliber_probe),不是正式出具:来源记 trial,
* 不进使用统计。SQL 不在页面上出现——服务端 descriptor 本来就不返回。
* 1. 独立语义指标卡片(SemanticMetric):即存即生效,直接绑定数据源、物理表、聚合 SQL 表达式、时间列与标准业务实体维度,
* 保存后自动接入问数页「快速直查」与 metric_query 工具;支持在线试算与从数据目录 / 业务文档批量提取。
* 2. 工作流受管口径(CaliberDescriptor):固定工作流版本@口径卡,用于多步编排与正式出具对账。
*/
import { useEffect, useMemo, useState } from 'react'
import { Link } from 'react-router-dom'
import { ChevronDown, Layers, Play, Plus, RefreshCw, Search } from 'lucide-react'
import { ChevronDown, Layers, Play, Plus, RefreshCw, Search, Sparkles, Zap } from 'lucide-react'
import clsx from 'clsx'
import { api } from '../../api/client'
import type { SemanticDecomposeJob, SemanticMetric } from '../../api/semanticMetrics'
import { SchemaForm } from '../../components/SchemaForm'
import { EmptyState, ErrorState, Notice, PageHeader, Skeleton, Spinner, toast } from '../../components/ui'
import { LEDGER_TEXT } from '../../lib/terms'
import { LEDGER_UI } from '../../lib/termsLedger'
import { useAtLeast } from '../../store/auth'
import {
DecomposeJobBanner, MetricEditorModal, MetricSuggestModal, SemanticMetricCardsSection,
} from './SemanticMetricStudio'
import { TrialResult, TrialValidation } from './TrialResult'
import type { FormSchema } from '../../typesApp'
import type { CaliberDescriptor, CaliberTrialOut } from '../../typesLedger'
import type { CaliberDescriptor, CaliberTrialOut, CanonicalEntity } from '../../typesLedger'
const keyOf = (d: CaliberDescriptor) => `${d.workflow_id}@${d.version}/${d.card.node_id}`
type ViewFilter = 'all' | 'metrics' | 'calibers'
function defaultsOf(schema: FormSchema): Record<string, unknown> {
const out: Record<string, unknown> = {}
for (const [name, field] of Object.entries(schema.properties ?? {})) {
@@ -29,7 +36,17 @@ function defaultsOf(schema: FormSchema): Record<string, unknown> {
}
export function CaliberCatalog() {
const builder = useAtLeast('builder')
const [items, setItems] = useState<CaliberDescriptor[] | null>(null)
const [metrics, setMetrics] = useState<SemanticMetric[]>([])
const [entities, setEntities] = useState<CanonicalEntity[]>([])
const [filter, setFilter] = useState<ViewFilter>('all')
const [editorOpen, setEditorOpen] = useState(false)
const [editingMetric, setEditingMetric] = useState<SemanticMetric | null>(null)
const [suggestOpen, setSuggestOpen] = useState(false)
const [bannerJob, setBannerJob] = useState<SemanticDecomposeJob | null>(null)
const [modalInitialJob, setModalInitialJob] = useState<SemanticDecomposeJob | null>(null)
const [dismissedJobId, setDismissedJobId] = useState<string | null>(null)
const [error, setError] = useState<unknown>(null)
const [query, setQuery] = useState('')
const [open, setOpen] = useState<string | null>(null)
@@ -37,6 +54,29 @@ export function CaliberCatalog() {
const [trials, setTrials] = useState<Record<string, CaliberTrialOut>>({})
const [busy, setBusy] = useState<string | null>(null)
const refreshDecomposeBanner = () => {
if (!builder) return
api.semanticMetrics.decomposeJobs.active().then(
({ active, latest }) => {
setBannerJob(active ?? latest ?? null)
},
() => {},
)
}
const loadMetrics = () => {
if (!builder) return
api.semanticMetrics.list().then(
(res) => setMetrics(res.items ?? []),
() => setMetrics([]),
)
api.ontology.listEntities().then(
(res) => setEntities(res.items ?? []),
() => setEntities([]),
)
refreshDecomposeBanner()
}
const load = () => {
setError(null)
setItems(null)
@@ -44,8 +84,44 @@ export function CaliberCatalog() {
(body) => setItems(body.calibers),
(e) => setError(e),
)
loadMetrics()
}
// eslint-disable-next-line react-hooks/exhaustive-deps
useEffect(load, [builder])
// 当有后台口径拆解任务正在排队或执行时,自动轮询最新阶段进度
useEffect(() => {
if (!builder || !bannerJob || (bannerJob.status !== 'queued' && bannerJob.status !== 'running')) return
let alive = true
const timer = window.setInterval(() => {
api.semanticMetrics.decomposeJobs.active().then(
({ active, latest }) => {
if (!alive) return
const next = active ?? latest ?? null
setBannerJob(next)
if (!active && latest?.status === 'succeeded') {
setDismissedJobId(null)
toast.ok('后台口径文档拆解已完成,点击横幅即可审阅并一键应用')
}
},
() => {},
)
}, 1800)
return () => {
alive = false
window.clearInterval(timer)
}
}, [builder, bannerJob?.id, bannerJob?.status])
const handleCancelBannerJob = async (jobId: string) => {
try {
const { job: cancelled } = await api.semanticMetrics.decomposeJobs.cancel(jobId)
setBannerJob(cancelled)
toast.info('已取消后台拆解任务')
} catch (e) {
toast.error(e)
}
}
useEffect(load, [])
const shown = useMemo(() => {
const q = query.trim().toLowerCase()
@@ -54,6 +130,14 @@ export function CaliberCatalog() {
...d.metrics.map((m) => m.name ?? m.id)].some((s) => String(s ?? '').toLowerCase().includes(q)))
}, [items, query])
const shownMetricsCount = useMemo(() => {
const q = query.trim().toLowerCase()
if (!q) return metrics.length
return metrics.filter((m) => [
m.name, m.key, m.table_name, m.source_name, m.description, ...(m.aliases ?? []),
].some((s) => String(s ?? '').toLowerCase().includes(q))).length
}, [metrics, query])
const toggle = (d: CaliberDescriptor) => {
const key = keyOf(d)
if (open === key) { setOpen(null); return }
@@ -77,12 +161,33 @@ export function CaliberCatalog() {
}
}
const showMetrics = filter !== 'calibers'
const showCalibers = filter !== 'metrics'
return (
<div className="flex h-full flex-col" data-caliber-catalog="">
<PageHeader icon={<Layers size={14} />} title="口径" subtitle="已发布口径的入口、指标、依赖与试运行"
<PageHeader icon={<Layers size={14} />} title="口径" subtitle="独立语义指标卡片(即存即生效)与工作流受管口径目录"
actions={<>
<Link to="/ledger/drafts" className="btn btn-xs btn-primary" data-caliber-drafts-link=""
title="从业务文档起草;已经有答案的,在草稿里接一次运行">
{builder && (
<>
<button type="button" className="btn btn-xs btn-primary" data-metric-create-btn=""
onClick={() => { setEditingMetric(null); setEditorOpen(true) }}>
<Zap size={12} aria-hidden /> 新建指标卡片
</button>
<button type="button" className="btn btn-xs" data-metric-suggest-btn=""
onClick={() => {
const activeRunning = bannerJob && (bannerJob.status === 'queued' || bannerJob.status === 'running')
? bannerJob
: null
setModalInitialJob(activeRunning)
setSuggestOpen(true)
}}>
<Sparkles size={12} aria-hidden /> 文档一键拆解 / 提取指标
</button>
</>
)}
<Link to="/ledger/drafts" className={clsx('btn btn-xs', !builder && 'btn-primary')} data-caliber-drafts-link=""
title="从业务文档起草工作流受管口径;已经有答案的,在草稿里接一次运行">
<Plus size={12} aria-hidden /> {LEDGER_UI.drafts.entry}
</Link>
<button type="button" className="btn btn-xs btn-ghost" onClick={load} title="重新加载">
@@ -91,141 +196,231 @@ export function CaliberCatalog() {
</>} />
<div className="min-h-0 flex-1 overflow-y-auto">
<div className="mx-auto max-w-5xl space-y-3 p-4">
<label className="relative block">
<Search size={13} className="pointer-events-none absolute left-2.5 top-1/2 -translate-y-1/2 text-faint" aria-hidden />
<input className="input w-full pl-8" value={query} onChange={(e) => setQuery(e.target.value)}
placeholder="按工作流、口径、指标名查找" aria-label="查找口径" />
</label>
{error ? <ErrorState error={error} onRetry={load} compact />
: !items ? <Skeleton rows={5} height={38} />
: !shown.length ? <EmptyState offline={false} icon={<Layers size={22} />} title="还没有可用的口径"
body={query ? '换个关键词试试' : '在工作流里发布一份带口径卡的版本后,这里就会出现'} />
: shown.map((d) => {
const key = keyOf(d)
const expanded = open === key
const result = trials[key]
return (
<section key={key} className="overflow-hidden rounded-lg border bg-panel" data-caliber={key}>
<button type="button" className="flex w-full items-start gap-3 px-3 py-2.5 text-left hover:bg-hover"
onClick={() => toggle(d)} aria-expanded={expanded} data-caliber-toggle="">
<div className="min-w-0 flex-1">
<div className="flex flex-wrap items-baseline gap-x-2 gap-y-0.5">
<span className="text-sm font-medium">{d.workflow_name}</span>
<span className="text-2xs text-faint">v{d.version}</span>
{d.card.caliber && <span className="text-xs text-dim">· {d.card.caliber}</span>}
<span className="rounded border px-1.5 py-0.5 text-[10px] text-dim">
{(LEDGER_TEXT.level as Record<string, string>)[d.level] ?? d.level}
</span>
</div>
<div className="mt-0.5 text-2xs text-faint">
{d.metrics.map((m) => m.name ?? m.id).join('、')}
{d.entry_schema.properties && Object.keys(d.entry_schema.properties).length > 0
? ` · ${Object.keys(d.entry_schema.properties).length} 个入口字段` : ' · 没有入口字段'}
{d.dependencies.tables.length > 0 ? ` · 读 ${d.dependencies.tables.join('、')}` : ''}
</div>
</div>
<div className="flex shrink-0 items-center gap-2 text-[10px] text-faint">
<span>{d.composable ? '可组合' : '不可组合'}</span>
<span>{d.issuance.auto_issue ? '可自动出具' : '只可一键出具'}</span>
<ChevronDown size={14} className={clsx('transition-transform', expanded && 'rotate-180')} aria-hidden />
</div>
</button>
{expanded && (
<div className="space-y-3 border-t px-3 py-3" data-caliber-detail="">
<div className="grid gap-3 md:grid-cols-2">
<div>
<h3 className="mb-1 text-2xs font-semibold text-dim">指标</h3>
<ul className="space-y-1 text-xs">
{d.metrics.map((m) => (
<li key={m.id} className="border-b border-hairline pb-1 last:border-0">
<div className="flex items-baseline justify-between gap-2">
<span>{m.name ?? m.id}</span>
<span className="text-2xs text-faint">{m.unit || '—'}</span>
</div>
{m.definition?.text && (
<p className="mt-0.5 text-2xs leading-relaxed text-faint">{m.definition.text}</p>
)}
{!!m.params?.length && (
<p className="mt-0.5 text-2xs text-faint">
参数:{m.params.map((p) => `${p.name}(${p.type})`).join('、')}
</p>
)}
</li>
))}
</ul>
</div>
<div>
<h3 className="mb-1 text-2xs font-semibold text-dim">依赖</h3>
<p className="text-2xs leading-relaxed text-dim">
数据源:{d.dependencies.sources.join('、') || '—'}<br />
表:{d.dependencies.tables.join('、') || '—'}<br />
{d.compose_reason ? `不能组合:${d.compose_reason}` : '可以按这一版组合'}
</p>
{d.issuance.reasons.length > 0 && (
<p className="mt-1 text-2xs leading-relaxed text-faint">
出具:{d.issuance.reasons.join(';')}
</p>
)}
</div>
</div>
<div>
<h3 className="mb-1 text-2xs font-semibold text-dim">入口字段</h3>
{d.entry_schema.properties && Object.keys(d.entry_schema.properties).length > 0 ? (
<SchemaForm schema={d.entry_schema} value={values[key] ?? {}}
onChange={(v) => {
setValues((all) => ({ ...all, [key]: v }))
setTrials((all) => {
const next = { ...all }
delete next[key]
return next
})
}}
idPrefix={`caliber-${key}`} />
) : <p className="text-2xs text-faint">这份口径不需要参数。</p>}
</div>
<div className="flex flex-wrap items-center gap-2">
<button type="button" className="btn btn-primary btn-sm" disabled={busy === key}
onClick={() => void trial(d)} data-caliber-trial="">
{busy === key ? <Spinner size={12} /> : <Play size={12} aria-hidden />} 试运行
</button>
<Link to={`/studio/${d.workflow_id}`} className="btn btn-sm btn-ghost">打开工作流</Link>
<span className="text-2xs text-faint">试运行是只读探索运行,不是正式出具</span>
</div>
{result && (
<div className="rounded-md border bg-elev px-3 py-2 text-xs" data-caliber-trial-result={result.status}>
{result.status === 'succeeded' ? (
<ul className="space-y-1">
{result.metrics.map((m) => (
<li key={m.id} className="flex items-baseline justify-between gap-2">
<span>{m.name ?? m.id}</span>
<span className="tnum font-medium">{m.rendered ?? String(m.value ?? '—')}{m.unit ? ` ${m.unit}` : ''}</span>
</li>
))}
</ul>
) : (
<Notice tone={result.status === 'running' ? 'info' : 'err'}>
{result.status === 'running' ? '还在运行:' : '试运行没有成功:'}
{result.error || result.status}
</Notice>
)}
{result.run_id && (
<div className="mt-1">
<Link className="text-2xs text-accent hover:underline" to={`/runs/${result.run_id}`}
data-caliber-trial-run="">
{LEDGER_UI.drafts.trialRun}
</Link>
</div>
)}
{result.status === 'succeeded' && <><TrialValidation report={result.validation} /><TrialResult queries={result.queries} /></>}
</div>
)}
{builder && bannerJob && bannerJob.id !== dismissedJobId && (
<DecomposeJobBanner
job={bannerJob}
onOpenModal={(j) => {
setModalInitialJob(j)
setSuggestOpen(true)
}}
onCancel={(jid) => void handleCancelBannerJob(jid)}
onDismiss={() => setDismissedJobId(bannerJob.id)}
/>
)}
<div className="flex flex-wrap items-center justify-between gap-2">
<div role="group" aria-label="口径视图筛选" className="inline-flex items-center rounded-md border bg-panel p-0.5 text-xs">
<button type="button" aria-pressed={filter === 'all'}
className={clsx('rounded px-2.5 py-1 transition-colors',
filter === 'all' ? 'bg-accent-soft font-medium text-[var(--accent)]' : 'text-dim hover:text-fg')}
onClick={() => setFilter('all')}>
全部({metrics.length + (items?.length ?? 0)})
</button>
<button type="button" aria-pressed={filter === 'metrics'} data-filter-metrics=""
className={clsx('rounded px-2.5 py-1 transition-colors',
filter === 'metrics' ? 'bg-accent-soft font-medium text-[var(--accent)]' : 'text-dim hover:text-fg')}
onClick={() => setFilter('metrics')}>
指标卡片 · 即存即用({metrics.length})
</button>
<button type="button" aria-pressed={filter === 'calibers'} data-filter-calibers=""
className={clsx('rounded px-2.5 py-1 transition-colors',
filter === 'calibers' ? 'bg-accent-soft font-medium text-[var(--accent)]' : 'text-dim hover:text-fg')}
onClick={() => setFilter('calibers')}>
工作流受管口径({items?.length ?? 0})
</button>
</div>
<label className="relative min-w-[240px] flex-1 sm:max-w-sm">
<Search size={13} className="pointer-events-none absolute left-2.5 top-1/2 -translate-y-1/2 text-faint" aria-hidden />
<input className="input w-full pl-8" value={query} onChange={(e) => setQuery(e.target.value)}
placeholder="按指标名、标识、物理表或工作流查找" aria-label="查找口径" />
</label>
</div>
{showMetrics && (
<SemanticMetricCardsSection
metrics={metrics}
entities={entities}
query={query}
onReload={loadMetrics}
onEdit={(m) => { setEditingMetric(m); setEditorOpen(true) }}
/>
)}
{filter === 'metrics' && shownMetricsCount === 0 && (
<EmptyState
offline={false}
icon={<Zap size={22} />}
title="还没有独立语义指标卡片"
body={query ? '换个关键词试试' : '点击右上角「新建指标卡片」或「从目录与文档提取」,保存后无需发布工作流即可在问数快速直查中直接命中'}
/>
)}
{showCalibers && (
error ? <ErrorState error={error} onRetry={load} compact />
: !items ? <Skeleton rows={5} height={38} />
: !shown.length ? (
shownMetricsCount === 0 ? (
<EmptyState offline={false} icon={<Layers size={22} />} title="还没有可用的口径"
body={query ? '换个关键词试试' : '可新建即存即用的指标卡片,或在工作流里发布带口径卡的版本'} />
) : null
) : (
<div className="space-y-2">
{showMetrics && shownMetricsCount > 0 && (
<div className="px-0.5 pt-2 text-xs font-semibold text-dim">
工作流受管口径(共 {shown.length} 项)
</div>
)}
</section>
{shown.map((d) => {
const key = keyOf(d)
const expanded = open === key
const result = trials[key]
return (
<section key={key} className="overflow-hidden rounded-lg border bg-panel" data-caliber={key}>
<button type="button" className="flex w-full items-start gap-3 px-3 py-2.5 text-left hover:bg-hover"
onClick={() => toggle(d)} aria-expanded={expanded} data-caliber-toggle="">
<div className="min-w-0 flex-1">
<div className="flex flex-wrap items-baseline gap-x-2 gap-y-0.5">
<span className="text-sm font-medium">{d.workflow_name}</span>
<span className="text-2xs text-faint">v{d.version}</span>
{d.card.caliber && <span className="text-xs text-dim">· {d.card.caliber}</span>}
<span className="rounded border px-1.5 py-0.5 text-[10px] text-dim">
{(LEDGER_TEXT.level as Record<string, string>)[d.level] ?? d.level}
</span>
</div>
<div className="mt-0.5 text-2xs text-faint">
{d.metrics.map((m) => m.name ?? m.id).join('、')}
{d.entry_schema.properties && Object.keys(d.entry_schema.properties).length > 0
? ` · ${Object.keys(d.entry_schema.properties).length} 个入口字段` : ' · 没有入口字段'}
{d.dependencies.tables.length > 0 ? ` · 读 ${d.dependencies.tables.join('、')}` : ''}
</div>
</div>
<div className="flex shrink-0 items-center gap-2 text-[10px] text-faint">
<span>{d.composable ? '可组合' : '不可组合'}</span>
<span>{d.issuance.auto_issue ? '可自动出具' : '只可一键出具'}</span>
<ChevronDown size={14} className={clsx('transition-transform', expanded && 'rotate-180')} aria-hidden />
</div>
</button>
{expanded && (
<div className="space-y-3 border-t px-3 py-3" data-caliber-detail="">
<div className="grid gap-3 md:grid-cols-2">
<div>
<h3 className="mb-1 text-2xs font-semibold text-dim">指标</h3>
<ul className="space-y-1 text-xs">
{d.metrics.map((m) => (
<li key={m.id} className="border-b border-hairline pb-1 last:border-0">
<div className="flex items-baseline justify-between gap-2">
<span>{m.name ?? m.id}</span>
<span className="text-2xs text-faint">{m.unit || '—'}</span>
</div>
{m.definition?.text && (
<p className="mt-0.5 text-2xs leading-relaxed text-faint">{m.definition.text}</p>
)}
{!!m.params?.length && (
<p className="mt-0.5 text-2xs text-faint">
参数:{m.params.map((p) => `${p.name}(${p.type})`).join('、')}
</p>
)}
</li>
))}
</ul>
</div>
<div>
<h3 className="mb-1 text-2xs font-semibold text-dim">依赖</h3>
<p className="text-2xs leading-relaxed text-dim">
数据源:{d.dependencies.sources.join('、') || '—'}<br />
表:{d.dependencies.tables.join('、') || '—'}<br />
{d.compose_reason ? `不能组合:${d.compose_reason}` : '可以按这一版组合'}
</p>
{d.issuance.reasons.length > 0 && (
<p className="mt-1 text-2xs leading-relaxed text-faint">
出具:{d.issuance.reasons.join(';')}
</p>
)}
</div>
</div>
<div>
<h3 className="mb-1 text-2xs font-semibold text-dim">入口字段</h3>
{d.entry_schema.properties && Object.keys(d.entry_schema.properties).length > 0 ? (
<SchemaForm schema={d.entry_schema} value={values[key] ?? {}}
onChange={(v) => {
setValues((all) => ({ ...all, [key]: v }))
setTrials((all) => {
const next = { ...all }
delete next[key]
return next
})
}}
idPrefix={`caliber-${key}`} />
) : <p className="text-2xs text-faint">这份口径不需要参数。</p>}
</div>
<div className="flex flex-wrap items-center gap-2">
<button type="button" className="btn btn-primary btn-sm" disabled={busy === key}
onClick={() => void trial(d)} data-caliber-trial="">
{busy === key ? <Spinner size={12} /> : <Play size={12} aria-hidden />} 试运行
</button>
<Link to={`/studio/${d.workflow_id}`} className="btn btn-sm btn-ghost">打开工作流</Link>
<span className="text-2xs text-faint">试运行是只读探索运行,不是正式出具</span>
</div>
{result && (
<div className="rounded-md border bg-elev px-3 py-2 text-xs" data-caliber-trial-result={result.status}>
{result.status === 'succeeded' ? (
<ul className="space-y-1">
{result.metrics.map((m) => (
<li key={m.id} className="flex items-baseline justify-between gap-2">
<span>{m.name ?? m.id}</span>
<span className="tnum font-medium">{m.rendered ?? String(m.value ?? '—')}{m.unit ? ` ${m.unit}` : ''}</span>
</li>
))}
</ul>
) : (
<Notice tone={result.status === 'running' ? 'info' : 'err'}>
{result.status === 'running' ? '还在运行:' : '试运行没有成功:'}
{result.error || result.status}
</Notice>
)}
{result.run_id && (
<div className="mt-1">
<Link className="text-2xs text-accent hover:underline" to={`/runs/${result.run_id}`}
data-caliber-trial-run="">
{LEDGER_UI.drafts.trialRun}
</Link>
</div>
)}
{result.status === 'succeeded' && <><TrialValidation report={result.validation} /><TrialResult queries={result.queries} /></>}
</div>
)}
</div>
)}
</section>
)
})}
</div>
)
})}
)}
</div>
</div>
{builder && (
<>
<MetricEditorModal
open={editorOpen}
metric={editingMetric}
entities={entities}
onClose={() => { setEditorOpen(false); setEditingMetric(null) }}
onSaved={loadMetrics}
/>
<MetricSuggestModal
open={suggestOpen}
initialJob={modalInitialJob}
onClose={() => setSuggestOpen(false)}
onSaved={loadMetrics}
onJobUpdated={(nextJob) => {
setBannerJob(nextJob)
setModalInitialJob(nextJob)
if (nextJob) setDismissedJobId(null)
}}
/>
</>
)}
</div>
)
}
File diff suppressed because it is too large Load Diff