mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 00:21:58 +00:00
合入集成分支(阶段 2A 数据剖析、4B 剖析与 SQL 检查界面及修正)到 4A:两边功能都保留
冲突的解法: - 后端 api/catalog.py:同时导入 catalog_impact 和 catalog_profile、SnapshotTampered;data/catalog.py:__all__ 取两边的并集重排。 - 2A 放宽了码值校验(含义可以为空,剖析写进来的是「含义待填写」):目录修改提案的码值补充改为含义为空的码不覆盖已有含义、 只在原来没有这个码时加进去,补了测试;卡片和过程里的码值把空含义写成「含义待填写」,和目录页同一个说法。 - 前端 types.ts、terms.ts:两边各在同一处追加了新段落,按顺序都留下(提案、影响面、目录版本在前,剖析、码值、SQL 检查在后), 没有重复的键;client.ts、decode.ts、CatalogPage.tsx、TableDetail.tsx 的导入取并集。 - check-catalog.mjs:影响面和剖析、保存数据源三条假路由都留下,各节和最后核对真库数量的逻辑不变。 - 4A 的测试数据源改用 tests/fixtures/sources.py 的 drop_source 删除,顺带作废连接池。
This commit is contained in:
@@ -8,9 +8,10 @@
|
||||
- POST /catalog/{table}/patch 保存目录修改提案(乐观锁),记为人工确认
|
||||
- GET /catalog/{table}/impact 影响面:引用这张表的已发布、受管模板(data/catalog_impact.py)
|
||||
- POST /catalog/draft 同步起草(注释、外键、命名推断,可选模型)
|
||||
- POST /catalog/profile 同步剖析:对业务库发少量只读查询,核对关系、取码值候选(data/catalog_profile.py)
|
||||
|
||||
规则都在 data/catalog.py,这里只做取源、转换形状和把异常翻成状态码:版本不符 409,结构不合规、审阅路径
|
||||
不对 422,源或表不存在 404。所有写操作记录请求头 X-Actor 的署名(自报、未认证)。
|
||||
不对 422,源或表不存在 404。所有写操作记录请求头 X-Actor 的署名(自报、未认证)。剖析没开启、正在剖析 409。
|
||||
|
||||
上传表格的源按当前快照的表结构回答(table_versions.resolve_source),和查询实际用的库是同一个版本。
|
||||
"""
|
||||
@@ -23,7 +24,8 @@ from pydantic import BaseModel, Field
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.api.runs import actor_of
|
||||
from app.data import catalog, catalog_impact, introspect, table_versions
|
||||
from app.data import catalog, catalog_impact, catalog_profile, introspect, table_versions
|
||||
from app.data.engine import SnapshotTampered
|
||||
from app.db.base import get_session
|
||||
from app.db.models import DataSource
|
||||
|
||||
@@ -65,6 +67,11 @@ class CatalogDraftIn(BaseModel):
|
||||
use_model: bool = False
|
||||
|
||||
|
||||
class CatalogProfileIn(BaseModel):
|
||||
#: 要剖析的表;不给或为空时按使用次数取前 10 张目录里有待核实关系的表
|
||||
tables: list[str] | None = Field(default=None, max_length=200)
|
||||
|
||||
|
||||
async def _resolved(session: AsyncSession, source_id: str) -> tuple[DataSource, Any]:
|
||||
"""(数据源记录, 用来读表结构的源):上传源解析成绑定当前快照的视图。"""
|
||||
row = await session.get(DataSource, source_id)
|
||||
@@ -227,6 +234,62 @@ async def draft_catalog(source_id: str, payload: CatalogDraftIn,
|
||||
}
|
||||
|
||||
|
||||
def _profile_table_out(t: catalog_profile.TableProfile) -> dict[str, Any]:
|
||||
return {
|
||||
"table_name": t.table,
|
||||
"queries": t.queries,
|
||||
"row_estimate": t.row_estimate.to_dict() if t.row_estimate else None,
|
||||
"findings": t.findings,
|
||||
"skipped": [s.to_dict() for s in t.skipped],
|
||||
"date_ranges": t.date_ranges,
|
||||
"added": t.added,
|
||||
"updated": t.updated,
|
||||
"removed": t.removed,
|
||||
"version": t.version,
|
||||
"error": t.error,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/{source_id}/catalog/profile")
|
||||
async def profile_catalog_tables(source_id: str, payload: CatalogProfileIn,
|
||||
x_actor: str | None = Header(default=None),
|
||||
session: AsyncSession = Depends(get_session)) -> dict[str, Any]:
|
||||
"""同步剖析。返回每张表用了几条查询、写进目录的发现、跳过了什么及原因。
|
||||
|
||||
默认关闭:数据源设置里没开启数据剖析、数据源停用,409;同一个数据源已经在剖析,409。
|
||||
"""
|
||||
row, source = await _resolved(session, source_id)
|
||||
try:
|
||||
catalog_profile.ensure_enabled(row)
|
||||
except catalog_profile.ProfileDisabled as e:
|
||||
raise HTTPException(409, str(e)) from e
|
||||
cache = source.schema_cache or {}
|
||||
if not cache.get("tables"):
|
||||
raise HTTPException(409, f"{introspect.why_empty(cache)},无法进行数据剖析。请先在数据源卡片上点「探查结构」")
|
||||
actor = actor_of(x_actor)
|
||||
try:
|
||||
report = await catalog_profile.profile_catalog(session, row, source, tables=payload.tables or None,
|
||||
actor=actor)
|
||||
except (catalog_profile.ProfileDisabled, catalog_profile.ProfileBusy) as e:
|
||||
raise HTTPException(409, str(e)) from e
|
||||
except SnapshotTampered as e:
|
||||
raise HTTPException(409, str(e)) from e
|
||||
rows = [_profile_table_out(t) for t in report.tables]
|
||||
note = None
|
||||
if not rows:
|
||||
note = "目录里还没有待核实的关联关系,请先起草数据目录,或指定要剖析的表"
|
||||
return {
|
||||
"profiled_at": report.profiled_at,
|
||||
"actor": actor,
|
||||
"settings": report.settings.to_dict(),
|
||||
"queries_used": report.queries_used,
|
||||
"stopped": report.stopped,
|
||||
"tables": rows,
|
||||
"total": {k: sum(r[k] for r in rows) for k in ("added", "updated", "removed")},
|
||||
"note": note,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/{source_id}/catalog/{table}/review")
|
||||
async def review_catalog_item(source_id: str, table: str, payload: CatalogReviewIn,
|
||||
x_actor: str | None = Header(default=None),
|
||||
|
||||
@@ -23,6 +23,7 @@ from app.core.errors import AUTH, NETWORK, TIMEOUT, classify, explain, raw
|
||||
from app.core.config import settings
|
||||
from app.core.crypto import encrypt, mask
|
||||
from app.data import catalog as data_catalog
|
||||
from app.data.catalog_profile import profile_settings_problem
|
||||
from app.data import introspect as introspect_mod
|
||||
from app.data import table_versions
|
||||
from app.data.engine import (
|
||||
@@ -336,7 +337,8 @@ _MASK_FIELD = {
|
||||
|
||||
|
||||
def _refuse_bad_options(options: dict[str, Any] | None) -> None:
|
||||
problem = query_timeout_problem(options) or mask_columns_problem(options)
|
||||
problem = (query_timeout_problem(options) or mask_columns_problem(options)
|
||||
or profile_settings_problem(options))
|
||||
if problem:
|
||||
raise HTTPException(422, problem)
|
||||
|
||||
|
||||
+80
-18
@@ -15,13 +15,14 @@ notes 结构(每个「项」都是 {value, source, status},可选 note、upd
|
||||
"valid_filter": 项(str,SQL 条件片段), "dedup": 项(str),
|
||||
"columns": {列名: {"label": 项, "meaning": 项, "unit": 项,
|
||||
"measure": 项("flow"|"stock"|"ratio"|"identifier"|"status"|"attribute"),
|
||||
"codes": 项(dict[str, str])}},
|
||||
"codes": 项(dict[码值, 含义];含义可为空串,表示待填写)}},
|
||||
"relations": [{"id", "columns", "to_table", "to_columns", "cardinality", "coverage",
|
||||
"source", "status", "note"?}]}
|
||||
|
||||
来源(source):comment 数据库注释、fk 外键约束、name 命名推断、profile 数据剖析(阶段 2)、llm 模型起草、
|
||||
human 人工填写。状态(status):proposed 推断(命名推断、模型起草、数据库注释)、verified 有确证(外键
|
||||
约束;阶段 2 的数据剖析也写这个状态)、confirmed 人工确认、rejected 人工驳回。只有 confirmed 和 verified
|
||||
来源(source):comment 数据库注释、fk 外键约束、name 命名推断、profile 数据剖析(data/catalog_profile.py)、
|
||||
llm 模型起草、human 人工填写。状态(status):proposed 推断(命名推断、模型起草、数据库注释)、verified 有确证
|
||||
(外键约束;数据剖析核实了覆盖率和父键唯一的关系也是,见 profile_relation_holds)、confirmed 人工确认、
|
||||
rejected 人工驳回。只有 confirmed 和 verified
|
||||
将来会触发错误级检查;给模型看时 proposed 的项标「推断,未确认」;rejected 的项留着(防止下次起草又提出
|
||||
来),但模型和检查都看不到。
|
||||
|
||||
@@ -98,6 +99,33 @@ def initial_status(source: str) -> str:
|
||||
return _INITIAL_STATUS.get(source, "proposed")
|
||||
|
||||
|
||||
#: 数据剖析把关系升为有确证的抽样覆盖率下限(data/catalog_profile.py 判定,撤销审阅时按它判回剖析的结论)
|
||||
PROFILE_VERIFY_COVERAGE = 0.95
|
||||
|
||||
|
||||
def profile_relation_holds(rel: Mapping[str, Any]) -> bool:
|
||||
"""一条数据剖析得出的关系站不站得住:抽样覆盖率不低于 PROFILE_VERIFY_COVERAGE,而且被指向列核实唯一。
|
||||
|
||||
剖析只在被指向列核实唯一时才写基数(多对一或一对一),所以「基数有值」就是「父键唯一」。结论全在关系
|
||||
自己的字段里,撤销审阅时照原样判得回来,不必另存一份「审阅前的状态」。
|
||||
"""
|
||||
coverage = rel.get("coverage")
|
||||
return (rel.get("source") == "profile" and isinstance(coverage, (int, float)) and not isinstance(coverage, bool)
|
||||
and coverage >= PROFILE_VERIFY_COVERAGE and rel.get("cardinality") in ("many_to_one", "one_to_one"))
|
||||
|
||||
|
||||
def unreviewed_status(item: Mapping[str, Any]) -> str:
|
||||
"""撤销审阅后回到的状态:来源的初始状态。
|
||||
|
||||
数据剖析是例外:它的状态随数据而定(覆盖率够、父键唯一的关系是有确证,其余是推断),不能一律回到
|
||||
initial_status("profile") 的推断——那样人工确认过再撤销,有确证就无故丢了,要等下一次剖析才回来。
|
||||
剖析提的码值候选、业务日期永远是推断。
|
||||
"""
|
||||
if item.get("source") == "profile" and "to_table" in item:
|
||||
return "verified" if profile_relation_holds(item) else "proposed"
|
||||
return initial_status(item.get("source"))
|
||||
|
||||
|
||||
def make_item(value: Any, source: str, status: str | None = None, *, note: str | None = None,
|
||||
at: str | None = None) -> dict[str, Any]:
|
||||
"""构造一个项。status 不给时按来源取初始状态(initial_status)。"""
|
||||
@@ -153,8 +181,10 @@ def _business_date_ok(value: Any) -> bool:
|
||||
|
||||
|
||||
def _codes_ok(value: Any) -> bool:
|
||||
"""码值对照:{码值: 含义},都是文字。含义可以是空串——数据剖析只知道列里出现过哪些取值,含义要等人填;
|
||||
以前要求含义非空,剖析得到的码值候选就写不进来。"""
|
||||
return (isinstance(value, dict) and bool(value)
|
||||
and all(isinstance(k, str) and isinstance(v, str) and v.strip() for k, v in value.items()))
|
||||
and all(isinstance(k, str) and isinstance(v, str) for k, v in value.items()))
|
||||
|
||||
|
||||
#: 表类型、度量类型在界面上的叫法(和 frontend/src/lib/terms.ts 的 CATALOG_KIND_LABEL / CATALOG_MEASURE_LABEL 一致)。
|
||||
@@ -587,7 +617,7 @@ def review_item(notes: Mapping[str, Any] | None, path: str, action: str, *, at:
|
||||
# 人工填写的项没有「推断时的样子」可回:撤销就是删掉,下次起草可以重新提出
|
||||
del slots[key]
|
||||
return _assemble(slots)
|
||||
item["status"] = {"confirm": "confirmed", "reject": "rejected"}.get(action) or initial_status(item.get("source"))
|
||||
item["status"] = {"confirm": "confirmed", "reject": "rejected"}.get(action) or unreviewed_status(item)
|
||||
item["updated_at"] = at or now_iso()
|
||||
return _assemble(slots)
|
||||
|
||||
@@ -652,7 +682,7 @@ def apply_human_edit(existing: Mapping[str, Any] | None, submitted: Mapping[str,
|
||||
continue
|
||||
kept = dict(prev)
|
||||
status = item.get("status")
|
||||
if status != prev.get("status") and status in ("confirmed", "rejected", initial_status(prev.get("source"))):
|
||||
if status != prev.get("status") and status in ("confirmed", "rejected", unreviewed_status(prev)):
|
||||
kept["status"] = status
|
||||
kept["updated_at"] = at
|
||||
if item.get("note") != prev.get("note"):
|
||||
@@ -779,6 +809,17 @@ def _patch_relation(table: str, raw: Any, path_id: str | None, tables: Mapping[s
|
||||
return value, None
|
||||
|
||||
|
||||
def _merge_codes(before: Mapping[str, str], value: Mapping[str, str]) -> dict[str, str]:
|
||||
"""码值的补充:给出的码加进去或改掉原来的含义。含义为空表示「只知道有这个码、含义待填写」(数据剖析的码值
|
||||
候选就是这样),所以提案里含义为空的码不覆盖已有的含义,只在原来没有这个码时加进去。"""
|
||||
out = dict(before)
|
||||
for code, meaning in value.items():
|
||||
if not str(meaning).strip() and str(out.get(code) or "").strip():
|
||||
continue
|
||||
out[code] = meaning
|
||||
return out
|
||||
|
||||
|
||||
def plan_patch(notes: Mapping[str, Any] | None, changes: Any, *, table: str,
|
||||
tables: Mapping[str, Any] | None = None) -> PatchPlan:
|
||||
"""逐项核对一条目录修改提案,算出改前、改后。不改入参、不抛异常:不合法的项写进 problems。
|
||||
@@ -790,7 +831,8 @@ def plan_patch(notes: Mapping[str, Any] | None, changes: Any, *, table: str,
|
||||
规则:
|
||||
- 取值格式和 validate_notes 同一套(_TABLE_VALUE_RULES / _COLUMN_VALUE_RULES / _relation_problems)。
|
||||
- 码值是补充:在现有(没被驳回的)码值上加或改给出的码,不整份替换——「status=9 表示作废」不该把已确认的
|
||||
其余码值删掉。
|
||||
其余码值删掉。含义为空的码(数据剖析写进来的「含义待填写」)可以被补上含义;提案里含义为空的码只在原来
|
||||
没有这个码时加进去,不会把已有的含义抹掉(_merge_codes)。
|
||||
- 看起来像计算公式的文字不收(_FORMULA):公式进口径卡。
|
||||
- 同一个路径出现两次只认第一次。
|
||||
"""
|
||||
@@ -864,7 +906,7 @@ def plan_patch(notes: Mapping[str, Any] | None, changes: Any, *, table: str,
|
||||
before = _value_of(key, live) if live is not None else None
|
||||
after = value
|
||||
if key[0] == "c" and key[2] == "codes" and isinstance(before, dict):
|
||||
after = {**before, **value}
|
||||
after = _merge_codes(before, value)
|
||||
if before is not None and before == after:
|
||||
state = "same" if live.get("status") == "confirmed" else "confirm"
|
||||
else:
|
||||
@@ -1094,6 +1136,15 @@ def infer_name_relations(schema_cache: Mapping[str, Any] | None, table: str, *,
|
||||
candidates: set[str] = set()
|
||||
for form in _singulars(words[-1]):
|
||||
candidates |= index.get((prefix, form), set())
|
||||
if not candidates:
|
||||
# 直接对不上:库里的表常带业务前缀(store_tags),列名却只写词根(tagId)。
|
||||
# 只认和引用它的表共用前缀的那张,从最长的共用前缀试起;仍然不止一张就不推
|
||||
own = _words(table)
|
||||
for k in range(len(own) - 1, 0, -1):
|
||||
for form in _singulars(words[-1]):
|
||||
candidates |= index.get(("".join(own[:k]) + prefix, form), set())
|
||||
if candidates:
|
||||
break
|
||||
if len(candidates) != 1:
|
||||
continue
|
||||
target = next(iter(candidates))
|
||||
@@ -1625,6 +1676,17 @@ def _date_text(value: Mapping[str, Any]) -> str:
|
||||
return ";".join(parts)
|
||||
|
||||
|
||||
def _codes_text(codes: Mapping[str, Any]) -> str:
|
||||
"""码值对照的文字:有含义的写「1=有效」,含义待填写的单独说明——不能写成「1=」,模型会当成含义就是空。"""
|
||||
filled = [f"{k}={v}" for k, v in codes.items() if str(v).strip()]
|
||||
pending = [str(k) for k, v in codes.items() if not str(v).strip()]
|
||||
if not pending:
|
||||
return "、".join(filled)
|
||||
if not filled:
|
||||
return f"{'、'.join(pending)}(含义待填写)"
|
||||
return f"{'、'.join(filled)},{'、'.join(pending)} 的含义待填写"
|
||||
|
||||
|
||||
def _relation_text(rel: Mapping[str, Any]) -> str:
|
||||
left = ", ".join(rel.get("columns") or [])
|
||||
right = ", ".join(f"{rel.get('to_table')}.{c}" for c in rel.get("to_columns") or [])
|
||||
@@ -1675,7 +1737,7 @@ def _notes_lines(notes: Mapping[str, Any] | None, *, meta: Mapping[str, Any] | N
|
||||
if name == "measure":
|
||||
text = MEASURE_LABEL.get(value, str(value))
|
||||
elif name == "codes":
|
||||
text = "、".join(f"{k}={v}" for k, v in value.items())
|
||||
text = _codes_text(value)
|
||||
else:
|
||||
text = str(value)
|
||||
# 第一段是中文名(没有中文名时是含义),不加前缀;其余写明是什么
|
||||
@@ -1932,13 +1994,13 @@ __all__ = [
|
||||
"CATALOG_DRAFT_SYSTEM", "COLUMN_FIELDS", "COLUMN_FIELD_LABEL", "COLUMN_MEASURES", "CatalogConflict",
|
||||
"CatalogEntry", "CatalogInvalid", "CatalogModelUnavailable", "CatalogPathError", "DRAFT_DEFAULT_TABLES",
|
||||
"DraftReport", "INFERRED_MARK", "ITEM_SOURCES", "ITEM_STATUSES", "JoinEdge", "KIND_LABEL", "MEASURE_LABEL",
|
||||
"MergeStats", "NEW_RELATION_PATH", "PATCH_MAX_CHANGES", "PatchChange", "PatchPlan", "QUERY_SNAPSHOT_KIND",
|
||||
"REVIEW_ACTIONS", "TABLE_FIELDS", "TABLE_FIELD_LABEL", "TABLE_KINDS", "TableDraft", "TableDraftResult",
|
||||
"UI_KIND_LABEL", "UI_MEASURE_LABEL", "apply_human_edit", "apply_patch", "describe_slot", "draft_catalog",
|
||||
"draft_structure", "draft_with_model", "fk_relations", "frozen_catalog", "infer_name_relations",
|
||||
"MergeStats", "NEW_RELATION_PATH", "PATCH_MAX_CHANGES", "PROFILE_VERIFY_COVERAGE", "PatchChange", "PatchPlan",
|
||||
"QUERY_SNAPSHOT_KIND", "REVIEW_ACTIONS", "TABLE_FIELDS", "TABLE_FIELD_LABEL", "TABLE_KINDS", "TableDraft",
|
||||
"TableDraftResult", "UI_KIND_LABEL", "UI_MEASURE_LABEL", "apply_human_edit", "apply_patch", "describe_slot",
|
||||
"draft_catalog", "draft_structure", "draft_with_model", "fk_relations", "frozen_catalog", "infer_name_relations",
|
||||
"initial_status", "join_paths", "make_item", "merge_notes", "now_iso", "parse_path", "plan_patch",
|
||||
"query_snapshot_meta", "read_catalog", "read_entry", "relation_graph", "relation_id", "render_table_index",
|
||||
"render_table_notes", "resolve_draft_model", "resolve_table_name", "review_entry", "review_item", "same_notes",
|
||||
"save_human_edit", "save_patch", "status_counts", "system_notes_source", "table_usage", "validate_notes",
|
||||
"visible_notes", "write_entry",
|
||||
"profile_relation_holds", "query_snapshot_meta", "read_catalog", "read_entry", "relation_graph", "relation_id",
|
||||
"render_table_index", "render_table_notes", "resolve_draft_model", "resolve_table_name", "review_entry",
|
||||
"review_item", "same_notes", "save_human_edit", "save_patch", "status_counts", "system_notes_source",
|
||||
"table_usage", "unreviewed_status", "validate_notes", "visible_notes", "write_entry",
|
||||
]
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -58,11 +58,16 @@ MAX_QUERY_TIMEOUT_S = 600
|
||||
#: 完整快照仍能按工件 id 取到,SQL 里给列起个别名也能绕开
|
||||
MASK_COLUMNS_OPTION = "mask_columns"
|
||||
|
||||
#: 数据源 options 里的数据剖析设置(一个对象:开关、查询次数上限、单条时限……)。取值和校验见
|
||||
#: data/catalog_profile.py;放在这里只为让拼连接串时摘掉它
|
||||
CATALOG_PROFILE_OPTION = "catalog_profile"
|
||||
|
||||
# options 里这些 key 是 AgentLab 自己的配置,不是驱动参数,拼 URL 时要摘掉。
|
||||
# schema:探查哪个 schema(企业库里只读账号名下常常什么都没有,数据在别处)
|
||||
# query_timeout_s:查询时限,由数据层按语句下发给数据库,见 _server_deadline
|
||||
# mask_columns:证据面板遮罩的列,驱动不认识它,拼进连接串会被当成未知参数拒掉
|
||||
_NON_DRIVER_OPTIONS = frozenset({"schema", QUERY_TIMEOUT_OPTION, MASK_COLUMNS_OPTION})
|
||||
# catalog_profile:数据剖析设置,同上;它还是个对象,拼进去连 URL 都不成形
|
||||
_NON_DRIVER_OPTIONS = frozenset({"schema", QUERY_TIMEOUT_OPTION, MASK_COLUMNS_OPTION, CATALOG_PROFILE_OPTION})
|
||||
|
||||
_MASK_SPLIT = re.compile(r"[,,、;;\n]+")
|
||||
|
||||
|
||||
@@ -169,13 +169,17 @@ class SqlChecker:
|
||||
和助手挑表看到的是同一张图(catalog.relation_graph):目录里的关系,加上没起草过的表按外键、命名现推的;
|
||||
被驳回的关系不进图,也不会被现推回来。
|
||||
"""
|
||||
return [e for e in self.edges_from(a) if e.to_table.lower() == b.lower()]
|
||||
|
||||
def edges_from(self, a: str) -> list[catalog.JoinEdge]:
|
||||
"""从 a 出发的全部边。关系图第一次用到时才建,之后复用。"""
|
||||
if self._edges is None:
|
||||
graph = catalog.relation_graph(self._raw, schema_cache=self.schema_cache)
|
||||
index: dict[str, list[catalog.JoinEdge]] = {}
|
||||
for table, edges in graph.items():
|
||||
index.setdefault(table.lower(), []).extend(edges)
|
||||
self._edges = index
|
||||
return [e for e in self._edges.get(a.lower(), []) if e.to_table.lower() == b.lower()]
|
||||
return list(self._edges.get(a.lower(), []))
|
||||
|
||||
# ---- 检查
|
||||
|
||||
@@ -557,10 +561,46 @@ class _ScopeCheck:
|
||||
continue # 两张表都没写目录:没有可对照的,不判
|
||||
condition = " AND ".join(filter(None, (self.excerpt(c) for c in group["conds"])))
|
||||
if best is None:
|
||||
self._join_checks.append(self._join_unknown(a, b, candidates, condition))
|
||||
parent = self._shared_parent(a, b, pairs)
|
||||
self._join_checks.append(self._join_sibling(a, b, parent, condition) if parent
|
||||
else self._join_unknown(a, b, candidates, condition))
|
||||
elif best.status not in _STRONG:
|
||||
self._join_checks.append(self._join_proposed(a, b, best, condition))
|
||||
|
||||
def _shared_parent(self, a: _Table, b: _Table, pairs: set[tuple[str, str]]) -> str | None:
|
||||
"""兄弟关联:每一对连接列都是两边各自指向同一张父表的同一组列(a.member_id → members.id ← b.member_id)。
|
||||
返回父表名;不是这种连法返回 None。
|
||||
|
||||
目录里不会记 a 和 b 之间的关系,但这不是「连错了列」:比如评价表和标签关联表各自按门店编号指向门店、
|
||||
彼此按门店编号相连。照「对不上任何关系」报警告是误报。
|
||||
"""
|
||||
def parents(table: _Table, column: str) -> set[tuple[str, tuple[str, ...]]]:
|
||||
return {(e.to_table.lower(), tuple(c.lower() for c in e.to_columns))
|
||||
for e in self.checker.edges_from(table.name)
|
||||
if tuple(c.lower() for c in e.from_columns) == (column,)
|
||||
and e.cardinality in (None, "many_to_one", "one_to_one")}
|
||||
|
||||
shared: set[tuple[str, tuple[str, ...]]] | None = None
|
||||
for left, right in pairs:
|
||||
common = parents(a, left) & parents(b, right)
|
||||
shared = common if shared is None else shared & common
|
||||
if not shared:
|
||||
return None
|
||||
names = {table for table, _ in shared or ()}
|
||||
return next(iter(names)) if len(names) == 1 else None
|
||||
|
||||
def _join_sibling(self, a: _Table, b: _Table, parent: str, condition: str) -> SqlCheck:
|
||||
item = (self.checker.notes_of(parent) or {}).get("label")
|
||||
label = str(item["value"]) if isinstance(item, dict) and isinstance(item.get("value"), str) \
|
||||
and item["value"].strip() else parent
|
||||
return SqlCheck(
|
||||
code="join_unconfirmed", level="info", table=b.name, sql_excerpt=condition or None,
|
||||
message=(f"「{a.label()}」与「{b.label()}」之间没有直接的关系,这里经共同的「{label}」相连。"
|
||||
"关联后行数可能成倍增加:确认其中一侧每个键只有一行,或先各自汇总再关联"),
|
||||
for_model=(f"{a.name} 和 {b.name} 按 {condition or '这组条件'} 关联:两边都指向 {parent},彼此是一对多对多。"
|
||||
"如果其中一侧每个键不止一行,关联后另一侧的行会被重复,求和、计数会放大;"
|
||||
"先把一侧筛到每个键一行(例如只取主记录),或分别汇总到这个键再关联"))
|
||||
|
||||
def _join_unknown(self, a: _Table, b: _Table, candidates: list[catalog.JoinEdge], condition: str) -> SqlCheck:
|
||||
known = ";".join(dict.fromkeys(e.condition() for e in candidates))
|
||||
hint = f"目录里这两张表的关系是:{known}。照它改关联条件" if known else \
|
||||
|
||||
+5
-3
@@ -9,8 +9,9 @@
|
||||
从 visits 到 channels 要经过它,正好是两跳的连接路径。group_visits 同理(旅行团与入园记录)。
|
||||
- **命名推断必须放过的列**(NOT_INFERRED):自引用(employees.employee_id 是工号,不是指向自己)、
|
||||
多个候选(guide 和 guides 两张表都在,tour_assignments.guide_id 不知道指哪张)、没有对应的表
|
||||
(parking_records.lot_id、operation_logs.operator_id、categories.parentId)、类型对不上
|
||||
(feedback_forms.store_id 是文字,stores.id 是整数)。
|
||||
(operation_logs.operator_id、categories.parentId)、类型对不上(feedback_forms.store_id 是文字,stores.id 是整数)。
|
||||
parking_records.lot_id 没有直接对上的 lot / lots,但和引用它的表共用 parking 前缀的 parking_lots 只有一张,
|
||||
推得出来(在 NAME_RELATIONS 里)。
|
||||
- **唯一约束 / 唯一索引。** orders.order_no、channels.channel_code 是列上的 UNIQUE;members.member_no、
|
||||
visits.ticket_no 是单独建的唯一索引。
|
||||
|
||||
@@ -188,6 +189,8 @@ NAME_RELATIONS = frozenset({
|
||||
("tour_groups", ("agency_id",), "agencies", ("id",)),
|
||||
("tour_assignments", ("tour_group_id",), "tour_groups", ("id",)),
|
||||
("lost_and_found", ("park_id",), "parks", ("id",)),
|
||||
# 没有 lot / lots,按引用表共用的 parking 前缀对上 parking_lots
|
||||
("parking_records", ("lot_id",), "parking_lots", ("id",)),
|
||||
# 视图没有约束,按列名同样推得出来:按日汇总的入园人数照样能按 park_id 连到景区
|
||||
("v_daily_visits", ("park_id",), "parks", ("id",)),
|
||||
})
|
||||
@@ -196,7 +199,6 @@ NAME_RELATIONS = frozenset({
|
||||
NOT_INFERRED = {
|
||||
("employees", "employee_id"): "自引用:工号,不是指向本表",
|
||||
("tour_assignments", "guide_id"): "多个候选:guide 与 guides",
|
||||
("parking_records", "lot_id"): "没有名为 lot / lots 的表",
|
||||
("operation_logs", "operator_id"): "没有名为 operator / operators 的表",
|
||||
("categories", "parentId"): "没有名为 parent / parents 的表",
|
||||
("feedback_forms", "store_id"): "类型对不上:文字列指向整数主键",
|
||||
|
||||
Vendored
+26
@@ -0,0 +1,26 @@
|
||||
"""测试里建的数据源,用完就删。
|
||||
|
||||
数据源表在每个测试进程的测试库里是共用的:留下来的源会出现在助手的数据源清单里(50 张表的合成库还会
|
||||
触发挑表、多一次模型调用),别的用例的结果就跟着执行顺序变。阶段 1B 就是这样被绊过一次。
|
||||
删数据源时它的数据目录随外键级联删除(测试库开着 PRAGMA foreign_keys=ON)。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import delete, select
|
||||
|
||||
from app.data.engine import engines
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import DataSource
|
||||
|
||||
|
||||
async def drop_source(source_id: str | None = None, *, name: str | None = None) -> None:
|
||||
"""按 id 或名字删一个数据源,并作废它的连接池。不存在就什么都不做。"""
|
||||
async with SessionLocal() as session:
|
||||
if source_id is None:
|
||||
source_id = (await session.execute(
|
||||
select(DataSource.id).where(DataSource.name == name))).scalar_one_or_none()
|
||||
if source_id is None:
|
||||
return
|
||||
await session.execute(delete(DataSource).where(DataSource.id == source_id))
|
||||
await session.commit()
|
||||
await engines.invalidate(source_id)
|
||||
@@ -11,12 +11,12 @@ import pytest
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
|
||||
from app.data import catalog
|
||||
from app.data.engine import engines
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import DataSource
|
||||
from app.main import app as _fastapi_app
|
||||
from tests.fixtures.catalog import scenic
|
||||
from tests.fixtures.catalog.fake_model import FakeModel
|
||||
from tests.fixtures.sources import drop_source
|
||||
|
||||
ACTOR = {"X-Actor": quote("王敏")}
|
||||
|
||||
@@ -38,7 +38,7 @@ async def source_id(client, scenic_db):
|
||||
r = await client.post(f"/api/datasources/{sid}/introspect")
|
||||
assert r.status_code == 200, r.text
|
||||
yield sid
|
||||
await engines.invalidate(sid)
|
||||
await drop_source(sid)
|
||||
|
||||
|
||||
def _base(sid: str) -> str:
|
||||
@@ -184,9 +184,12 @@ async def test_draft_without_schema_is_refused(client):
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
sid = row.id
|
||||
r = await client.post(f"{_base(sid)}/draft", json={})
|
||||
assert r.status_code == 409 and "探查结构" in r.json()["detail"]
|
||||
assert (await client.get(_base(sid))).json()["tables"] == []
|
||||
try:
|
||||
r = await client.post(f"{_base(sid)}/draft", json={})
|
||||
assert r.status_code == 409 and "探查结构" in r.json()["detail"]
|
||||
assert (await client.get(_base(sid))).json()["tables"] == []
|
||||
finally:
|
||||
await drop_source(sid)
|
||||
|
||||
|
||||
async def test_schema_endpoint_detail_carries_catalog(client, source_id):
|
||||
|
||||
@@ -15,6 +15,7 @@ from app.data.catalog import make_item
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import DataSource
|
||||
from tests.fixtures.catalog.fake_model import FakeModel
|
||||
from tests.fixtures.sources import drop_source
|
||||
|
||||
|
||||
def _col(name: str, type_: str = "INTEGER", comment: str | None = None, nullable: bool = True) -> dict:
|
||||
@@ -156,6 +157,31 @@ def test_name_inference_negative(tables, why):
|
||||
assert _infer(tables, next(iter(tables))) == set(), why
|
||||
|
||||
|
||||
def test_name_inference_falls_back_to_a_table_sharing_the_prefix():
|
||||
"""库里没有 tags,只有带前缀的 store_tags 和 menu_tag:和引用它的表共用 store 前缀的那张才是它要指的。
|
||||
以前这种命名推不出关系,SQL 检查跟着误报「对不上任何关系」。"""
|
||||
tables = {
|
||||
"store_tag_links": _table(_col("id"), _col("tagId"), _col("storeID")),
|
||||
"store_tags": _table(_col("id")),
|
||||
"menu_tag": _table(_col("id")),
|
||||
"store": _table(_col("id")),
|
||||
}
|
||||
assert _infer(tables, "store_tag_links") == {
|
||||
(("tagId",), "store_tags", ("id",)), (("storeID",), "store", ("id",))}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tables, why", [
|
||||
# 直接对得上就用直接的,不去找带前缀的
|
||||
({"shop_category_links": _table(_col("id"), _col("category_id")), "categories": _table(_col("id")),
|
||||
"shop_categories": _table(_col("id"))}, {(("category_id",), "categories", ("id",))}),
|
||||
# 引用它的表没有可共用的前缀:带前缀的表不止一张时照旧不推
|
||||
({"links": _table(_col("id"), _col("tag_id")), "store_tags": _table(_col("id")),
|
||||
"menu_tags": _table(_col("id"))}, set()),
|
||||
])
|
||||
def test_prefix_fallback_only_when_direct_match_is_missing(tables, why):
|
||||
assert _infer(tables, next(iter(tables))) == why
|
||||
|
||||
|
||||
def test_name_inference_one_to_one_when_column_is_own_key():
|
||||
tables = {"member_profiles": _table(_col("member_id"), pk=("member_id",)), "members": _table(_col("id"))}
|
||||
rel = catalog.infer_name_relations(_source(tables).schema_cache, "member_profiles")[0]
|
||||
@@ -255,6 +281,16 @@ async def test_model_draft_respects_system_notes_of_imported_tables():
|
||||
# ---------------------------------------------------------------- 起草并写入
|
||||
|
||||
|
||||
_CREATED: list[str] = []
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
async def _drop_created_sources():
|
||||
yield
|
||||
while _CREATED:
|
||||
await drop_source(_CREATED.pop())
|
||||
|
||||
|
||||
async def _db_source(tables: dict, **kw) -> DataSource:
|
||||
for name, meta in tables.items():
|
||||
meta["qualified"] = name
|
||||
@@ -263,6 +299,7 @@ async def _db_source(tables: dict, **kw) -> DataSource:
|
||||
schema_cache={"tables": tables}, **kw)
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
_CREATED.append(row.id)
|
||||
return row
|
||||
|
||||
|
||||
|
||||
@@ -19,6 +19,7 @@ from app.data.engine import engines
|
||||
from app.data.introspect import introspect
|
||||
# 模型要在建表之前注册进 metadata:conftest 的建表只导入了 app.db.base,这个文件单独跑时没有别的模块先导入它们
|
||||
from app.db import models # noqa: F401
|
||||
from tests.fixtures.sources import drop_source
|
||||
|
||||
_CACHE: dict[str, dict] = {}
|
||||
|
||||
@@ -106,10 +107,9 @@ async def world(scenic_db):
|
||||
for wid in created:
|
||||
if (wf := await session.get(Workflow, wid)) is not None:
|
||||
await session.delete(wf)
|
||||
for sid in (ids.source, ids.other):
|
||||
if (row := await session.get(DataSource, sid)) is not None:
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
for sid in (ids.source, ids.other):
|
||||
await drop_source(sid)
|
||||
|
||||
|
||||
async def _impact(source_id: str, table: str) -> dict:
|
||||
|
||||
@@ -25,6 +25,7 @@ from app.data import catalog
|
||||
from app.data.engine import engines
|
||||
from app.data.introspect import introspect
|
||||
from tests.fixtures.catalog import scenic_notes
|
||||
from tests.fixtures.sources import drop_source
|
||||
|
||||
_CACHE: dict[str, dict] = {}
|
||||
|
||||
@@ -76,6 +77,32 @@ async def test_codes_are_supplemented_not_replaced(scenic_db):
|
||||
assert change.value == {"9": "作废"}
|
||||
|
||||
|
||||
async def test_codes_with_pending_meanings(scenic_db):
|
||||
"""数据剖析写进来的码值候选含义为空(界面上写「含义待填写」):提案可以补上含义;提案里含义为空的码只在原来
|
||||
没有时加进去,不抹掉已有的含义;含义为空的码合法,保存不报错。"""
|
||||
tables = (await _schema(scenic_db))["tables"]
|
||||
profiled = _visits()
|
||||
profiled["columns"]["status"]["codes"] = catalog.make_item({"1": "有效", "0": "", "9": ""}, "profile", "proposed")
|
||||
plan = catalog.plan_patch(profiled, [{"path": "columns.status.codes", "value": {"9": "作废", "1": ""}}],
|
||||
table="visits", tables=tables)
|
||||
assert plan.problems == []
|
||||
[change] = plan.changes
|
||||
assert change.after == {"1": "有效", "0": "", "9": "作废"} and change.state == "change"
|
||||
# 只给了含义为空的新码:照样加进去,含义待人填
|
||||
plan = catalog.plan_patch(profiled, [{"path": "columns.status.codes", "value": {"7": ""}}], table="visits",
|
||||
tables=tables)
|
||||
assert plan.changes[0].after == {"1": "有效", "0": "", "9": "", "7": ""}
|
||||
# 只给了含义为空的已有码:什么都没变,这一项是「保存即确认」(原来是推断)
|
||||
plan = catalog.plan_patch(profiled, [{"path": "columns.status.codes", "value": {"1": ""}}], table="visits",
|
||||
tables=tables)
|
||||
assert plan.changes[0].state == "confirm"
|
||||
out = catalog.apply_patch(profiled, [{"path": "columns.status.codes", "value": {"9": "作废"}}], table="visits",
|
||||
tables=tables)
|
||||
codes = out["columns"]["status"]["codes"]
|
||||
assert codes["value"] == {"1": "有效", "0": "", "9": "作废"} and (codes["source"], codes["status"]) == (
|
||||
"human", "confirmed")
|
||||
|
||||
|
||||
async def test_new_relation_gets_its_id_from_both_ends(scenic_db):
|
||||
tables = (await _schema(scenic_db))["tables"]
|
||||
value = {"columns": ["visit_id"], "to_table": "visits", "to_columns": ["id"], "cardinality": "many_to_one"}
|
||||
@@ -190,11 +217,7 @@ async def scenic_source(scenic_db):
|
||||
await catalog.write_entry(session, row.id, table, notes, if_version=0, actor="王敏")
|
||||
source_id = row.id
|
||||
yield SimpleNamespace(id=source_id, name=name)
|
||||
async with SessionLocal() as session:
|
||||
row = await session.get(DataSource, source_id)
|
||||
if row is not None:
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
await drop_source(source_id)
|
||||
|
||||
|
||||
async def _stream(monkeypatch, model, source, base_graph=None):
|
||||
|
||||
@@ -0,0 +1,430 @@
|
||||
"""数据剖析接口:POST /api/datasources/{id}/catalog/profile(阶段 2A)。
|
||||
|
||||
剖析是数据目录里唯一对业务库发查询的地方,这里逐条钉住方案定的约束:默认关闭、只读、每条查询都过守卫、
|
||||
查询次数和单条时限、总时长、遮罩的列不取样、大表不做整表统计、同一个数据源同时只有一次剖析;
|
||||
以及剖析的结论怎么进目录:命名推断对上数据的升为有确证(来源记为数据剖析,和外键分开)、对不上的写明
|
||||
覆盖率、人工确认过的只补覆盖率和基数、状态列得到码值候选、只有一个日期列的表提议业务日期。
|
||||
|
||||
所有测试都在合成的景区库上做(tests/fixtures/catalog/scenic.py),数据和名称全部虚构。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import sqlite3
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from urllib.parse import quote
|
||||
|
||||
import pytest
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
from sqlalchemy import event
|
||||
from sqlalchemy.engine import Engine
|
||||
|
||||
from app.data import catalog, catalog_profile
|
||||
from app.data import engine as data_engine
|
||||
from app.data.catalog_profile import PROFILE_OPTION, ProfileSettings
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import DataSource
|
||||
from app.main import app as _fastapi_app
|
||||
from tests.fixtures.catalog import scenic
|
||||
from tests.fixtures.sources import drop_source
|
||||
|
||||
ACTOR = {"X-Actor": quote("周宁")}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def client():
|
||||
async with AsyncClient(transport=ASGITransport(app=_fastapi_app), base_url="http://test") as c:
|
||||
yield c
|
||||
|
||||
|
||||
def _on(**settings) -> dict:
|
||||
return {PROFILE_OPTION: {"enabled": True, **settings}}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def make_source(client):
|
||||
"""登记一个接上 path 的数据源并走真实的探查接口;draft 给了就先起草这些表的目录。"""
|
||||
made: list[str] = []
|
||||
|
||||
async def _make(path: str, *, options: dict | None = None, draft: list[str] | None = None) -> str:
|
||||
async with SessionLocal() as session:
|
||||
row = DataSource(name=f"prof_{uuid.uuid4().hex[:10]}", kind="sqlite", database=str(path),
|
||||
options=options or {})
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
sid = row.id
|
||||
made.append(sid)
|
||||
r = await client.post(f"/api/datasources/{sid}/introspect")
|
||||
assert r.status_code == 200, r.text
|
||||
if draft is not None:
|
||||
r = await client.post(f"/api/datasources/{sid}/catalog/draft", json={"tables": draft})
|
||||
assert r.status_code == 200, r.text
|
||||
return sid
|
||||
|
||||
yield _make
|
||||
for sid in made:
|
||||
await drop_source(sid)
|
||||
|
||||
|
||||
class _Spy:
|
||||
"""记下两样东西:守卫放行(guard.check 返回)的语句,和真正发到这个业务库的语句。
|
||||
|
||||
后者挂在 SQLAlchemy 的 before_cursor_execute 上,任何一条到达驱动的语句都逃不掉——包括绕开查询层
|
||||
另开连接发的。断言「发到库的 ⊆ 守卫放行的」就是「所有查询都经过守卫」。
|
||||
"""
|
||||
|
||||
def __init__(self, db_name: str) -> None:
|
||||
self.db_name = db_name
|
||||
self.checked: list[str] = []
|
||||
self.executed: list[str] = []
|
||||
|
||||
def clear(self) -> None:
|
||||
self.checked.clear()
|
||||
self.executed.clear()
|
||||
|
||||
def all_guarded(self) -> bool:
|
||||
return bool(self.executed) and all(s in self.checked for s in self.executed)
|
||||
|
||||
def mentions(self, word: str) -> list[str]:
|
||||
return [s for s in self.executed if word.lower() in s.lower()]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def spy(monkeypatch, scenic_db):
|
||||
holder = SimpleNamespace(spy=None)
|
||||
|
||||
def watch(path: str) -> _Spy:
|
||||
holder.spy = _Spy(Path(path).name)
|
||||
return holder.spy
|
||||
|
||||
real_check = data_engine.check
|
||||
|
||||
def checking(sql, **kw):
|
||||
out = real_check(sql, **kw)
|
||||
if holder.spy is not None:
|
||||
holder.spy.checked.append(out)
|
||||
return out
|
||||
|
||||
def on_execute(conn, cursor, statement, parameters, context, executemany):
|
||||
if holder.spy is not None and holder.spy.db_name in str(conn.engine.url):
|
||||
holder.spy.executed.append(statement)
|
||||
|
||||
monkeypatch.setattr(data_engine, "check", checking)
|
||||
event.listen(Engine, "before_cursor_execute", on_execute)
|
||||
yield watch
|
||||
event.remove(Engine, "before_cursor_execute", on_execute)
|
||||
|
||||
|
||||
def _url(sid: str) -> str:
|
||||
return f"/api/datasources/{sid}/catalog/profile"
|
||||
|
||||
|
||||
async def _notes(client, sid: str, table: str) -> dict:
|
||||
r = await client.get(f"/api/datasources/{sid}/catalog/{table}")
|
||||
assert r.status_code == 200, r.text
|
||||
return r.json()
|
||||
|
||||
|
||||
def _relation(notes: dict, column: str) -> dict:
|
||||
return next(r for r in notes.get("relations") or [] if r["columns"] == [column])
|
||||
|
||||
|
||||
def _table(body: dict, name: str) -> dict:
|
||||
return next(t for t in body["tables"] if t["table_name"] == name)
|
||||
|
||||
|
||||
def _reasons(table: dict) -> set[str]:
|
||||
return {s["reason"] for s in table["skipped"]}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 开关与设置
|
||||
|
||||
|
||||
async def test_profile_is_off_by_default_and_says_where_to_turn_it_on(client, make_source, scenic_db, spy):
|
||||
sid = await make_source(scenic_db, draft=["visits"])
|
||||
watch = spy(scenic_db)
|
||||
watch.clear()
|
||||
r = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR)
|
||||
assert r.status_code == 409
|
||||
detail = r.json()["detail"]
|
||||
assert "数据源设置" in detail and "开启数据剖析" in detail
|
||||
assert watch.executed == [] # 一条查询都没发
|
||||
|
||||
|
||||
async def test_settings_are_saved_and_validated_through_datasource_update(client, make_source, scenic_db):
|
||||
sid = await make_source(scenic_db)
|
||||
r = await client.patch(f"/api/datasources/{sid}", json={"options": _on(max_queries=0)})
|
||||
assert r.status_code == 422 and "查询次数上限" in r.json()["detail"]
|
||||
r = await client.patch(f"/api/datasources/{sid}", json={"options": _on(max_queries=40, sample_size=500)})
|
||||
assert r.status_code == 200, r.text
|
||||
assert r.json()["options"][PROFILE_OPTION] == {"enabled": True, "max_queries": 40, "sample_size": 500}
|
||||
# 设好以后就能剖析;返回里带着这次实际用的设置
|
||||
r = await client.post(_url(sid), json={"tables": ["channels"]}, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
assert r.json()["settings"]["max_queries"] == 40 and r.json()["settings"]["sample_size"] == 500
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 剖析的结论
|
||||
|
||||
|
||||
async def test_profile_verifies_name_relations_and_proposes_codes_and_business_date(client, make_source, scenic_db,
|
||||
spy):
|
||||
sid = await make_source(scenic_db, options=_on(), draft=["visits", "channels"])
|
||||
watch = spy(scenic_db)
|
||||
watch.clear()
|
||||
r = await client.post(_url(sid), json={"tables": ["visits", "channels"]}, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
body = r.json()
|
||||
|
||||
# 所有发到业务库的语句都经过了守卫
|
||||
assert watch.all_guarded(), watch.executed
|
||||
assert body["queries_used"] == len(watch.executed) <= body["settings"]["max_queries"]
|
||||
assert body["queries_used"] == sum(t["queries"] for t in body["tables"])
|
||||
assert body["stopped"] is None and body["actor"] == "周宁" and body["profiled_at"]
|
||||
|
||||
notes = (await _notes(client, sid, "visits"))
|
||||
assert notes["updated_by"] == "周宁"
|
||||
notes = notes["notes"]
|
||||
# 命名推断出来的两条关系:数据对得上、父表主键唯一 → 有确证,来源记为数据剖析
|
||||
for column, parent in (("gate_id", "gates"), ("member_id", "members")):
|
||||
rel = _relation(notes, column)
|
||||
assert rel["to_table"] == parent
|
||||
assert (rel["source"], rel["status"]) == ("profile", "verified")
|
||||
assert rel["coverage"] == 1.0 and rel["cardinality"] == "many_to_one"
|
||||
assert "数据剖析" in rel["note"] and "抽样" in rel["note"]
|
||||
# 外键约束来的关系不动:外键是外键,剖析是剖析
|
||||
park = _relation(notes, "park_id")
|
||||
assert (park["source"], park["status"], park["coverage"]) == ("fk", "verified", None)
|
||||
|
||||
# 状态列:码值候选,值有了、含义留空,推断
|
||||
codes = notes["columns"]["status"]["codes"]
|
||||
assert codes["value"] == {"1": "", "0": ""}
|
||||
assert (codes["source"], codes["status"]) == ("profile", "proposed") and "1500" in codes["note"]
|
||||
# 只有一个日期类列:提议为业务日期,备注里写明取值范围
|
||||
bdate = notes["business_date"]
|
||||
assert bdate["value"] == {"column": "visit_time"} and (bdate["source"], bdate["status"]) == ("profile", "proposed")
|
||||
assert "2026-07" in bdate["note"] and "2026-09" in bdate["note"]
|
||||
|
||||
channels = (await _notes(client, sid, "channels"))["notes"]
|
||||
assert set(channels["columns"]["channel_type"]["codes"]["value"]) == {"线上直销", "线下", "分销"}
|
||||
assert "channel_code" not in channels.get("columns", {}) # 唯一约束列不是码值
|
||||
|
||||
visits = _table(body, "visits")
|
||||
kinds = sorted(f["kind"] for f in visits["findings"])
|
||||
assert kinds == ["business_date", "codes", "relation", "relation"]
|
||||
gate = next(f for f in visits["findings"] if f["kind"] == "relation" and f["columns"] == ["gate_id"])
|
||||
assert gate["status"] == "verified" and gate["path"] == f"relations.{_relation(notes, 'gate_id')['id']}"
|
||||
assert gate["sample"] == 8 and gate["matched"] == 8 and gate["summary"]
|
||||
assert visits["row_estimate"] == {"rows": 1500, "method": "count", "at_least": None}
|
||||
assert visits["added"] == 2 and visits["updated"] == 2 and visits["error"] is None
|
||||
|
||||
|
||||
async def test_wrong_name_relation_stays_proposed_with_coverage_in_note(client, make_source, tmp_path):
|
||||
"""推错的命名关系:parking_records.lot_id 按名字对上了新建的 lots 表,但停车记录里的区号大多不在里面。"""
|
||||
path = scenic.build(tmp_path / "lots.db")
|
||||
db = sqlite3.connect(path)
|
||||
try:
|
||||
db.execute("CREATE TABLE lots (id INTEGER PRIMARY KEY, name TEXT NOT NULL)")
|
||||
db.executemany("INSERT INTO lots VALUES (?, ?)", [(i, f"{i} 号停车区") for i in (1, 2, 3)])
|
||||
db.executemany("INSERT INTO parking_records VALUES (?, ?, ?, ?, ?, ?)",
|
||||
[(i, 1 + i % 20, None, "2026-08-01 09:00:00", None, 10.0) for i in range(1, 101)])
|
||||
db.commit()
|
||||
finally:
|
||||
db.close()
|
||||
sid = await make_source(path, options=_on(), draft=["parking_records"])
|
||||
before = _relation((await _notes(client, sid, "parking_records"))["notes"], "lot_id")
|
||||
assert (before["to_table"], before["source"], before["status"]) == ("lots", "name", "proposed")
|
||||
|
||||
r = await client.post(_url(sid), json={"tables": ["parking_records"]}, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
rel = _relation((await _notes(client, sid, "parking_records"))["notes"], "lot_id")
|
||||
assert rel["status"] == "proposed" and rel["source"] == "profile"
|
||||
assert rel["coverage"] == 0.15
|
||||
assert "抽样覆盖率 15%" in rel["note"] and "可能不是这条关系" in rel["note"] and "20 个" in rel["note"]
|
||||
finding = _table(r.json(), "parking_records")["findings"][0]
|
||||
assert finding["status"] == "proposed" and finding["coverage"] == 0.15
|
||||
# 两个日期类列:不提议业务日期,取值范围只在返回里
|
||||
ranges = {d["column"] for d in _table(r.json(), "parking_records")["date_ranges"]}
|
||||
assert ranges == {"entered_at"} # exited_at 全是空值
|
||||
assert "business_date" not in (await _notes(client, sid, "parking_records"))["notes"]
|
||||
|
||||
|
||||
async def test_confirmed_relation_only_gets_coverage_and_cardinality(client, make_source, scenic_db):
|
||||
sid = await make_source(scenic_db, options=_on(), draft=["visits"])
|
||||
detail = await _notes(client, sid, "visits")
|
||||
gate = _relation(detail["notes"], "gate_id")
|
||||
r = await client.post(f"/api/datasources/{sid}/catalog/visits/review",
|
||||
json={"path": f"relations.{gate['id']}", "action": "confirm",
|
||||
"if_version": detail["version"]})
|
||||
assert r.status_code == 200, r.text
|
||||
|
||||
r = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
rel = _relation((await _notes(client, sid, "visits"))["notes"], "gate_id")
|
||||
assert (rel["source"], rel["status"]) == ("name", "confirmed")
|
||||
assert rel["coverage"] == 1.0 and rel["cardinality"] == "many_to_one"
|
||||
finding = next(f for f in _table(r.json(), "visits")["findings"] if f.get("columns") == ["gate_id"])
|
||||
assert finding["status"] == "confirmed" and finding["confirmed"] is True
|
||||
|
||||
|
||||
async def test_rerun_with_same_data_changes_nothing(client, make_source, scenic_db):
|
||||
"""同一天、数据没变:第二次剖析不改目录、不升版本(冻结进快照的目录不会无故变)。"""
|
||||
sid = await make_source(scenic_db, options=_on(), draft=["visits"])
|
||||
first = (await client.post(_url(sid), json={"tables": ["visits"]})).json()
|
||||
second = (await client.post(_url(sid), json={"tables": ["visits"]})).json()
|
||||
a, b = _table(first, "visits"), _table(second, "visits")
|
||||
assert (b["added"], b["updated"], b["removed"]) == (0, 0, 0)
|
||||
assert b["version"] == a["version"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 安全约束
|
||||
|
||||
|
||||
async def test_masked_columns_are_never_queried(client, make_source, scenic_db, spy):
|
||||
sid = await make_source(scenic_db, options={**_on(), "mask_columns": "member_id, Channel_Type"},
|
||||
draft=["visits", "channels"])
|
||||
watch = spy(scenic_db)
|
||||
watch.clear()
|
||||
r = await client.post(_url(sid), json={"tables": ["visits", "channels"]}, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
assert watch.all_guarded()
|
||||
assert watch.mentions("member_id") == [] and watch.mentions("channel_type") == []
|
||||
visits, channels = _table(r.json(), "visits"), _table(r.json(), "channels")
|
||||
masked = [s for s in visits["skipped"] + channels["skipped"] if s["reason"] == "masked"]
|
||||
assert {s["target"] for s in masked} == {"member_id → members.id", "channel_type"}
|
||||
assert all("遮罩" in s["detail"] for s in masked)
|
||||
notes = (await _notes(client, sid, "visits"))["notes"]
|
||||
assert _relation(notes, "member_id")["source"] == "name" # 没动
|
||||
assert _relation(notes, "gate_id")["status"] == "verified" # 其余照常
|
||||
assert "channel_type" not in ((await _notes(client, sid, "channels"))["notes"].get("columns") or {})
|
||||
|
||||
|
||||
async def test_query_budget_stops_profiling_and_reports_the_rest(client, make_source, scenic_db, spy):
|
||||
sid = await make_source(scenic_db, options=_on(max_queries=3), draft=["visits"])
|
||||
watch = spy(scenic_db)
|
||||
watch.clear()
|
||||
r = await client.post(_url(sid), json={"tables": ["visits", "channels"]}, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
body = r.json()
|
||||
assert body["queries_used"] == 3 == len(watch.executed) and body["stopped"] == "budget"
|
||||
visits, channels = _table(body, "visits"), _table(body, "channels")
|
||||
assert "budget" in _reasons(visits)
|
||||
assert channels["queries"] == 0 and _reasons(channels) == {"budget"}
|
||||
assert all("查询次数" in s["detail"] for s in visits["skipped"] + channels["skipped"] if s["reason"] == "budget")
|
||||
# 没查完的关系不改:原来的命名推断原样留着
|
||||
rel = _relation((await _notes(client, sid, "visits"))["notes"], "member_id")
|
||||
assert (rel["source"], rel["status"]) == ("name", "proposed")
|
||||
|
||||
|
||||
async def test_single_query_timeout_is_recorded_and_profiling_continues(client, make_source, scenic_db, monkeypatch):
|
||||
"""闸口的抽样查询换成一条停不下来的递归查询:1 秒时限到点由数据库掐断,记为超时,其余照常。"""
|
||||
sid = await make_source(scenic_db, options=_on(query_timeout_s=1), draft=["visits"])
|
||||
real = data_engine.run_query
|
||||
endless = "WITH RECURSIVE r(x) AS (SELECT 1 UNION ALL SELECT x + 1 FROM r) SELECT COUNT(*) FROM r"
|
||||
|
||||
async def slow_gate(source, sql, *, limits=None):
|
||||
if "gate_id" in sql and "DISTINCT" in sql and "COUNT" not in sql:
|
||||
assert limits is not None and limits.timeout_seconds == 1
|
||||
return await real(source, endless, limits=limits)
|
||||
return await real(source, sql, limits=limits)
|
||||
|
||||
monkeypatch.setattr(data_engine, "run_query", slow_gate)
|
||||
r = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
visits = _table(r.json(), "visits")
|
||||
timeout = [s for s in visits["skipped"] if s["reason"] == "timeout"]
|
||||
assert [s["target"] for s in timeout] == ["gate_id → gates.id"] and "超时" in timeout[0]["detail"]
|
||||
assert r.json()["stopped"] is None
|
||||
notes = (await _notes(client, sid, "visits"))["notes"]
|
||||
assert _relation(notes, "gate_id")["source"] == "name" # 超时的不改
|
||||
assert _relation(notes, "member_id")["status"] == "verified" # 其余照常
|
||||
|
||||
|
||||
async def test_total_deadline_stops_profiling(make_source, scenic_db, monkeypatch):
|
||||
sid = await make_source(scenic_db, draft=["visits"])
|
||||
real = data_engine.run_query
|
||||
|
||||
async def slow(source, sql, *, limits=None):
|
||||
await asyncio.sleep(0.5)
|
||||
return await real(source, sql, limits=limits)
|
||||
|
||||
monkeypatch.setattr(data_engine, "run_query", slow)
|
||||
async with SessionLocal() as session:
|
||||
row = await session.get(DataSource, sid)
|
||||
settings = ProfileSettings(enabled=True, max_total_s=1.4)
|
||||
report = await catalog_profile.profile_catalog(session, row, row, tables=["visits"], actor=None,
|
||||
settings=settings)
|
||||
assert report.stopped == "deadline"
|
||||
assert 1 <= report.queries_used <= 3
|
||||
reasons = {s.reason for t in report.tables for s in t.skipped}
|
||||
assert "deadline" in reasons
|
||||
|
||||
|
||||
async def test_large_tables_get_no_full_scans(client, make_source, scenic_db, spy):
|
||||
"""行数超过整表统计上限(这里设成 100 行):不做 COUNT(DISTINCT)、MIN/MAX,抽样和取值分布都带上限。"""
|
||||
sid = await make_source(scenic_db, options=_on(max_scan_rows=100), draft=["visits"])
|
||||
watch = spy(scenic_db)
|
||||
watch.clear()
|
||||
r = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
visits = _table(r.json(), "visits")
|
||||
assert visits["row_estimate"] == {"rows": None, "method": "count", "at_least": 101}
|
||||
on_visits = [s for s in watch.executed if "from visits" in s.lower()]
|
||||
assert on_visits and all("LIMIT" in s for s in on_visits), on_visits
|
||||
assert not any("MIN(" in s for s in on_visits)
|
||||
assert "too_large" in _reasons(visits)
|
||||
assert any("太大" in s["detail"] or "上限" in s["detail"] for s in visits["skipped"] if s["reason"] == "too_large")
|
||||
# 父表有主键,覆盖率又够:照样有确证;子表一侧没核对唯一,按多对一记
|
||||
rel = _relation((await _notes(client, sid, "visits"))["notes"], "gate_id")
|
||||
assert rel["status"] == "verified" and rel["cardinality"] == "many_to_one"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 接口
|
||||
|
||||
|
||||
async def test_concurrent_profile_on_same_source_returns_409(client, make_source, scenic_db, monkeypatch):
|
||||
sid = await make_source(scenic_db, options=_on(), draft=["visits"])
|
||||
real = data_engine.run_query
|
||||
started, release = asyncio.Event(), asyncio.Event()
|
||||
|
||||
async def held(source, sql, *, limits=None):
|
||||
started.set()
|
||||
await release.wait()
|
||||
return await real(source, sql, limits=limits)
|
||||
|
||||
monkeypatch.setattr(data_engine, "run_query", held)
|
||||
first = asyncio.create_task(client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR))
|
||||
await asyncio.wait_for(started.wait(), 10)
|
||||
second = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR)
|
||||
assert second.status_code == 409 and "正在进行数据剖析" in second.json()["detail"]
|
||||
release.set()
|
||||
done = await asyncio.wait_for(first, 30)
|
||||
assert done.status_code == 200, done.text
|
||||
# 做完以后可以再来
|
||||
again = await client.post(_url(sid), json={"tables": ["channels"]}, headers=ACTOR)
|
||||
assert again.status_code == 200, again.text
|
||||
|
||||
|
||||
async def test_default_tables_are_used_tables_with_relation_candidates(client, make_source, scenic_db):
|
||||
sid = await make_source(scenic_db, options=_on(), draft=[])
|
||||
r = await client.post(_url(sid), json={}, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
names = [t["table_name"] for t in r.json()["tables"]]
|
||||
assert names and len(names) <= catalog_profile.PROFILE_DEFAULT_TABLES
|
||||
async with SessionLocal() as session:
|
||||
entries = await catalog.read_catalog(session, sid)
|
||||
for name in names:
|
||||
rels = entries[name].notes.get("relations") or []
|
||||
assert any(rel["source"] not in ("fk",) for rel in rels), name
|
||||
assert "parks" not in names and "channels" not in names # 没有待核实关系的表不选
|
||||
|
||||
|
||||
async def test_unknown_table_and_missing_source(client, make_source, scenic_db):
|
||||
sid = await make_source(scenic_db, options=_on(), draft=["visits"])
|
||||
r = await client.post(_url(sid), json={"tables": ["nope"]}, headers=ACTOR)
|
||||
assert r.status_code == 200
|
||||
assert _table(r.json(), "nope")["error"] and r.json()["queries_used"] == 0
|
||||
r = await client.post(_url("0" * 32), json={}, headers=ACTOR)
|
||||
assert r.status_code == 404
|
||||
@@ -0,0 +1,205 @@
|
||||
"""数据剖析(阶段 2A)里不连库的部分:设置的解析与校验、SQL 字面量、各方言的剖析 SQL 都过得了守卫、
|
||||
目录对剖析结果的容纳(待填含义的码值、撤销审阅回到剖析的结论)。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from app.data import catalog, catalog_profile, guard
|
||||
from app.data.catalog import make_item
|
||||
from app.data.catalog_profile import PROFILE_OPTION, ProfileSettings, profile_settings, profile_settings_problem
|
||||
from app.data.engine import build_url
|
||||
|
||||
AT = "2026-10-03T08:00:00+00:00"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 设置
|
||||
|
||||
|
||||
def test_settings_default_off_and_parse_leniently():
|
||||
s = profile_settings({})
|
||||
assert s == ProfileSettings()
|
||||
assert s.enabled is False and s.max_queries == 60 and s.query_timeout_s == 10 and s.sample_size == 2000
|
||||
assert s.max_scan_rows == 100_000 and s.max_total_s == 120
|
||||
on = profile_settings({PROFILE_OPTION: {"enabled": True, "max_queries": "30", "sample_size": 500}})
|
||||
assert on.enabled is True and on.max_queries == 30 and on.sample_size == 500
|
||||
# 库里存着的值不对(老数据、手改过):那一项用缺省,不因此开启,也不报错
|
||||
odd = profile_settings({PROFILE_OPTION: {"enabled": "yes", "max_queries": 0, "query_timeout_s": "x"}})
|
||||
assert odd.enabled is False and odd.max_queries == 60 and odd.query_timeout_s == 10
|
||||
assert profile_settings({PROFILE_OPTION: "on"}) == ProfileSettings()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value, label", [
|
||||
({"max_queries": 0}, "查询次数上限"),
|
||||
({"max_queries": 501}, "查询次数上限"),
|
||||
({"max_queries": 2.5}, "查询次数上限"),
|
||||
({"query_timeout_s": 61}, "单条查询时限"),
|
||||
({"sample_size": 5}, "抽样键值数"),
|
||||
({"max_scan_rows": -1}, "整表统计行数上限"),
|
||||
({"max_total_s": 5}, "总时长上限"),
|
||||
({"enabled": "true"}, "开启数据剖析"),
|
||||
({"max_queries": True}, "查询次数上限"),
|
||||
])
|
||||
def test_settings_problem_names_the_field_in_chinese(value, label):
|
||||
problem = profile_settings_problem({PROFILE_OPTION: value})
|
||||
assert problem and label in problem and "数据剖析" in problem
|
||||
|
||||
|
||||
def test_settings_problem_rejects_unknown_keys_and_wrong_shape():
|
||||
assert "无法识别" in profile_settings_problem({PROFILE_OPTION: {"enabled": True, "rows": 1}})
|
||||
assert profile_settings_problem({PROFILE_OPTION: ["enabled"]})
|
||||
assert profile_settings_problem({}) is None
|
||||
assert profile_settings_problem({PROFILE_OPTION: None}) is None
|
||||
assert profile_settings_problem({PROFILE_OPTION: {"enabled": True, "max_queries": 500, "query_timeout_s": 1.5,
|
||||
"sample_size": 10, "max_scan_rows": 0,
|
||||
"max_total_s": 600}}) is None
|
||||
|
||||
|
||||
def test_profile_option_never_reaches_the_connection_url():
|
||||
"""剖析设置是 AgentLab 自己的配置:拼进连接串,驱动会把它当未知参数拒掉。"""
|
||||
source = SimpleNamespace(kind="mysql", host="db.example", port=3306, database="shop", username="ro",
|
||||
password=None, options={"charset": "utf8mb4", PROFILE_OPTION: {"enabled": True}})
|
||||
url = build_url(source)
|
||||
assert "charset=utf8mb4" in url and PROFILE_OPTION not in url and "enabled" not in url
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 字面量
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value, kind, expected", [
|
||||
(42, "number", "42"),
|
||||
(-7, None, "-7"),
|
||||
(3.0, "number", "3"),
|
||||
("12345678901234567890", "number", "12345678901234567890"), # DECIMAL 经查询层转成了字符串
|
||||
("A-01", "text", "'A-01'"),
|
||||
("O'Neil", "text", "'O''Neil'"),
|
||||
("12:30", "text", "'12:30'"),
|
||||
])
|
||||
def test_sql_literal_quotes_safe_values(value, kind, expected):
|
||||
assert catalog_profile.sql_literal(value, kind) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value, kind", [
|
||||
(True, "boolean"), # 布尔不当键
|
||||
(2.5, "number"), # 非整数的数不当键:浮点相等比较靠不住
|
||||
("12.50", "number"),
|
||||
("1e3", "number"),
|
||||
("2026-07-01", "date"), # 日期的字面量各家写法不同,不猜
|
||||
("a\\b", "text"), # 反斜杠:MySQL 不同设置下读法不同,守卫会拒
|
||||
("a :b", "text"), # 「:名字」会被查询层当成绑定参数
|
||||
(":b", "text"),
|
||||
("a\nb", "text"), # 控制字符
|
||||
("x" * 201, "text"),
|
||||
(None, "text"),
|
||||
([1], None),
|
||||
])
|
||||
def test_sql_literal_refuses_unsafe_or_unsupported_values(value, kind):
|
||||
assert catalog_profile.sql_literal(value, kind) is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 各方言的剖析 SQL
|
||||
|
||||
|
||||
_META = {"visits": {"schema": None, "columns": [{"name": "memberTagId", "type": "INTEGER"}], "primary_key": ["id"]},
|
||||
"orders": {"schema": "sales", "columns": [{"name": "status", "type": "INTEGER"}], "primary_key": ["id"]}}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind", ["mysql", "mariadb", "postgresql", "oracle", "sqlite"])
|
||||
def test_profile_sql_passes_readonly_guard_on_every_dialect(kind):
|
||||
"""剖析的每一种查询(含读统计信息的系统视图)都得过只读守卫——守卫不为剖析放宽。"""
|
||||
d = catalog_profile.SqlDialect(kind)
|
||||
sqls = []
|
||||
for table in ("visits", "orders"):
|
||||
stats = d.stats_sql(_META[table], table)
|
||||
if kind == "sqlite":
|
||||
assert stats is None # SQLite 没有可读的统计信息,改为数到上限
|
||||
else:
|
||||
assert stats
|
||||
sqls.append(stats)
|
||||
tbl = d.table(_META[table], table)
|
||||
col = d.quote(_META[table]["columns"][0]["name"])
|
||||
sqls += [
|
||||
d.bounded_count_sql(tbl, 100_001),
|
||||
d.distinct_sample_sql(tbl, col, 2000, scan_cap=None),
|
||||
d.distinct_sample_sql(tbl, col, 2000, scan_cap=100_000),
|
||||
d.match_count_sql(tbl, col, ["1", "'A'"]),
|
||||
d.unique_check_sql(tbl, col),
|
||||
d.value_counts_sql(tbl, col, 21, scan_cap=None),
|
||||
d.value_counts_sql(tbl, col, 21, scan_cap=100_000),
|
||||
d.min_max_sql(tbl, [col]),
|
||||
]
|
||||
for sql in sqls:
|
||||
assert guard.check(sql, readonly=True, dialect=kind) == sql, sql
|
||||
|
||||
|
||||
def test_dialect_quoting_and_row_limits():
|
||||
my, ora, pg = (catalog_profile.SqlDialect(k) for k in ("mysql", "oracle", "postgresql"))
|
||||
assert my.quote("memberTagId") == "`memberTagId`" and pg.quote("memberTagId") == '"memberTagId"'
|
||||
assert my.table(_META["orders"], "orders") == "sales.orders"
|
||||
assert pg.table({"schema": "Sales"}, "memberTags") == '"Sales"."memberTags"'
|
||||
# Oracle 没有 LIMIT(11g 也没有 FETCH FIRST):一律用 ROWNUM
|
||||
sample = ora.distinct_sample_sql("visits", '"memberTagId"', 50, scan_cap=None)
|
||||
assert "LIMIT" not in sample and "ROWNUM <= 50" in sample
|
||||
assert "LIMIT 50" in pg.distinct_sample_sql("visits", "x", 50, scan_cap=None)
|
||||
# 统计信息按方言读各自的系统视图
|
||||
assert "information_schema.TABLES" in my.stats_sql(_META["visits"], "visits")
|
||||
assert "pg_class" in pg.stats_sql(_META["visits"], "visits")
|
||||
stats = ora.stats_sql(_META["orders"], "orders")
|
||||
assert "ALL_TABLES" in stats and "'ORDERS'" in stats and "'SALES'" in stats
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 目录对剖析结果的容纳
|
||||
|
||||
|
||||
def test_codes_with_pending_meanings_are_valid_and_render_as_pending():
|
||||
"""剖析得到的码值只有取值、含义待人填写:目录要收得下,给模型看时写明含义待填写。"""
|
||||
notes = {"columns": {"status": {"codes": make_item({"1": "", "0": ""}, "profile", "proposed")},
|
||||
"channel": {"codes": make_item({"WEB": "官网", "BOX": ""}, "profile", "proposed")}}}
|
||||
assert catalog.validate_notes(notes) == []
|
||||
text = catalog.render_table_notes(notes)
|
||||
assert "1、0(含义待填写)" in text
|
||||
assert "WEB=官网,BOX 的含义待填写" in text
|
||||
# 码值仍得是对照表:值不是文字的照样拒收
|
||||
assert catalog.validate_notes({"columns": {"s": {"codes": make_item({"1": 2}, "profile")}}})
|
||||
|
||||
|
||||
def _profile_rel(coverage, cardinality, status):
|
||||
return {"id": catalog.relation_id("visits", ["gate_id"], "gates", ["id"]), "columns": ["gate_id"],
|
||||
"to_table": "gates", "to_columns": ["id"], "cardinality": cardinality, "coverage": coverage,
|
||||
"source": "profile", "status": status}
|
||||
|
||||
|
||||
def test_reset_returns_profile_relation_to_what_profiling_concluded():
|
||||
"""撤销审阅回到「来源的初始状态」。剖析的结论随数据而定:覆盖率够、父键唯一(基数有值)就是有确证。"""
|
||||
verified = {"relations": [_profile_rel(1.0, "many_to_one", "verified")]}
|
||||
rid = verified["relations"][0]["id"]
|
||||
out = catalog.review_item(verified, f"relations.{rid}", "confirm", at=AT)
|
||||
assert out["relations"][0]["status"] == "confirmed"
|
||||
out = catalog.review_item(out, f"relations.{rid}", "reset", at=AT)
|
||||
assert out["relations"][0]["status"] == "verified"
|
||||
out = catalog.review_item(catalog.review_item(out, f"relations.{rid}", "reject", at=AT),
|
||||
f"relations.{rid}", "reset", at=AT)
|
||||
assert out["relations"][0]["status"] == "verified"
|
||||
# 覆盖率低、父键没核实唯一的:撤销后仍是推断
|
||||
for coverage, cardinality in ((0.62, "many_to_one"), (1.0, None)):
|
||||
low = {"relations": [_profile_rel(coverage, cardinality, "proposed")]}
|
||||
out = catalog.review_item(catalog.review_item(low, f"relations.{rid}", "confirm", at=AT),
|
||||
f"relations.{rid}", "reset", at=AT)
|
||||
assert out["relations"][0]["status"] == "proposed"
|
||||
# 剖析提的码值候选、业务日期永远是推断
|
||||
codes = {"columns": {"status": {"codes": make_item({"1": ""}, "profile", "proposed")}}}
|
||||
out = catalog.review_item(catalog.review_item(codes, "columns.status.codes", "confirm", at=AT),
|
||||
"columns.status.codes", "reset", at=AT)
|
||||
assert out["columns"]["status"]["codes"]["status"] == "proposed"
|
||||
|
||||
|
||||
def test_human_edit_may_restore_profile_conclusion_but_not_invent_one():
|
||||
rel = _profile_rel(1.0, "many_to_one", "confirmed")
|
||||
existing = {"relations": [rel]}
|
||||
back = catalog.apply_human_edit(existing, {"relations": [{**rel, "status": "verified"}]}, table="visits", at=AT)
|
||||
assert back["relations"][0]["status"] == "verified" and back["relations"][0]["source"] == "profile"
|
||||
low = _profile_rel(0.4, "many_to_one", "proposed")
|
||||
kept = catalog.apply_human_edit({"relations": [low]}, {"relations": [{**low, "status": "verified"}]},
|
||||
table="visits", at=AT)
|
||||
assert kept["relations"][0]["status"] == "proposed"
|
||||
@@ -24,6 +24,7 @@ from app.db.models import DataSource, Run, RunEvent, Workflow, WorkflowVersion
|
||||
from app.engine.runner import run_manager
|
||||
from app.main import app
|
||||
from tests.fixtures.catalog import scenic_notes
|
||||
from tests.fixtures.sources import drop_source
|
||||
|
||||
_CACHE: dict[str, dict] = {}
|
||||
SQL = "SELECT COUNT(*) AS n FROM visits v JOIN parks p ON p.id = v.park_id WHERE v.status = 1"
|
||||
@@ -72,9 +73,8 @@ async def source(scenic_db):
|
||||
for wid in made:
|
||||
if (wf := await session.get(Workflow, wid)) is not None:
|
||||
await session.delete(wf)
|
||||
if (row := await session.get(DataSource, sid)) is not None:
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
await drop_source(sid)
|
||||
|
||||
|
||||
def node(nid, ntype, **config):
|
||||
|
||||
@@ -7,6 +7,7 @@ from app.data import catalog
|
||||
from app.data.catalog import make_item
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import Artifact, DataSource, Run
|
||||
from tests.fixtures.sources import drop_source
|
||||
|
||||
MARK = "(推断,未确认)"
|
||||
|
||||
@@ -184,4 +185,5 @@ async def test_table_usage_counts_run_queries_per_table(monkeypatch):
|
||||
await session.commit()
|
||||
source = await session.get(DataSource, sid)
|
||||
usage = await catalog.table_usage(session, source)
|
||||
await drop_source(sid)
|
||||
assert usage == {"visits": 4, "明细": 2, "parks": 1}
|
||||
|
||||
@@ -14,12 +14,12 @@ from types import SimpleNamespace
|
||||
from app.core.artifact_store import canonical_json, content_hash, load
|
||||
from app.data import catalog
|
||||
from app.data.catalog import make_item
|
||||
from app.data.engine import engines
|
||||
from app.data.introspect import describe_table, introspect
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import Artifact, DataSource
|
||||
from app.tools.datasource import _store_schema, build_datasource_tools
|
||||
from app.tools.registry import ToolContext
|
||||
from tests.fixtures.sources import drop_source
|
||||
|
||||
MARK = "(推断,未确认)"
|
||||
|
||||
@@ -182,4 +182,4 @@ async def test_tools_show_and_freeze_catalog_and_record_query_tables(scenic_db):
|
||||
ref = await session.get(Artifact, (result["artifact"], run_id))
|
||||
assert ref.meta == {"source_id": row.id, "source": row.name, "tables": ["visits", "parks"]}
|
||||
finally:
|
||||
await engines.invalidate(row.id)
|
||||
await drop_source(row.id)
|
||||
|
||||
@@ -20,6 +20,7 @@ from app.data.engine import engines
|
||||
from app.data.introspect import introspect
|
||||
from app.data.sqlcheck import SqlChecker
|
||||
from tests.fixtures.catalog import scenic_notes
|
||||
from tests.fixtures.sources import drop_source
|
||||
|
||||
FANOUT_SQL = ("SELECT SUM(o.total_amount) AS gmv FROM orders o JOIN order_items i ON i.order_id = o.id "
|
||||
"WHERE o.status = 1")
|
||||
@@ -177,10 +178,7 @@ async def scenic_source(scenic_db):
|
||||
await catalog.write_entry(session, row.id, table, notes, if_version=0, actor="王敏")
|
||||
source_id = row.id
|
||||
yield name
|
||||
async with SessionLocal() as session:
|
||||
row = await session.get(DataSource, source_id)
|
||||
row.enabled = False
|
||||
await session.commit()
|
||||
await drop_source(source_id)
|
||||
|
||||
|
||||
async def _stream(monkeypatch, model, source):
|
||||
|
||||
@@ -23,6 +23,7 @@ from app.engine.governance import lint_for_publish, publish_issues
|
||||
from app.engine.schema import GraphSpec
|
||||
from app.main import app
|
||||
from tests.fixtures.catalog import scenic_notes
|
||||
from tests.fixtures.sources import drop_source
|
||||
|
||||
FANOUT_SQL = ("SELECT SUM(o.total_amount) AS gmv FROM orders o JOIN order_items i ON i.order_id = o.id "
|
||||
"WHERE o.status = 1")
|
||||
@@ -133,10 +134,7 @@ async def scenic_source(scenic_db):
|
||||
await catalog.write_entry(session, row.id, table, notes, if_version=0, actor="王敏")
|
||||
source_id = row.id
|
||||
yield name
|
||||
async with SessionLocal() as session:
|
||||
row = await session.get(DataSource, source_id)
|
||||
row.enabled = False
|
||||
await session.commit()
|
||||
await drop_source(source_id)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
||||
@@ -32,6 +32,7 @@ from app.main import app
|
||||
from app.tools.datasource import build_datasource_tools
|
||||
from app.tools.registry import ToolContext
|
||||
from tests.fixtures.catalog import scenic_notes
|
||||
from tests.fixtures.sources import drop_source
|
||||
|
||||
FANOUT_SQL = ("SELECT SUM(o.total_amount) AS gmv FROM orders o JOIN order_items i ON i.order_id = o.id "
|
||||
"WHERE o.status = 1")
|
||||
@@ -54,6 +55,16 @@ async def _schema(path: str) -> dict:
|
||||
return _CACHE[path]
|
||||
|
||||
|
||||
_CREATED: list[str] = []
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
async def _drop_created_sources():
|
||||
yield
|
||||
while _CREATED:
|
||||
await drop_source(_CREATED.pop())
|
||||
|
||||
|
||||
async def _source(path: str, *, with_catalog: bool = True) -> tuple[str, str]:
|
||||
name = f"scenic_{uuid.uuid4().hex[:8]}"
|
||||
async with SessionLocal() as session:
|
||||
@@ -61,6 +72,7 @@ async def _source(path: str, *, with_catalog: bool = True) -> tuple[str, str]:
|
||||
row.schema_cache = await _schema(path)
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
_CREATED.append(row.id)
|
||||
if with_catalog:
|
||||
for table, notes in scenic_notes.notes().items():
|
||||
await catalog.write_entry(session, row.id, table, notes, if_version=0, actor="王敏")
|
||||
@@ -69,9 +81,8 @@ async def _source(path: str, *, with_catalog: bool = True) -> tuple[str, str]:
|
||||
|
||||
@pytest.fixture
|
||||
async def scenic_source(scenic_db):
|
||||
source_id, name = await _source(scenic_db)
|
||||
_, name = await _source(scenic_db)
|
||||
yield name
|
||||
await engines.invalidate(source_id)
|
||||
|
||||
|
||||
async def _query_tool(name: str):
|
||||
|
||||
@@ -254,6 +254,16 @@ async def test_join_unconfirmed_negative(cache, sql):
|
||||
assert "join_unconfirmed" not in codes(cache, sql)
|
||||
|
||||
|
||||
async def test_sibling_join_through_a_shared_parent_is_only_a_note(cache):
|
||||
"""两张表各自按 member_id 指向会员(都是外键),彼此按 member_id 相连:目录里没有这两张表之间的关系,
|
||||
但这是经共同父表推得出来的连法,不是「连错了列」。只提示一句:关联后行数可能成倍增加。"""
|
||||
sql = "SELECT COUNT(*) FROM orders o JOIN member_tag_links t ON t.member_id = o.member_id WHERE o.status = 1"
|
||||
[check] = [c for c in run(cache, sql) if c.code == "join_unconfirmed"]
|
||||
assert check.level == "info"
|
||||
assert "会员" in check.message or "members" in check.message
|
||||
assert "成倍" in check.message
|
||||
|
||||
|
||||
async def test_join_level_follows_status(cache):
|
||||
sql = "SELECT SUM(i.amount) FROM orders o JOIN order_items i ON i.order_id = o.id WHERE o.status = 1"
|
||||
assert "join_unconfirmed" not in codes(cache, sql, "confirmed")
|
||||
|
||||
@@ -34,6 +34,7 @@ from app.engine.expressions import TruncatedUse, eval_expression
|
||||
from app.engine.issuance import decide_tier, incomplete_gaps
|
||||
from app.engine.runner import run_manager
|
||||
from app.main import app
|
||||
from tests.fixtures.sources import drop_source
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 求值器:整组用到截断结果时记一笔,单格不记
|
||||
@@ -345,7 +346,7 @@ async def park(ticket_db, engine_up):
|
||||
# 连接池按数据源 id 缓存:换了库文件不作废,查的还是上一个测试的库
|
||||
await engines.invalidate(source_id)
|
||||
yield
|
||||
await engines.invalidate(source_id)
|
||||
await drop_source(source_id)
|
||||
|
||||
|
||||
def node(nid, ntype, **config):
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import type {
|
||||
ActivateSnapshotBody, ActivateSnapshotOut, AiPreview, CatalogDetail, CatalogDraftOut, CatalogImpact, CatalogList, CatalogNotes,
|
||||
CatalogPatchPreview, CatalogPatchSubmit, CatalogReviewAction, CommitOut, CurrentRecipe, EditPreview, EditRequest, ImportRecord,
|
||||
CatalogPatchPreview, CatalogPatchSubmit, CatalogProfileOut, CatalogReviewAction, CommitOut, CurrentRecipe, EditPreview,
|
||||
EditRequest, ImportRecord,
|
||||
ManifestOut, PurgeRawBody, PurgeRawOut, QuestionAnswer, Recipe, RedraftRulesOut, RemovePeriodBody, RemovePeriodOut,
|
||||
RevokeAcceptanceBody, RevokeAcceptanceOut, SnapshotOut, Staging,
|
||||
} from '../types'
|
||||
@@ -687,6 +688,13 @@ export const api = {
|
||||
draft: (sourceId: string, body: { tables?: string[]; use_model: boolean }, opts?: RequestOptions) =>
|
||||
request<CatalogDraftOut>(`/datasources/${encodeURIComponent(sourceId)}/catalog/draft`,
|
||||
{ method: 'POST', body: JSON.stringify(body), ...opts }),
|
||||
/**
|
||||
* 同步剖析:对业务库发少量只读查询,核对关系、取码值候选,结论写进目录。不给 tables 时按使用次数取前 10 张
|
||||
* 有待核实关系的表。没开启、数据源停用、没有表结构、同一个数据源正在剖析、快照被改动时回 409(detail 是一句中文)
|
||||
*/
|
||||
profile: (sourceId: string, body: { tables?: string[] }, opts?: RequestOptions) =>
|
||||
request<CatalogProfileOut>(`/datasources/${encodeURIComponent(sourceId)}/catalog/profile`,
|
||||
{ method: 'POST', body: JSON.stringify(body), ...opts }),
|
||||
},
|
||||
|
||||
runs: {
|
||||
|
||||
@@ -77,9 +77,28 @@ export function revealField(nodeId: string, field?: FieldRef | string | null, op
|
||||
const to = target.querySelector<HTMLElement>(FILLABLE) ?? target.querySelector<HTMLElement>(PRESSABLE)
|
||||
to?.focus({ preventScroll: true })
|
||||
}
|
||||
// 子键没有自己的一行、字段是一段 JSON(调用工具的「参数」,问题落在 args.sql 上):在文字里找到那个键,
|
||||
// 框内滚到那一行,聚焦时光标放在它的值开头
|
||||
if (!part && at.sub) revealJsonKey(target, at.sub, !!opts.focus)
|
||||
}))
|
||||
}
|
||||
|
||||
/** JSON 文字框里的某个键:滚到它那一行;focus 时把光标放在值的开头(不选中,免得一敲键就把整段 SQL 换掉) */
|
||||
function revealJsonKey(field: HTMLElement, key: string, focus: boolean): void {
|
||||
const box = field.querySelector<HTMLTextAreaElement>('textarea')
|
||||
if (!box) return
|
||||
const m = new RegExp(`"${key.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}"\\s*:\\s*"?`).exec(box.value)
|
||||
if (!m) return
|
||||
const line = box.value.slice(0, m.index).split('\n').length - 1
|
||||
const height = parseFloat(getComputedStyle(box).lineHeight) || 16
|
||||
box.scrollTop = Math.max(0, (line - 1) * height)
|
||||
if (focus) {
|
||||
const at = m.index + m[0].length
|
||||
box.focus({ preventScroll: true })
|
||||
box.setSelectionRange(at, at)
|
||||
}
|
||||
}
|
||||
|
||||
// 选中 → 面板开不开。放在模块级:选中可能来自别处(问题面板定位、发布弹窗、快捷键),
|
||||
// 规则只写这一份
|
||||
useStudio.subscribe((s, prev) => {
|
||||
|
||||
@@ -13,6 +13,7 @@ import { useCatalog } from '../store/catalog'
|
||||
import { Modal, Spinner, toast, useRadioGroup } from '../components/ui'
|
||||
import { humanizeError } from '../lib/errors'
|
||||
import { PUBLISH_FIX_TEXT as T, WORKFLOW_STATUS_HINT, WORKFLOW_STATUS_LABEL } from '../lib/terms'
|
||||
import { isSqlCheckCode, sqlRuleLabel } from '../lib/sqlcheck'
|
||||
import { setLocalActor, useLocalActor } from '../lib/actor'
|
||||
import {
|
||||
autoFixIds, choiceReady, contentSig, fixFieldLabel, fixFor, fixValueLines, fixValueText, isMissingEndpoint, mergeIssues,
|
||||
@@ -42,7 +43,8 @@ export function PublishDialog({ workflow, onClose, onDone, onLocate }: {
|
||||
workflow: Workflow
|
||||
onClose: () => void
|
||||
onDone: () => void
|
||||
onLocate: (nodeId: string) => void
|
||||
/** 定位一条问题:选中节点,能落到具体配置项(调用工具的 SQL)就落到那一项 */
|
||||
onLocate: (issue: ValidationIssue) => void
|
||||
}) {
|
||||
const load = useStudio((s) => s.load)
|
||||
const dirty = useStudio((s) => s.dirty)
|
||||
@@ -208,7 +210,7 @@ export function PublishDialog({ workflow, onClose, onDone, onLocate }: {
|
||||
</dl>
|
||||
|
||||
<div className="mt-3" data-preflight="dialog">
|
||||
<Preflight pf={pf} gate={gate} onLocate={(issue) => { if (issue.node_id) { onLocate(issue.node_id); onClose() } }} />
|
||||
<Preflight pf={pf} gate={gate} onLocate={(issue) => { if (issue.node_id) { onLocate(issue); onClose() } }} />
|
||||
</div>
|
||||
</Modal>
|
||||
)
|
||||
@@ -564,6 +566,8 @@ function PreflightRow({ issue, others, fix, where, pf, onLocate }: {
|
||||
aria-label={err ? '错误' : '提示'} />
|
||||
<span className="min-w-0 flex-1 [overflow-wrap:anywhere]">
|
||||
{where && !String(issue.message).includes(`「${where}」`) && <b className="font-semibold">「{where}」</b>}
|
||||
{/* 对照数据目录的 SQL 检查:先写中文规则名,不露规则编号 */}
|
||||
{isSqlCheckCode(issue.code) && <span className="font-medium" data-preflight-rule="">{sqlRuleLabel(issue.code)}:</span>}
|
||||
{issue.message}
|
||||
</span>
|
||||
{issue.node_id && <CornerDownRight size={10} className="mt-px shrink-0 text-faint" aria-hidden />}
|
||||
|
||||
@@ -310,6 +310,7 @@ function ToastCard({ item }: { item: ToastItem }) {
|
||||
<div
|
||||
className="fade-up pointer-events-auto flex items-start gap-2 rounded-lg border bg-panel px-3 py-2 shadow-elev-2"
|
||||
style={{ borderColor: item.kind === 'error' ? 'var(--err)' : item.kind === 'warn' ? 'var(--warn)' : 'var(--border)' }}
|
||||
data-toast={item.kind}
|
||||
onMouseEnter={() => setPaused(true)}
|
||||
onMouseLeave={() => setPaused(false)}
|
||||
onFocus={() => setPaused(true)}
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
import type { SqlCheckCode, SqlCheckItem, SqlCheckLevel } from '../types'
|
||||
import { SQL_CHECK_LEVEL_LABEL, SQL_CHECK_RULE_LABEL, SQL_CHECK_TEXT } from './terms'
|
||||
|
||||
// ===========================================================================
|
||||
// 基于数据目录的 SQL 检查(后端 data/sqlcheck.py)在界面上的读法:规则编号 → 中文规则名、级别 → 语义色,
|
||||
// 以及从证据接口、助手自查、发布前检查的问题里读出一条检查。证据面板、运行时间线、助手面板、发布前检查共用。
|
||||
// ===========================================================================
|
||||
|
||||
/** 七条规则的编号(服务端 CHECK_CODES),按严重程度大致排序 */
|
||||
export const SQL_CHECK_CODES: readonly SqlCheckCode[] = [
|
||||
'fanout_sum', 'stock_summed', 'join_unconfirmed', 'ratio_aggregated', 'missing_valid_filter', 'unknown_code', 'wrong_date_column',
|
||||
]
|
||||
|
||||
export const isSqlCheckCode = (code: unknown): code is SqlCheckCode =>
|
||||
typeof code === 'string' && (SQL_CHECK_CODES as readonly string[]).includes(code)
|
||||
|
||||
/** 级别:认不出的按「提示」(最轻的一档),不把拿不准的说重 */
|
||||
export const sqlCheckLevelOf = (v: unknown): SqlCheckLevel => (v === 'error' || v === 'warning' ? v : 'info')
|
||||
|
||||
/** 规则的中文名;服务端新加、这里还不认识的规则统称「SQL 检查」,不露编号 */
|
||||
export const sqlRuleLabel = (code: unknown): string => (isSqlCheckCode(code) ? SQL_CHECK_RULE_LABEL[code] : SQL_CHECK_TEXT.unknownRule)
|
||||
|
||||
/** 各级别的颜色(语义令牌) */
|
||||
export const SQL_CHECK_TONE: Record<SqlCheckLevel, string> = {
|
||||
error: 'var(--st-failed)',
|
||||
warning: 'var(--st-waiting)',
|
||||
info: 'var(--accent)',
|
||||
}
|
||||
|
||||
const LEVEL_RANK: Record<SqlCheckLevel, number> = { error: 0, warning: 1, info: 2 }
|
||||
|
||||
/** 错误在前,同级保持原来的顺序 */
|
||||
export function sortSqlChecks<T extends { level?: unknown }>(list: T[]): T[] {
|
||||
return list.map((c, i) => ({ c, i }))
|
||||
.sort((a, b) => LEVEL_RANK[sqlCheckLevelOf(a.c.level)] - LEVEL_RANK[sqlCheckLevelOf(b.c.level)] || a.i - b.i)
|
||||
.map((x) => x.c)
|
||||
}
|
||||
|
||||
/** 涉及的表和列:「orders.amount」「orders」 */
|
||||
export const sqlCheckWhere = (c: Pick<SqlCheckItem, 'table' | 'column'>): string =>
|
||||
(c.table ? (c.column ? `${c.table}.${c.column}` : c.table) : (c.column ?? ''))
|
||||
|
||||
/** 从任意对象读一条检查(证据接口的 checks、助手自查和发布前检查的问题);不是 SQL 检查的返回 null */
|
||||
export function sqlCheckOf(v: unknown): SqlCheckItem | null {
|
||||
if (!v || typeof v !== 'object') return null
|
||||
const o = v as Record<string, unknown>
|
||||
if (!isSqlCheckCode(o.code)) return null
|
||||
const str = (x: unknown) => (typeof x === 'string' && x ? x : undefined)
|
||||
return {
|
||||
code: o.code,
|
||||
level: sqlCheckLevelOf(o.level),
|
||||
message: String(o.message ?? ''),
|
||||
table: str(o.table) ?? '',
|
||||
...(str(o.column) ? { column: str(o.column) } : {}),
|
||||
...(str(o.relation_id) ? { relation_id: str(o.relation_id) } : {}),
|
||||
...(str(o.sql_excerpt) ? { sql_excerpt: str(o.sql_excerpt) } : {}),
|
||||
}
|
||||
}
|
||||
|
||||
/** 一条检查写成一行(时间线的展开区、交给助手再修的清单):「错误 · 一对多关联后重复计算:……(涉及 orders.amount)」 */
|
||||
export function sqlCheckLine(c: SqlCheckItem): string {
|
||||
const where = sqlCheckWhere(c)
|
||||
return `${SQL_CHECK_LEVEL_LABEL[sqlCheckLevelOf(c.level)]} · ${sqlRuleLabel(c.code)}:${c.message}`
|
||||
+ (where ? `(${SQL_CHECK_TEXT.where} ${where})` : '')
|
||||
}
|
||||
+287
-2
@@ -43,7 +43,8 @@
|
||||
*/
|
||||
|
||||
import type {
|
||||
CatalogCardinality, CatalogMeasure, CatalogSource, CatalogStatus, CatalogTableKind, NodeType, SnapshotReasonCode, ToolTrust,
|
||||
CatalogCardinality, CatalogMeasure, CatalogProfileNumberKey, CatalogSource, CatalogStatus, CatalogTableKind, NodeType, SnapshotReasonCode,
|
||||
SqlCheckCode, SqlCheckLevel, ToolTrust,
|
||||
} from '../types'
|
||||
import { formatNumber } from './format'
|
||||
|
||||
@@ -247,6 +248,9 @@ export const EVIDENCE_TEXT = {
|
||||
/** 指标拿截断的查询结果整组算出来(计数、求和……):值只算到了取回的那部分,出具按缺口降档 */
|
||||
incomplete: '结果不完整',
|
||||
incompleteFallback: '基于被截断的查询结果计算,结果不完整',
|
||||
/** 指标所依据的查询对照数据目录查出了错误级的问题:值照算,出具按缺口降档 */
|
||||
sqlCheckFailed: 'SQL 检查未通过',
|
||||
sqlCheckFallback: '所依据的查询未通过 SQL 检查,结果不可靠',
|
||||
/** 输入是 Agent 交来、被截断切开的数组字段 */
|
||||
inputTruncated: '查询结果已截断,该字段只含取回的部分行',
|
||||
unshowable: (value: string) => `有值(${value}),但无法按口径卡的格式显示`,
|
||||
@@ -2127,7 +2131,7 @@ export const CATALOG_TEXT = {
|
||||
keysPlaceholder: '列名,用顿号或逗号分隔',
|
||||
keysUnknown: (cols: string) => `表结构中没有这些列:${cols}`,
|
||||
codesPlaceholder: '1=已支付',
|
||||
codesHint: '每行一个,写成「码值=含义」',
|
||||
codesHint: '每行一个,写成「码值=含义」,含义可暂不填写',
|
||||
codesInvalid: (line: number) => `第 ${formatNumber(line)} 行应写成「码值=含义」`,
|
||||
codesDuplicate: (code: string) => `码值 ${code} 重复`,
|
||||
dateColumnRequired: '填写规则或时区时,日期列必填',
|
||||
@@ -2241,3 +2245,284 @@ export const CATALOG_DRIFT_TEXT = {
|
||||
unchanged: '发布之后这些表的目录没有变化',
|
||||
empty: '这一版的 SQL 没有用到数据库表',
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// 数据剖析(数据源设置里的开关和预算;数据目录页的「数据剖析」)
|
||||
//
|
||||
// 术语:
|
||||
// - 「数据剖析」:对业务库发少量只读查询,核实推断的关联关系(覆盖率、基数),取状态类列的码值候选,提议业务日期。
|
||||
// 动作写「剖析」,不写扫描、探测。默认关闭,按数据源开启。
|
||||
// - 预算五项:查询次数上限、单条查询时限、抽样键值数、整表统计行数上限、总时长上限。标签和服务端校验报错里的叫法
|
||||
// 一致(catalog_profile._NUMBER_FIELDS),保存被拒时按它认出是哪一项。
|
||||
// ===========================================================================
|
||||
|
||||
export const PROFILE_FIELD_LABEL: Record<'enabled' | CatalogProfileNumberKey, string> = {
|
||||
enabled: '开启数据剖析',
|
||||
max_queries: '查询次数上限',
|
||||
query_timeout_s: '单条查询时限',
|
||||
sample_size: '抽样键值数',
|
||||
max_scan_rows: '整表统计行数上限',
|
||||
max_total_s: '总时长上限',
|
||||
}
|
||||
|
||||
export const PROFILE_FIELD_UNIT: Record<CatalogProfileNumberKey, string> = {
|
||||
max_queries: '条',
|
||||
query_timeout_s: '秒',
|
||||
sample_size: '个',
|
||||
max_scan_rows: '行',
|
||||
max_total_s: '秒',
|
||||
}
|
||||
|
||||
export const PROFILE_FIELD_HINT: Record<CatalogProfileNumberKey, string> = {
|
||||
max_queries: '一次剖析最多发出的查询条数,用完即停止',
|
||||
query_timeout_s: '超过即中断这条查询;不超过数据源自身的查询时限',
|
||||
sample_size: '核对一条关联关系时,从本表抽取的不同键值个数',
|
||||
max_scan_rows: '不超过此行数的表才做去重计数、最小值和最大值等整表统计',
|
||||
max_total_s: '一次剖析的总时长,到时即停止',
|
||||
}
|
||||
|
||||
export const PROFILE_TEXT = {
|
||||
// ---- 设置
|
||||
section: '数据剖析',
|
||||
enableHint: '默认关闭。开启后可在数据目录中核实推断的关联关系、取状态类列的码值候选',
|
||||
risk: (queries: number, timeout: number, total: number) =>
|
||||
`开启后,每次剖析会对这个数据源发出只读查询:最多 ${formatNumber(queries)} 条,单条不超过 ${formatNumber(timeout)} 秒,`
|
||||
+ `总时长不超过 ${formatNumber(total)} 秒。不读取明细行,遮罩的列不取样。`,
|
||||
range: (min: number, max: number, unit: string, def: number) =>
|
||||
`范围 ${formatNumber(min)}–${formatNumber(max)} ${unit},默认 ${formatNumber(def)}`,
|
||||
scanZero: '填 0 表示一律不做整表统计',
|
||||
placeholder: (def: number) => `默认 ${formatNumber(def)}`,
|
||||
rangeError: (min: number, max: number, integer: boolean) =>
|
||||
`请填写 ${formatNumber(min)} 到 ${formatNumber(max)} 之间的${integer ? '整数' : '数'}`,
|
||||
invalid: (n: number) => `数据剖析有 ${formatNumber(n)} 项设置不正确`,
|
||||
// ---- 数据源卡片
|
||||
chip: '数据剖析已开启',
|
||||
chipHint: (queries: number, timeout: number) =>
|
||||
`数据剖析已开启:每次最多 ${formatNumber(queries)} 条只读查询,单条不超过 ${formatNumber(timeout)} 秒`,
|
||||
open: '设置数据剖析',
|
||||
dialogTitle: (name: string) => `「${name}」的数据剖析设置`,
|
||||
dialogBody: '设置保存在数据源上,对数据目录中的每次剖析生效。',
|
||||
savedOn: (name: string) => `「${name}」已开启数据剖析`,
|
||||
savedOff: (name: string) => `「${name}」已关闭数据剖析`,
|
||||
saved: (name: string) => `已保存「${name}」的数据剖析设置`,
|
||||
// ---- 数据目录页:范围与确认
|
||||
action: '数据剖析',
|
||||
actionHint: '对业务库发少量只读查询:核实推断的关联关系、取码值候选、提议业务日期',
|
||||
profileSelected: (n: number) => `剖析所选(${formatNumber(n)})`,
|
||||
title: '数据剖析',
|
||||
intro: '对业务库发少量只读查询,核实推断的关联关系(覆盖率、基数),取状态类列的码值候选,提议业务日期。'
|
||||
+ '结论写入数据目录;已确认、已驳回的项不会被改动。',
|
||||
scope: '剖析范围',
|
||||
scopeSelected: (n: number) => `已选的 ${formatNumber(n)} 张表`,
|
||||
scopeFiltered: (n: number) => `当前筛选结果中的 ${formatNumber(n)} 张表`,
|
||||
scopeDefault: '使用次数最多、有待核实关联关系的表',
|
||||
scopeDefaultHint: (n: number) => `由服务端挑选,至多 ${formatNumber(n)} 张`,
|
||||
scopeTooMany: (max: number) => `超过 ${formatNumber(max)} 张,请缩小范围`,
|
||||
budget: '本次预算',
|
||||
budgetQueries: (queries: number, timeout: number) =>
|
||||
`最多 ${formatNumber(queries)} 条只读查询,单条不超过 ${formatNumber(timeout)} 秒`,
|
||||
budgetTotal: (total: number) => `总时长不超过 ${formatNumber(total)} 秒,到时即停止`,
|
||||
budgetSample: (n: number) => `每条关联关系抽样 ${formatNumber(n)} 个键值`,
|
||||
budgetScan: (n: number) => (n > 0 ? `行数超过 ${formatNumber(n)} 的表不做整表统计` : '不做整表统计'),
|
||||
budgetFrom: (name: string) => `按「${name}」的数据剖析设置`,
|
||||
editSettings: '修改设置',
|
||||
noCancel: '剖析在服务端一次完成,开始后无法中途停止。',
|
||||
start: '开始剖析',
|
||||
again: '重新剖析',
|
||||
disabledTitle: '这个数据源未开启数据剖析',
|
||||
disabledBody: '剖析会对业务库发出只读查询,需要先在数据源设置中开启。',
|
||||
enable: '开启数据剖析',
|
||||
// ---- 进行中
|
||||
running: '正在剖析…',
|
||||
elapsed: (used: string, total: string) => `已用时 ${used}(总时长上限 ${total})`,
|
||||
runningHint: '剖析在服务端一次完成,无法中途停止。关闭此窗口不影响剖析,完成后会提示结果。',
|
||||
background: '在后台继续',
|
||||
headerRunning: (used: string) => `正在剖析 ${used}`,
|
||||
headerRunningHint: '数据剖析进行中,点击查看',
|
||||
leavePage: '离开本页不会中止剖析,结论仍会写入数据目录,但不再显示本次的报告。',
|
||||
// ---- 报告
|
||||
done: '剖析完成',
|
||||
stoppedTitle: (reason: string) => `剖析已停止:${reason}`,
|
||||
summary: (used: number, max: number, tables: number) =>
|
||||
`用了 ${formatNumber(used)} / ${formatNumber(max)} 条查询,剖析 ${formatNumber(tables)} 张表`,
|
||||
changes: (added: number, updated: number, removed: number) =>
|
||||
`新增 ${formatNumber(added)} 项、更新 ${formatNumber(updated)} 项${removed ? `、删除 ${formatNumber(removed)} 项` : ''}`,
|
||||
noChange: '目录没有变化',
|
||||
toastDone: (summary: string) => `数据剖析完成:${summary}`,
|
||||
toastStopped: (reason: string) => `数据剖析已停止:${reason}`,
|
||||
toastFailed: '数据剖析未能完成',
|
||||
viewReport: '查看报告',
|
||||
close: '完成',
|
||||
tableQueries: (n: number) => `${formatNumber(n)} 条查询`,
|
||||
rowsStats: (n: number) => `约 ${formatNumber(n)} 行`,
|
||||
rowsCount: (n: number) => `${formatNumber(n)} 行`,
|
||||
rowsAtLeast: (n: number) => `超过 ${formatNumber(n - 1)} 行`,
|
||||
rowsHint: { stats: '数据库统计信息中的估算值', count: '计数到整表统计行数上限为止' } as Record<string, string>,
|
||||
openTable: '打开',
|
||||
openTableLabel: (name: string) => `打开「${name}」`,
|
||||
noFindings: '没有新的发现',
|
||||
findingRelations: '关联关系',
|
||||
findingCodes: '码值候选',
|
||||
findingDate: '业务日期',
|
||||
relationVerified: '升为已验证',
|
||||
relationKept: '保持推断',
|
||||
relationConfirmed: '已人工确认,只补充覆盖率和基数',
|
||||
relationLowCoverage: '覆盖率不足,可能不是这条关系',
|
||||
relationNotUnique: '被指向列未核实唯一',
|
||||
coverage: (pct: string) => `覆盖率 ${pct}`,
|
||||
sampled: (sample: number, matched: number) => `抽样 ${formatNumber(sample)} 个键值,对上 ${formatNumber(matched)} 个`,
|
||||
cardinalityUnknown: '基数未核实',
|
||||
codeRows: (n: number) => `${formatNumber(n)} 行`,
|
||||
codesSummary: (values: number, rows: number) => `${formatNumber(values)} 个取值,共 ${formatNumber(rows)} 行`,
|
||||
codesPendingN: (n: number) => `${formatNumber(n)} 个含义待填写`,
|
||||
fillMeanings: '填写含义',
|
||||
fillMeaningsLabel: (col: string) => `填写含义:列 ${col} 的码值`,
|
||||
dateProposal: (col: string) => `提议按 ${col} 作为业务日期`,
|
||||
dateSpan: (min: string, max: string) => `取值从 ${min} 到 ${max}`,
|
||||
dateRanges: '日期列的取值范围',
|
||||
skippedTitle: (n: number) => `跳过 ${formatNumber(n)} 项`,
|
||||
moreSkipped: (n: number) => `…另有 ${formatNumber(n)} 项`,
|
||||
tableError: '这张表未能剖析',
|
||||
// ---- 无法开始(409)
|
||||
blocked: {
|
||||
disabled: '未开启数据剖析',
|
||||
inactive: '数据源已停用',
|
||||
noSchema: '还没有表结构',
|
||||
busy: '这个数据源正在剖析',
|
||||
tampered: '数据文件核对未通过',
|
||||
other: '无法开始剖析',
|
||||
} as Record<string, string>,
|
||||
blockedNext: {
|
||||
disabled: '开启后即可在这里剖析。',
|
||||
inactive: '启用后回到这里重试。',
|
||||
noSchema: '探查完成后回到这里重试。',
|
||||
busy: '同一个数据源同一时间只能进行一次剖析。',
|
||||
tampered: '请重新上传表格,或联系管理员核对数据文件。',
|
||||
other: '请稍后重试。',
|
||||
} as Record<string, string>,
|
||||
goSource: '前往数据源',
|
||||
retry: '重试',
|
||||
failedTitle: '剖析未能完成',
|
||||
backToSetup: '返回',
|
||||
}
|
||||
|
||||
/** 整次剖析中途停下的原因 */
|
||||
export const PROFILE_STOP_LABEL: Record<string, string> = {
|
||||
budget: '查询次数已用完',
|
||||
deadline: '总时长已用完',
|
||||
failed: '连续多条查询失败',
|
||||
}
|
||||
|
||||
/** 停下之后怎么办 */
|
||||
export const PROFILE_STOP_NEXT: Record<string, string> = {
|
||||
budget: '其余检查未执行。可在数据剖析设置中调高查询次数上限,或缩小剖析范围后重新剖析。',
|
||||
deadline: '其余检查未执行。可在数据剖析设置中调高总时长上限,或缩小剖析范围后重新剖析。',
|
||||
failed: '其余检查未执行。请先在数据源卡片上测试连接,确认可用后重新剖析。',
|
||||
}
|
||||
|
||||
/** 跳过一项的原因(skipped.reason)。整句说明在 skipped.detail 里 */
|
||||
export const PROFILE_SKIP_REASON_LABEL: Record<string, string> = {
|
||||
budget: '查询次数用完',
|
||||
deadline: '总时长用完',
|
||||
failed: '连续失败',
|
||||
timeout: '查询超时',
|
||||
error: '查询失败',
|
||||
rejected: '未通过安全守卫',
|
||||
masked: '列已遮罩',
|
||||
too_large: '表太大',
|
||||
view: '视图',
|
||||
unsupported: '暂不支持',
|
||||
no_data: '没有数据',
|
||||
missing: '表结构中没有',
|
||||
high_cardinality: '取值太多',
|
||||
}
|
||||
|
||||
/** 跳过的是哪一类检查(skipped.kind) */
|
||||
export const PROFILE_SKIP_KIND_LABEL: Record<string, string> = {
|
||||
relation: '关联关系',
|
||||
codes: '码值',
|
||||
date: '日期列',
|
||||
row_estimate: '行数估算',
|
||||
table: '整张表',
|
||||
}
|
||||
|
||||
/** 码值候选:剖析只知道出现过哪些取值,含义等人填 */
|
||||
export const CODES_TEXT = {
|
||||
pending: '含义待填写',
|
||||
pendingN: (n: number) => `${formatNumber(n)} 个含义待填写`,
|
||||
fill: '填写含义',
|
||||
fillLabel: (col: string) => `填写含义:列 ${col} 的码值`,
|
||||
title: (col: string) => `列 ${col} 的码值含义`,
|
||||
hint: '保存后这一项记为人工填写、已确认。暂不清楚的含义可以留空,之后再填。',
|
||||
head: { code: '码值', meaning: '含义' },
|
||||
note: '来源说明',
|
||||
placeholder: '含义待填写',
|
||||
saved: (n: number) => (n ? `已填写 ${formatNumber(n)} 个码值的含义` : '已保存'),
|
||||
save: '保存',
|
||||
cancel: '取消',
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// 基于数据目录的 SQL 检查(后端 data/sqlcheck.py)
|
||||
//
|
||||
// 术语:
|
||||
// - 「SQL 检查」:对照数据目录找出能执行、数却可能不对的写法。七条规则,界面上写中文规则名,不露规则编号。
|
||||
// - 级别:错误(依据已确认,结果必然有误)、提醒(依据已确认,结果很可能有误)、提示(依据只是推断,仅供参考)。
|
||||
// 颜色走语义令牌:错误 --st-failed,提醒 --st-waiting,提示 --accent。
|
||||
// ===========================================================================
|
||||
|
||||
export const SQL_CHECK_RULE_LABEL: Record<SqlCheckCode, string> = {
|
||||
fanout_sum: '一对多关联后重复计算',
|
||||
stock_summed: '存量跨期加总',
|
||||
join_unconfirmed: '关联条件未经确认',
|
||||
ratio_aggregated: '比率直接加总或平均',
|
||||
missing_valid_filter: '未筛选有效记录',
|
||||
unknown_code: '码值不在码值表中',
|
||||
wrong_date_column: '未按业务日期统计',
|
||||
}
|
||||
|
||||
export const SQL_CHECK_RULE_HINT: Record<SqlCheckCode, string> = {
|
||||
fanout_sum: '沿一对多方向关联后,对「一」那一侧的度量求和或计数,会按明细行数重复计算',
|
||||
stock_summed: '对存量列求和却没有按业务日期分组,不同日期的存量被加在一起',
|
||||
join_unconfirmed: '关联条件对不上数据目录中有确证的关系,可能连错了列',
|
||||
ratio_aggregated: '对比率列直接求和或求平均,得不到正确的比率',
|
||||
missing_valid_filter: '表定义了有效记录条件,查询没有按它筛选',
|
||||
unknown_code: '按码值筛选时用了码值表中没有的值',
|
||||
wrong_date_column: '表定义了业务日期,查询却按另一个时间列分组或筛选',
|
||||
}
|
||||
|
||||
export const SQL_CHECK_LEVEL_LABEL: Record<SqlCheckLevel, string> = {
|
||||
error: '错误',
|
||||
warning: '提醒',
|
||||
info: '提示',
|
||||
}
|
||||
|
||||
export const SQL_CHECK_LEVEL_HINT: Record<SqlCheckLevel, string> = {
|
||||
error: '依据已确认,结果必然有误',
|
||||
warning: '依据已确认,结果很可能有误',
|
||||
info: '依据尚未确认(推断),仅供参考',
|
||||
}
|
||||
|
||||
export const SQL_CHECK_TEXT = {
|
||||
title: 'SQL 检查',
|
||||
/** 规则编号不认识(服务端新加了规则)时的叫法 */
|
||||
unknownRule: 'SQL 检查',
|
||||
listLabel: (n: number) => `SQL 检查发现 ${formatNumber(n)} 处问题`,
|
||||
where: '涉及',
|
||||
excerpt: 'SQL 片段',
|
||||
catalogHint: '检查依据来自数据目录。依据有误时,请在数据目录中修正对应项。',
|
||||
// ---- 运行时间线:指标所依据的查询没通过 SQL 检查(日志 metric_sql_check)
|
||||
metricTitle: (name: string, n = 1) =>
|
||||
(n > 1 ? `指标「${name}」所依据的查询有 ${formatNumber(n)} 处未通过 SQL 检查` : `指标「${name}」所依据的查询未通过 SQL 检查`),
|
||||
metricTitleAnon: '指标所依据的查询未通过 SQL 检查',
|
||||
metricNext: '指标照常计算,但结果不可靠,出具按缺口降档。请在证据面板的查询步骤中查看问题,修改 SQL 后重新运行;'
|
||||
+ '检查依据有误时,请在数据目录中修正对应项',
|
||||
// ---- 助手:搭图时查出的问题
|
||||
counts: (errors: number, warnings: number, infos: number) =>
|
||||
[errors ? `${formatNumber(errors)} 处错误` : '', warnings ? `${formatNumber(warnings)} 处提醒` : '',
|
||||
infos ? `${formatNumber(infos)} 处提示` : ''].filter(Boolean).join('、'),
|
||||
assistantTitle: (counts: string) => `SQL 检查:${counts}`,
|
||||
assistantNext: '请打开对应节点,在「参数」中修改 SQL;检查依据有误时,请在数据目录中修正对应项',
|
||||
assistantNextChat: '可在画布中打开此工作流,修改对应节点「参数」中的 SQL;检查依据有误时,请在数据目录中修正对应项',
|
||||
assistantInfoNext: '这些检查的依据尚未确认,仅供参考。在数据目录中确认相关项后,检查结论更可靠',
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@ import { useEffect, useId, useMemo, useRef, useState } from 'react'
|
||||
import { useNavigate } from 'react-router-dom'
|
||||
import {
|
||||
AlertTriangle, BookMarked, ChevronRight, Database, EyeOff, FileSpreadsheet, History, Info, KeyRound, Layers, Lock, Plug, Plus,
|
||||
RefreshCw, Search, Table2, Upload, X,
|
||||
RefreshCw, ScanSearch, Search, Table2, Upload, X,
|
||||
} from 'lucide-react'
|
||||
import clsx from 'clsx'
|
||||
import { ApiError, api, uploadDecision } from '../api/client'
|
||||
@@ -21,9 +21,16 @@ import { checkHealth, forgetHealth, healthFromServer, setHealth, useHealth } fro
|
||||
import type { HealthRecord } from '../lib/health'
|
||||
import { workflowList, workflowsMentioning } from '../lib/mentions'
|
||||
import {
|
||||
CATALOG_TEXT, RAW_STATE_LABEL, RECIPE_ORIGIN_LABEL, RECIPE_TEXT, UPLOAD_CONVERSION_LABEL, UPLOAD_SHAPE_LABEL, UPLOAD_TEXT, VERSIONS_TEXT,
|
||||
CATALOG_TEXT, PROFILE_TEXT, RAW_STATE_LABEL, RECIPE_ORIGIN_LABEL, RECIPE_TEXT, UPLOAD_CONVERSION_LABEL, UPLOAD_SHAPE_LABEL, UPLOAD_TEXT,
|
||||
VERSIONS_TEXT,
|
||||
} from '../lib/terms'
|
||||
import { useRunClock } from '../run/useRunClock'
|
||||
import { focusProfileField, ProfileSettingsDialog, ProfileSettingsSection } from './catalog/ProfileSettings'
|
||||
import type { ProfileErrorKey } from './catalog/ProfileSettings'
|
||||
import {
|
||||
hasProfileOption, PROFILE_OPTION, profileFieldOfRejection, profileFormOf, profileOptionOf, profileProblems, profileSettingsOf,
|
||||
} from './catalog/profile'
|
||||
import type { ProfileForm } from './catalog/profile'
|
||||
import { ImportWizard } from './import/ImportWizard'
|
||||
import type { WizardEntry } from './import/ImportWizard'
|
||||
import { VersionsDialog } from './import/VersionsDialog'
|
||||
@@ -285,9 +292,11 @@ function SourceCard({ row, meta, onChange, onRemoved, onEdit, onReupload, onReci
|
||||
// 传上来的表通常只有一两张,直接摊开;库动辄几十上百个对象,默认收着
|
||||
const [open, setOpen] = useState(() => uploaded && row.table_count > 0 && row.table_count <= 3)
|
||||
const [masking, setMasking] = useState(false)
|
||||
const [profiling, setProfiling] = useState(false)
|
||||
/** 版本页(当前版本的各期、历史版本、导入记录) */
|
||||
const [versions, setVersions] = useState(false)
|
||||
const masks = maskList(row.options?.[MASK_KEY])
|
||||
const profile = profileSettingsOf(row.options)
|
||||
const clock = useRunClock(!!busy)
|
||||
const busySince = useRef(0)
|
||||
useTicker(60_000)
|
||||
@@ -525,12 +534,28 @@ function SourceCard({ row, meta, onChange, onRemoved, onEdit, onReupload, onReci
|
||||
{masks.length ? '修改遮罩列' : '设置遮罩列'}
|
||||
</button>
|
||||
)}
|
||||
{profile.enabled && (
|
||||
<span className="chip" data-profile-on="" title={PROFILE_TEXT.chipHint(profile.max_queries, profile.query_timeout_s)}>
|
||||
<ScanSearch size={10} aria-hidden /> {PROFILE_TEXT.chip}
|
||||
</span>
|
||||
)}
|
||||
{uploaded && (
|
||||
// 数据剖析的设置同理:手工登记的库在编辑框里改
|
||||
<button type="button" className="text-2xs underline decoration-dotted underline-offset-2 hover:text-dim"
|
||||
onClick={() => setProfiling(true)} data-edit-profile="">
|
||||
{PROFILE_TEXT.open}
|
||||
</button>
|
||||
)}
|
||||
{!!row.tools?.length && <span className="mono">{row.tools.join(' · ')}</span>}
|
||||
</div>
|
||||
{masking && (
|
||||
<MaskColumnsDialog row={row} onClose={() => setMasking(false)}
|
||||
onSaved={(next) => { setMasking(false); onChange(next); syncCatalog() }} />
|
||||
)}
|
||||
{profiling && (
|
||||
<ProfileSettingsDialog row={row} onClose={() => setProfiling(false)}
|
||||
onSaved={(next) => { setProfiling(false); onChange(next); syncCatalog() }} />
|
||||
)}
|
||||
{versions && (
|
||||
// onChange 是父级的「更新这一行并刷新全局目录」:回滚之后表结构可能变了,检查器、问数据读的目录要跟上
|
||||
<VersionsDialog source={row} onClose={() => setVersions(false)} onChange={onChange} onReload={onReload} />
|
||||
@@ -940,11 +965,16 @@ interface SourceForm {
|
||||
sid: string
|
||||
/** options.mask_columns:证据面板展示查询原始行时遮掉的列 */
|
||||
maskColumns: string[]
|
||||
/** options 里除 schema / service_name / sid / mask_columns 之外的键,全部摊开可编辑 */
|
||||
/** options.catalog_profile:数据剖析的开关和预算 */
|
||||
profile: ProfileForm
|
||||
/** 打开时 options 里已经有剖析设置:没开、也没填时也要照写(关掉之后写回 enabled: false) */
|
||||
profileStored: boolean
|
||||
/** options 里除 schema / service_name / sid / mask_columns / catalog_profile 之外的键,全部摊开可编辑 */
|
||||
advanced: [string, string][]
|
||||
}
|
||||
|
||||
const OWN_OPTION_KEYS = new Set(['schema', 'service_name', 'sid', MASK_KEY])
|
||||
// 剖析设置是一个对象,不能摊进「高级连接参数」:那里按文字回显,保存时会写成「[object Object]」被服务端拒收
|
||||
const OWN_OPTION_KEYS = new Set(['schema', 'service_name', 'sid', MASK_KEY, PROFILE_OPTION])
|
||||
|
||||
function toForm(source: any): SourceForm {
|
||||
const o = source.options ?? {}
|
||||
@@ -966,6 +996,8 @@ function toForm(source: any): SourceForm {
|
||||
serviceName: o.service_name ?? (oracle ? source.database ?? '' : ''),
|
||||
sid: o.sid ?? '',
|
||||
maskColumns: maskList(o[MASK_KEY]),
|
||||
profile: profileFormOf(o),
|
||||
profileStored: hasProfileOption(o),
|
||||
advanced: Object.entries(o)
|
||||
.filter(([k]) => !OWN_OPTION_KEYS.has(k))
|
||||
.map(([k, v]) => [k, String(v ?? '')]),
|
||||
@@ -973,10 +1005,12 @@ function toForm(source: any): SourceForm {
|
||||
}
|
||||
|
||||
function toBody(form: SourceForm, isNew: boolean, wasOracle = true): any {
|
||||
const options: Record<string, string | string[]> = {}
|
||||
const options: Record<string, unknown> = {}
|
||||
for (const [k, v] of form.advanced) if (k.trim() && v.trim()) options[k.trim()] = v.trim()
|
||||
if (form.schema.trim()) options.schema = form.schema.trim()
|
||||
if (form.maskColumns.length) options[MASK_KEY] = form.maskColumns
|
||||
const profile = profileOptionOf(form.profile, form.profileStored)
|
||||
if (profile) options[PROFILE_OPTION] = profile
|
||||
const oracle = form.kind === 'oracle'
|
||||
if (oracle) {
|
||||
// 只写选中的那一个,另一个不带:两处都有值时引擎听 service_name,
|
||||
@@ -1005,10 +1039,12 @@ function toBody(form: SourceForm, isNew: boolean, wasOracle = true): any {
|
||||
}
|
||||
|
||||
/**
|
||||
* 422 的中文 detail 说的是哪一项。眼下只有查询时限按字段拒(后端 query_timeout_problem),
|
||||
* 认不出的交回 toast
|
||||
* 422 的中文 detail 说的是哪一项:查询时限(后端 query_timeout_problem);数据剖析的某一项(profile_settings_problem,
|
||||
* 记为 profile.<键>,它的「单条查询时限」也带「查询时限」四个字,要先认)。认不出的交回 toast
|
||||
*/
|
||||
function fieldOfRejection(message: string): string | null {
|
||||
const profile = profileFieldOfRejection(message)
|
||||
if (profile) return `profile.${profile}`
|
||||
if (/查询时限/.test(message)) return 'query_timeout_s'
|
||||
return null
|
||||
}
|
||||
@@ -1019,10 +1055,11 @@ const FIELD_LABEL: Record<string, string> = {
|
||||
|
||||
/**
|
||||
* 决定连到哪个库的那几项(提交体里的)。schema 不在内:它只影响探查哪一片,不影响
|
||||
* 连不连得上,后端判断测连接结果还作不作数时同样不看它;遮罩列只管面板上怎么显示,也不在内
|
||||
* 连不连得上,后端判断测连接结果还作不作数时同样不看它;遮罩列只管面板上怎么显示,数据剖析的设置只管剖析,
|
||||
* 都不在内(服务端拼连接串时同样摘掉这几项)
|
||||
*/
|
||||
function connectionOf(body: any): string {
|
||||
const { schema: _schema, [MASK_KEY]: _mask, ...options } = body.options ?? {}
|
||||
const { schema: _schema, [MASK_KEY]: _mask, [PROFILE_OPTION]: _profile, ...options } = body.options ?? {}
|
||||
return JSON.stringify([
|
||||
body.kind, body.host ?? null, body.port ?? null, body.database ?? null, body.username ?? null,
|
||||
options, !!body.password,
|
||||
@@ -1076,6 +1113,12 @@ function SourceEditor({ source, kinds, onClose, onSaved }: {
|
||||
}
|
||||
const required = (key: string) => needs.includes(key)
|
||||
const blocked = missing.length > 0 || !!nameError
|
||||
// 剖析设置填错只拦保存,不拦测试连接(和连接无关)
|
||||
const profileBad = Object.keys(profileProblems(form.profile)).length
|
||||
const saveBlocked = blocked || profileBad > 0
|
||||
const saveHint = blocked ? `缺少必填项:${missing.join('、') || '标识格式'}` : profileBad ? PROFILE_TEXT.invalid(profileBad) : undefined
|
||||
const profileError = fieldError?.key.startsWith('profile.')
|
||||
? { key: fieldError.key.slice('profile.'.length) as ProfileErrorKey, message: fieldError.message } : null
|
||||
const body = toBody(form, isNew, source.kind === 'oracle')
|
||||
const sig = JSON.stringify({ ...body, password: form.password })
|
||||
const dirty = JSON.stringify(form) !== JSON.stringify(initial)
|
||||
@@ -1097,7 +1140,7 @@ function SourceEditor({ source, kinds, onClose, onSaved }: {
|
||||
}
|
||||
|
||||
const save = async () => {
|
||||
if (blocked) return
|
||||
if (saveBlocked) return
|
||||
setSaving(true)
|
||||
setFieldError(null)
|
||||
try {
|
||||
@@ -1109,7 +1152,10 @@ function SourceEditor({ source, kinds, onClose, onSaved }: {
|
||||
} catch (e) {
|
||||
const rejected = e instanceof ApiError && e.status === 422 ? e.message : ''
|
||||
const key = rejected ? fieldOfRejection(rejected) : null
|
||||
if (key) {
|
||||
if (key?.startsWith('profile.')) {
|
||||
setFieldError({ key, message: rejected })
|
||||
focusProfileField('ds-profile', key.slice('profile.'.length) as ProfileErrorKey)
|
||||
} else if (key) {
|
||||
setFieldError({ key, message: rejected })
|
||||
setAdvOpen(true)
|
||||
requestAnimationFrame(() => document.getElementById(`ds-adv-${key}`)?.focus())
|
||||
@@ -1200,8 +1246,8 @@ function SourceEditor({ source, kinds, onClose, onSaved }: {
|
||||
title={blocked ? `缺少必填项:${missing.join('、') || '标识格式'}` : '使用当前配置测试连接,不保存'}>
|
||||
{test.since ? <Spinner size={11} /> : <Plug size={12} aria-hidden />} 测试连接
|
||||
</button>
|
||||
<button className="btn btn-primary" disabled={saving || blocked} onClick={() => void save()}
|
||||
title={blocked ? `缺少必填项:${missing.join('、') || '标识格式'}` : undefined}>
|
||||
<button className="btn btn-primary" disabled={saving || saveBlocked} onClick={() => void save()}
|
||||
title={saveHint} data-source-save="">
|
||||
{saving ? <Spinner size={11} /> : null} 保存
|
||||
</button>
|
||||
</>
|
||||
@@ -1318,6 +1364,12 @@ function SourceEditor({ source, kinds, onClose, onSaved }: {
|
||||
<MaskColumnsField sourceId={source.id} synced={!!source.table_count} value={form.maskColumns}
|
||||
onChange={(maskColumns) => set({ maskColumns })} />
|
||||
|
||||
<ProfileSettingsSection idPrefix="ds-profile" form={form.profile} serverError={profileError}
|
||||
onChange={(profile) => {
|
||||
if (profileError) setFieldError(null)
|
||||
set({ profile })
|
||||
}} />
|
||||
|
||||
<label className="flex cursor-pointer items-start gap-2 rounded-lg border p-2.5"
|
||||
style={form.readonly ? undefined : { borderColor: 'var(--warn)', background: 'color-mix(in srgb, var(--warn) 7%, transparent)' }}>
|
||||
<input type="checkbox" className="mt-0.5" checked={form.readonly}
|
||||
|
||||
@@ -14,7 +14,7 @@ import { LineageLayer } from '../canvas/LineageLayer'
|
||||
import { VersionsSheet } from '../canvas/VersionsSheet'
|
||||
import { PublishDialog } from '../canvas/PublishDialog'
|
||||
import { WorkflowPicker, confirmDiscard, createWorkflow, takeDiscarded } from '../canvas/WorkflowPicker'
|
||||
import { problemsOf, type Problem } from '../canvas/issues'
|
||||
import { fieldOfIssue, problemsOf, type Problem } from '../canvas/issues'
|
||||
import { STUDIO_SHORTCUTS, hintOf, studioShortcut, type StudioShortcutId } from '../canvas/shortcuts'
|
||||
import { AssistantPanel } from '../run/AssistantPanel'
|
||||
import { copilotProgress } from '../run/Composer'
|
||||
@@ -620,7 +620,12 @@ export function StudioPage() {
|
||||
{publishing && workflow && (
|
||||
<PublishDialog workflow={workflow} onClose={() => setPublishing(false)}
|
||||
onDone={() => { setPublishing(false); void refresh() }}
|
||||
onLocate={(id) => { select(id); focusNode(id) }} />
|
||||
onLocate={(issue) => {
|
||||
// 和问题面板同一个落点:选中节点、取景,问题落在某一项配置上(调用工具的 SQL)就翻到那一项
|
||||
if (!issue.node_id) return
|
||||
const node = useStudio.getState().nodes.find((n) => n.id === issue.node_id)
|
||||
revealField(issue.node_id, fieldOfIssue(issue, node))
|
||||
}} />
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
|
||||
@@ -1,15 +1,19 @@
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from 'react'
|
||||
import type { ReactNode } from 'react'
|
||||
import { useNavigate, useParams } from 'react-router-dom'
|
||||
import { ArrowLeft, BookMarked, Database, ListFilter as ListFilterIcon, Sparkles } from 'lucide-react'
|
||||
import { ArrowLeft, BookMarked, Database, ListFilter as ListFilterIcon, ScanSearch, Sparkles } from 'lucide-react'
|
||||
import clsx from 'clsx'
|
||||
import { ApiError, api } from '../../api/client'
|
||||
import type { CatalogDetail, CatalogList, CatalogTableRow } from '../../types'
|
||||
import { EmptyState, ErrorState, PageHeader, Skeleton, toast } from '../../components/ui'
|
||||
import { useDatasources, useOnReconnect } from '../../store/catalog'
|
||||
import { CATALOG_TEXT as CT } from '../../lib/terms'
|
||||
import { EmptyState, ErrorState, PageHeader, Skeleton, Spinner, toast, useTicker } from '../../components/ui'
|
||||
import { useCatalog, useDatasources, useOnReconnect } from '../../store/catalog'
|
||||
import { CATALOG_TEXT as CT, PROFILE_STOP_LABEL, PROFILE_TEXT as PT } from '../../lib/terms'
|
||||
import { StatusLegend, SystemNotesNotice } from './parts'
|
||||
import { DraftDialog } from './DraftDialog'
|
||||
import { ProfileDialog } from './ProfileDialog'
|
||||
import type { ProfileJob } from './ProfileDialog'
|
||||
import { ProfileSettingsDialog } from './ProfileSettings'
|
||||
import { profileBlockOf, profileSettingsOf, secondsText } from './profile'
|
||||
import { TableDetail } from './TableDetail'
|
||||
import { TableIndex } from './TableIndex'
|
||||
import { filterCounts, progressOf, rowFromDetail, usageSummary, visibleRows } from './model'
|
||||
@@ -65,6 +69,24 @@ export function CatalogPage() {
|
||||
const [drafting, setDrafting] = useState(false)
|
||||
/** 起草写进了这些表:详情据此重新载入(编辑中的不动,保存时服务端会用版本号拦下) */
|
||||
const [drafted, setDrafted] = useState<{ seq: number; tables: Set<string> }>({ seq: 0, tables: new Set() })
|
||||
const [profiling, setProfiling] = useState(false)
|
||||
const [profileSettingsOpen, setProfileSettingsOpen] = useState(false)
|
||||
/** 一次剖析的状态。关掉弹窗剖析照常进行:状态留在页面上,页头有入口,完成后提示并可再打开报告 */
|
||||
const [job, setJob] = useState<(ProfileJob & { sourceId: string }) | null>(null)
|
||||
/** 剖析设置刚存过:数据源列表重取回来之前按存下的这份算(预算、开没开) */
|
||||
const [savedOptions, setSavedOptions] = useState<{ sourceId: string; options: Record<string, unknown> } | null>(null)
|
||||
/** 从剖析报告点「填写含义」:打开那张表并弹出那一列的码值 */
|
||||
const [fillCodes, setFillCodes] = useState<{ table: string; column: string; seq: number } | null>(null)
|
||||
const profilingRef = useRef(profiling)
|
||||
profilingRef.current = profiling
|
||||
const alive = useRef(true)
|
||||
// 开发模式的严格模式会先卸载再挂载一次:挂载时要重新置为真
|
||||
useEffect(() => {
|
||||
alive.current = true
|
||||
return () => { alive.current = false }
|
||||
}, [])
|
||||
const sourceRef = useRef(sourceId)
|
||||
sourceRef.current = sourceId
|
||||
|
||||
const rows = useMemo(() => data?.tables ?? [], [data])
|
||||
const visible = useMemo(() => visibleRows(rows, query, filter, sort), [rows, query, filter, sort])
|
||||
@@ -87,6 +109,77 @@ export function CatalogPage() {
|
||||
void load()
|
||||
}, [load])
|
||||
|
||||
const options = savedOptions?.sourceId === sourceId ? savedOptions.options : source?.options
|
||||
const profileSettings = useMemo(() => profileSettingsOf(options), [options])
|
||||
const myJob = job?.sourceId === sourceId ? job : null
|
||||
const jobRunning = myJob?.phase === 'running'
|
||||
useTicker(1000, jobRunning)
|
||||
|
||||
/**
|
||||
* 剖析:同步请求,做完才返回。期间关掉弹窗不影响;回来时页面已经换了数据源或卸载了,只提示、不再改这一页。
|
||||
* 写进去的表重新载入清单和正在看的详情(编辑中的不动,同起草)
|
||||
*/
|
||||
const startProfile = useCallback(async (tables: string[] | null) => {
|
||||
const src = sourceId
|
||||
const startedAt = Date.now()
|
||||
setJob({ sourceId: src, phase: 'running', startedAt, tables, settings: profileSettings })
|
||||
let next: ProfileJob
|
||||
try {
|
||||
const report = await api.dataCatalog.profile(src, tables ? { tables } : {})
|
||||
next = { phase: 'done', report, ms: Date.now() - startedAt }
|
||||
} catch (e) {
|
||||
next = e instanceof ApiError && e.status === 409
|
||||
? { phase: 'blocked', kind: profileBlockOf(e.message), message: e.message, tables }
|
||||
: { phase: 'failed', error: e, tables }
|
||||
}
|
||||
const here = alive.current && sourceRef.current === src
|
||||
if (here) setJob({ ...next, sourceId: src })
|
||||
if (next.phase === 'done') {
|
||||
const r = next.report
|
||||
const touched = r.tables.filter((t) => !t.error && t.added + t.updated + t.removed > 0).map((t) => t.table_name)
|
||||
if (here) {
|
||||
if (touched.length) setDrafted((cur) => ({ seq: cur.seq + 1, tables: new Set(touched) }))
|
||||
void load()
|
||||
}
|
||||
// 弹窗开着就看报告;关了(在后台继续)才提示,并给「查看报告」
|
||||
if (!here || !profilingRef.current) {
|
||||
const summary = PT.summary(r.queries_used, r.settings.max_queries, r.tables.length)
|
||||
const text = r.stopped ? PT.toastStopped(PROFILE_STOP_LABEL[r.stopped] ?? r.stopped) : PT.toastDone(summary)
|
||||
const show = r.stopped ? toast.warn : toast.ok
|
||||
show(text, here ? { key: `profile:${src}`, duration: 10000, action: { label: PT.viewReport, onClick: () => setProfiling(true) } }
|
||||
: { key: `profile:${src}` })
|
||||
}
|
||||
} else if (!here || !profilingRef.current) {
|
||||
toast.error(next.phase === 'blocked' ? next.message : next.error, here
|
||||
? { key: `profile:${src}`, action: { label: PT.viewReport, onClick: () => setProfiling(true) } } : { key: `profile:${src}` })
|
||||
}
|
||||
}, [sourceId, profileSettings, load])
|
||||
|
||||
const openProfile = useCallback(() => {
|
||||
// 上一次的结果(报告、被拒)看过了,再点就是重新开始;正在剖析时打开的是进度
|
||||
setJob((cur) => (cur && cur.phase === 'running' && cur.sourceId === sourceId ? cur : null))
|
||||
setProfiling(true)
|
||||
}, [sourceId])
|
||||
|
||||
const onProfileSettingsSaved = useCallback((row: any) => {
|
||||
setProfileSettingsOpen(false)
|
||||
setSavedOptions({ sourceId, options: row?.options ?? {} })
|
||||
void useCatalog.getState().reload('datasources')
|
||||
// 因为没开启被拒的:开了就回到选范围,可以直接开始
|
||||
setJob((cur) => (cur && cur.phase === 'blocked' && cur.kind === 'disabled' ? null : cur))
|
||||
}, [sourceId])
|
||||
|
||||
const openFromReport = useCallback((t: string) => {
|
||||
setProfiling(false)
|
||||
openTable(t)
|
||||
}, [openTable])
|
||||
|
||||
const fillFromReport = useCallback((t: string, column: string) => {
|
||||
setProfiling(false)
|
||||
setFillCodes((cur) => ({ table: t, column, seq: (cur?.seq ?? 0) + 1 }))
|
||||
openTable(t)
|
||||
}, [openTable])
|
||||
|
||||
// 上一张 / 下一张按清单当前的筛选和排序走;正在看的表不在筛选结果里(刚确认完、从关联关系跳过来)时按全部表走
|
||||
const [prev, next] = useMemo(() => {
|
||||
if (!table) return [null, null]
|
||||
@@ -116,6 +209,19 @@ export function CatalogPage() {
|
||||
<Sparkles size={11} aria-hidden /> {CT.draft}
|
||||
</button>
|
||||
)
|
||||
// 页头的「数据剖析」:剖析进行中换成计时,点开看进度(关掉弹窗剖析照常进行)
|
||||
const headerProfile = jobRunning && myJob?.phase === 'running'
|
||||
? (
|
||||
<button type="button" className="btn btn-sm" onClick={() => setProfiling(true)} title={PT.headerRunningHint}
|
||||
aria-label={PT.headerRunning(secondsText(Date.now() - myJob.startedAt))} data-catalog-profile="running">
|
||||
<Spinner size={11} /> <span className="tnum hidden sm:inline">{PT.headerRunning(secondsText(Date.now() - myJob.startedAt))}</span>
|
||||
</button>
|
||||
)
|
||||
: (
|
||||
<button type="button" className="btn btn-sm" onClick={openProfile} aria-label={PT.action} title={PT.actionHint} data-catalog-profile="">
|
||||
<ScanSearch size={11} aria-hidden /> <span className="hidden sm:inline">{PT.action}</span>
|
||||
</button>
|
||||
)
|
||||
// 页头的「起草」窄屏只留图标,标题才放得下
|
||||
const headerDraft = (
|
||||
<button type="button" className="btn btn-sm" onClick={() => setDrafting(true)} aria-label={CT.draft} title={CT.draft} data-catalog-draft="">
|
||||
@@ -171,12 +277,13 @@ export function CatalogPage() {
|
||||
active={table}
|
||||
onOpen={openTable}
|
||||
onDraft={() => setDrafting(true)}
|
||||
onProfile={openProfile}
|
||||
/>
|
||||
</aside>
|
||||
<section className={clsx('min-h-0 min-w-0 flex-1 flex-col lg:flex', table ? 'flex' : 'hidden')} aria-label={table ?? CT.overviewTitle}>
|
||||
{table
|
||||
? (
|
||||
<TableDetail sourceId={sourceId} table={table} rows={rows} drafted={drafted} prev={prev} next={next}
|
||||
<TableDetail sourceId={sourceId} table={table} rows={rows} drafted={drafted} fillCodes={fillCodes} prev={prev} next={next}
|
||||
onOpen={openTable} onBack={() => navigate(base)} onDetail={onDetail} />
|
||||
)
|
||||
: <Overview rows={rows} empty={empty} systemNotes={data.system_notes} draft={draftButton(true)} onOpen={openTable} />}
|
||||
@@ -193,6 +300,7 @@ export function CatalogPage() {
|
||||
subtitle={CT.subtitle}
|
||||
actions={(
|
||||
<div className="flex shrink-0 items-center gap-1.5">
|
||||
{hasTables && headerProfile}
|
||||
{hasTables && headerDraft}
|
||||
<button type="button" className="btn btn-sm btn-ghost" onClick={() => navigate(back)} aria-label={CT.back} data-catalog-back="">
|
||||
<ArrowLeft size={12} aria-hidden /> <span className="hidden sm:inline">{CT.back}</span>
|
||||
@@ -213,6 +321,28 @@ export function CatalogPage() {
|
||||
onDrafted={onDrafted}
|
||||
/>
|
||||
)}
|
||||
{profiling && data && (
|
||||
<ProfileDialog
|
||||
sourceName={source?.name ?? sourceId}
|
||||
settings={profileSettings}
|
||||
rows={rows}
|
||||
visible={visible}
|
||||
selected={[...selected]}
|
||||
filtered={filtered}
|
||||
job={myJob}
|
||||
onStart={(tables) => void startProfile(tables)}
|
||||
onClose={() => setProfiling(false)}
|
||||
onSettings={() => setProfileSettingsOpen(true)}
|
||||
onGoSource={() => navigate(back)}
|
||||
onOpenTable={openFromReport}
|
||||
onFillCodes={fillFromReport}
|
||||
onReset={() => setJob(null)}
|
||||
/>
|
||||
)}
|
||||
{profileSettingsOpen && source && (
|
||||
<ProfileSettingsDialog row={{ id: source.id, name: source.name, options }} onClose={() => setProfileSettingsOpen(false)}
|
||||
onSaved={onProfileSettingsSaved} />
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
import { useState } from 'react'
|
||||
import type { CatalogItem } from '../../types'
|
||||
import { isComposing, Modal, Spinner } from '../../components/ui'
|
||||
import { CODES_TEXT as KT } from '../../lib/terms'
|
||||
import { StatusChip } from './parts'
|
||||
|
||||
// ===========================================================================
|
||||
// 填写码值的含义。数据剖析只知道列里出现过哪些取值(码值候选的含义是空串),含义要人填:逐个码值一个输入框,
|
||||
// 比在编辑模式里改「码值=含义」的多行文字省事。保存走整份提交,这一项记为人工填写、已确认;
|
||||
// 暂不清楚的含义可以留空,表格里照旧显示「含义待填写」。
|
||||
// ===========================================================================
|
||||
|
||||
export function CodesDialog({ column, item, saving, onSave, onClose }: {
|
||||
column: string
|
||||
item: CatalogItem<Record<string, string>>
|
||||
saving: boolean
|
||||
/** 新的码值对照(码值不变,只改含义) */
|
||||
onSave: (value: Record<string, string>) => void
|
||||
onClose: () => void
|
||||
}) {
|
||||
const [initial] = useState(() => ({ ...item.value }))
|
||||
const [value, setValue] = useState<Record<string, string>>(initial)
|
||||
const codes = Object.keys(initial)
|
||||
const dirty = codes.some((k) => (value[k] ?? '').trim() !== (initial[k] ?? '').trim())
|
||||
const save = () => {
|
||||
if (!dirty || saving) return
|
||||
onSave(Object.fromEntries(codes.map((k) => [k, (value[k] ?? '').trim()])))
|
||||
}
|
||||
return (
|
||||
<Modal open onClose={onClose} dirty={dirty} width={520} title={KT.title(column)}
|
||||
footer={(
|
||||
<>
|
||||
<button type="button" className="btn" onClick={onClose}>{KT.cancel}</button>
|
||||
<button type="button" className="btn btn-primary" disabled={!dirty || saving} onClick={save} data-codes-save="">
|
||||
{saving ? <Spinner size={11} /> : null} {KT.save}
|
||||
</button>
|
||||
</>
|
||||
)}>
|
||||
<div className="space-y-3" data-codes-dialog={column}>
|
||||
<div className="flex flex-wrap items-center gap-1.5 text-2xs text-faint">
|
||||
<StatusChip status={item.status} source={item.source} />
|
||||
</div>
|
||||
{item.note && (
|
||||
<div className="rounded border bg-bg px-2 py-1.5 text-2xs leading-relaxed text-dim">
|
||||
<span className="text-faint">{KT.note}:</span>{item.note}
|
||||
</div>
|
||||
)}
|
||||
<div className="overflow-hidden rounded-lg border">
|
||||
<table className="w-full border-collapse text-xs">
|
||||
<thead>
|
||||
<tr className="border-b bg-elev text-left text-2xs text-faint">
|
||||
<th scope="col" className="w-[32%] px-3 py-1.5 font-medium">{KT.head.code}</th>
|
||||
<th scope="col" className="px-2 py-1.5 font-medium">{KT.head.meaning}</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{codes.map((k, i) => (
|
||||
<tr key={k} className="border-b border-hairline last:border-b-0">
|
||||
<th scope="row" className="mono px-3 py-1 text-left font-normal [overflow-wrap:anywhere]">
|
||||
<label htmlFor={`codes-${column}-${i}`}>{k}</label>
|
||||
</th>
|
||||
<td className="px-2 py-1">
|
||||
<input id={`codes-${column}-${i}`} className="field !py-1" value={value[k] ?? ''} placeholder={KT.placeholder}
|
||||
maxLength={500} autoFocus={i === codes.findIndex((c) => !(initial[c] ?? '').trim())}
|
||||
onChange={(e) => setValue((v) => ({ ...v, [k]: e.target.value }))}
|
||||
// 回车保存;输入法组字时的回车是选词,不算
|
||||
onKeyDown={(e) => { if (e.key === 'Enter' && !isComposing(e)) { e.preventDefault(); save() } }}
|
||||
data-code={k} />
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<p className="text-2xs leading-relaxed text-faint">{KT.hint}</p>
|
||||
</div>
|
||||
</Modal>
|
||||
)
|
||||
}
|
||||
@@ -1,13 +1,13 @@
|
||||
import { memo } from 'react'
|
||||
import type { ReactNode } from 'react'
|
||||
import { AlertTriangle, CheckCheck, Lock } from 'lucide-react'
|
||||
import { AlertTriangle, CheckCheck, Lock, PencilLine } from 'lucide-react'
|
||||
import clsx from 'clsx'
|
||||
import type {
|
||||
CatalogColumnNotes, CatalogItem, CatalogMeasure, CatalogReviewAction, CatalogStructureColumn,
|
||||
} from '../../types'
|
||||
import { IconButton } from '../../components/ui'
|
||||
import {
|
||||
CATALOG_COLUMN_FIELD_LABEL, CATALOG_MEASURE_HINT, CATALOG_MEASURE_LABEL, CATALOG_TEXT as CT,
|
||||
CATALOG_COLUMN_FIELD_LABEL, CATALOG_MEASURE_HINT, CATALOG_MEASURE_LABEL, CATALOG_TEXT as CT, CODES_TEXT as KT,
|
||||
} from '../../lib/terms'
|
||||
import { ItemMark } from './parts'
|
||||
import type { ReviewTarget } from './parts'
|
||||
@@ -32,7 +32,7 @@ export interface ColumnModel {
|
||||
|
||||
type ColumnTexts = Partial<Record<ColumnField, string>>
|
||||
|
||||
export function ColumnTable({ columns, form, initial, problems, busy, systemNotes, onReview, onConfirmColumn, onChange }: {
|
||||
export function ColumnTable({ columns, form, initial, problems, busy, systemNotes, onReview, onConfirmColumn, onFillCodes, onChange }: {
|
||||
columns: ColumnModel[]
|
||||
/** 编辑中:列名 → 字段 → 文字(form 是当前的,initial 是进入编辑时的);不在编辑时为 null */
|
||||
form: Record<string, ColumnTexts> | null
|
||||
@@ -44,6 +44,8 @@ export function ColumnTable({ columns, form, initial, problems, busy, systemNote
|
||||
systemNotes: boolean
|
||||
onReview: (target: ReviewTarget, action: CatalogReviewAction) => void
|
||||
onConfirmColumn: (col: string) => void
|
||||
/** 码值里有含义空着的:打开逐个填写含义的弹窗 */
|
||||
onFillCodes: (col: string) => void
|
||||
onChange: (col: string, field: ColumnField, value: string) => void
|
||||
}) {
|
||||
return (
|
||||
@@ -66,7 +68,7 @@ export function ColumnTable({ columns, form, initial, problems, busy, systemNote
|
||||
<tbody>
|
||||
{columns.map((c) => (
|
||||
<ColumnRow key={c.name} col={c} editing={!!form} draft={form?.[c.name]} initial={initial?.[c.name]} problems={problems} busy={busy}
|
||||
systemNotes={systemNotes} onReview={onReview} onConfirmColumn={onConfirmColumn} onChange={onChange} />
|
||||
systemNotes={systemNotes} onReview={onReview} onConfirmColumn={onConfirmColumn} onFillCodes={onFillCodes} onChange={onChange} />
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
@@ -74,7 +76,7 @@ export function ColumnTable({ columns, form, initial, problems, busy, systemNote
|
||||
)
|
||||
}
|
||||
|
||||
const ColumnRow = memo(function ColumnRow({ col, editing, draft, initial, problems, busy, systemNotes, onReview, onConfirmColumn, onChange }: {
|
||||
const ColumnRow = memo(function ColumnRow({ col, editing, draft, initial, problems, busy, systemNotes, onReview, onConfirmColumn, onFillCodes, onChange }: {
|
||||
col: ColumnModel
|
||||
editing: boolean
|
||||
/** 这一列的表单。只有改了这一列时引用才会变,别的行不重画 */
|
||||
@@ -85,6 +87,7 @@ const ColumnRow = memo(function ColumnRow({ col, editing, draft, initial, proble
|
||||
systemNotes: boolean
|
||||
onReview: (target: ReviewTarget, action: CatalogReviewAction) => void
|
||||
onConfirmColumn: (col: string) => void
|
||||
onFillCodes: (col: string) => void
|
||||
onChange: (col: string, field: ColumnField, value: string) => void
|
||||
}) {
|
||||
const { name, structure, items } = col
|
||||
@@ -166,7 +169,10 @@ const ColumnRow = memo(function ColumnRow({ col, editing, draft, initial, proble
|
||||
)}
|
||||
{cell(
|
||||
'codes',
|
||||
<Value item={codes} render={(v) => <Codes value={v as Record<string, string>} />} />,
|
||||
<Value item={codes} render={(v) => (
|
||||
<Codes value={v as Record<string, string>} label={KT.fillLabel(name)}
|
||||
onFill={codes?.status !== 'rejected' && !busy ? () => onFillCodes(name) : undefined} />
|
||||
)} />,
|
||||
<textarea className="field mono !min-h-0 !px-1.5 !py-1 !text-2xs" rows={Math.min(4, Math.max(1, (draft?.codes ?? '').split('\n').length))}
|
||||
value={draft?.codes ?? ''}
|
||||
placeholder={codes?.status === 'rejected' ? CT.rejectedPlaceholder(codesInline(codes.value)) : CT.codesPlaceholder}
|
||||
@@ -183,7 +189,8 @@ const ColumnRow = memo(function ColumnRow({ col, editing, draft, initial, proble
|
||||
</tr>
|
||||
)
|
||||
}, (a, b) => a.col === b.col && a.editing === b.editing && a.draft === b.draft && a.initial === b.initial && a.busy === b.busy
|
||||
&& a.systemNotes === b.systemNotes && a.onReview === b.onReview && a.onConfirmColumn === b.onConfirmColumn && a.onChange === b.onChange
|
||||
&& a.systemNotes === b.systemNotes && a.onReview === b.onReview && a.onConfirmColumn === b.onConfirmColumn && a.onFillCodes === b.onFillCodes
|
||||
&& a.onChange === b.onChange
|
||||
// 格式问题每次都是新对象:只比这一列的几格
|
||||
&& COLUMN_FIELDS.every((f) => a.problems[`c:${a.col.name}:${f}`] === b.problems[`c:${b.col.name}:${f}`]))
|
||||
|
||||
@@ -195,17 +202,33 @@ export function Value<T>({ item, render }: { item: CatalogItem<T> | undefined; r
|
||||
return <span className={clsx(item.status === 'rejected' && 'text-faint line-through')}>{render(item.value)}</span>
|
||||
}
|
||||
|
||||
function Codes({ value }: { value: Record<string, string> }) {
|
||||
/** 含义还空着的码值个数(数据剖析给的码值候选只有取值,含义等人填) */
|
||||
export const pendingCodes = (v: Record<string, string> | undefined): number =>
|
||||
Object.values(v ?? {}).filter((x) => !String(x ?? '').trim()).length
|
||||
|
||||
function Codes({ value, label, onFill }: { value: Record<string, string>; label: string; onFill?: () => void }) {
|
||||
const entries = Object.entries(value)
|
||||
const pending = pendingCodes(value)
|
||||
return (
|
||||
<span className="flex flex-wrap gap-1" title={entries.map(([k, v]) => `${k}=${v}`).join('\n')}>
|
||||
{entries.slice(0, CODES_SHOWN).map(([k, v]) => (
|
||||
<span key={k} className="inline-flex max-w-full items-center gap-1 rounded border bg-bg px-1 text-2xs">
|
||||
<span className="mono text-dim">{k}</span>
|
||||
<span className="truncate">{v}</span>
|
||||
</span>
|
||||
))}
|
||||
{entries.length > CODES_SHOWN && <span className="text-2xs text-faint">+{entries.length - CODES_SHOWN}</span>}
|
||||
<span className="block">
|
||||
<span className="flex flex-wrap gap-1" title={entries.map(([k, v]) => `${k}=${v.trim() || KT.pending}`).join('\n')}>
|
||||
{entries.slice(0, CODES_SHOWN).map(([k, v]) => (
|
||||
<span key={k} className="inline-flex max-w-full items-center gap-1 rounded border bg-bg px-1 text-2xs" data-code={k}>
|
||||
<span className="mono text-dim">{k}</span>
|
||||
{v.trim()
|
||||
? <span className="truncate">{v}</span>
|
||||
: <span className="truncate" style={{ color: 'var(--st-waiting)' }} data-code-pending="">{KT.pending}</span>}
|
||||
</span>
|
||||
))}
|
||||
{entries.length > CODES_SHOWN && <span className="text-2xs text-faint">+{entries.length - CODES_SHOWN}</span>}
|
||||
</span>
|
||||
{pending > 0 && onFill && (
|
||||
// 各码值的「含义待填写」已经写在上面,这里只给入口;几个待填写写在悬停里
|
||||
<button type="button" className="mt-1 inline-flex items-center gap-1 whitespace-nowrap rounded px-1 text-2xs text-dim underline decoration-dotted underline-offset-2 hover:text-fg"
|
||||
onClick={onFill} aria-label={label} title={KT.pendingN(pending)} data-codes-fill={pending}>
|
||||
<PencilLine size={10} aria-hidden /> {KT.fill}
|
||||
</button>
|
||||
)}
|
||||
</span>
|
||||
)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,450 @@
|
||||
import { useMemo, useState } from 'react'
|
||||
import type { ReactNode } from 'react'
|
||||
import { ArrowUpRight, CircleCheck, OctagonX, ScanSearch, ShieldAlert, TriangleAlert } from 'lucide-react'
|
||||
import clsx from 'clsx'
|
||||
import type {
|
||||
CatalogProfileCodesFinding, CatalogProfileDateFinding, CatalogProfileOut, CatalogProfileRelationFinding, CatalogProfileSettings,
|
||||
CatalogProfileSize, CatalogProfileSkip, CatalogProfileTable, CatalogTableRow,
|
||||
} from '../../types'
|
||||
import { ErrorState, Modal, Notice, Spinner, useTicker } from '../../components/ui'
|
||||
import {
|
||||
CATALOG_CARDINALITY_LABEL, CATALOG_TEXT as CT, PROFILE_SKIP_KIND_LABEL, PROFILE_SKIP_REASON_LABEL, PROFILE_STOP_LABEL, PROFILE_STOP_NEXT, PROFILE_TEXT as PT,
|
||||
} from '../../lib/terms'
|
||||
import { STATUS_TONE, StatusChip } from './parts'
|
||||
import { PROFILE_DEFAULT_TABLES, PROFILE_MAX_TABLES, coverageText, secondsText } from './profile'
|
||||
import type { ProfileBlock } from './profile'
|
||||
|
||||
// ===========================================================================
|
||||
// 数据目录页的「数据剖析」。接口是同步的:一次请求做完整个剖析才返回,服务端不支持中途停止,所以这里不给「停止」,
|
||||
// 进行中只写已用时长和总时长上限,并明说无法中途停止。关掉窗口剖析照常进行(状态在页面上,页头有进度入口),
|
||||
// 完成后弹提示、可以再打开报告。
|
||||
//
|
||||
// 四段:选范围、看预算、确认开始 → 进行中 → 报告(用了几条查询、为什么停、每张表的发现和跳过的项);
|
||||
// 开始不了(409)时写明是哪一种情况和下一步,未开启的就地打开剖析设置。
|
||||
// ===========================================================================
|
||||
|
||||
/** 一次剖析的状态。放在页面上而不是弹窗里:关掉弹窗剖析照常进行,完成后还能再打开报告 */
|
||||
export type ProfileJob =
|
||||
| { phase: 'running'; startedAt: number; tables: string[] | null; settings: CatalogProfileSettings }
|
||||
| { phase: 'done'; report: CatalogProfileOut; ms: number }
|
||||
| { phase: 'blocked'; kind: ProfileBlock; message: string; tables: string[] | null }
|
||||
| { phase: 'failed'; error: unknown; tables: string[] | null }
|
||||
|
||||
type Scope = 'selected' | 'filtered' | 'default'
|
||||
|
||||
/** 报告里每类最多逐条列几项,其余合成一句 */
|
||||
const SKIP_SHOWN = 6
|
||||
const CODES_SHOWN = 12
|
||||
|
||||
export function ProfileDialog({
|
||||
sourceName, settings, rows, visible, selected, filtered, job, onStart, onClose, onSettings, onGoSource, onOpenTable, onFillCodes, onReset,
|
||||
}: {
|
||||
sourceName: string
|
||||
/** 这个数据源此刻的剖析设置(数据源列表里读的) */
|
||||
settings: CatalogProfileSettings
|
||||
rows: CatalogTableRow[]
|
||||
visible: CatalogTableRow[]
|
||||
selected: string[]
|
||||
filtered: boolean
|
||||
/** null:还没开始(选范围) */
|
||||
job: ProfileJob | null
|
||||
/** tables 为 null 时由服务端挑表 */
|
||||
onStart: (tables: string[] | null) => void
|
||||
onClose: () => void
|
||||
/** 打开这个数据源的剖析设置 */
|
||||
onSettings: () => void
|
||||
/** 去数据源卡片(停用、没有表结构、快照被改动时) */
|
||||
onGoSource: () => void
|
||||
onOpenTable: (table: string) => void
|
||||
onFillCodes: (table: string, column: string) => void
|
||||
/** 回到选范围 */
|
||||
onReset: () => void
|
||||
}) {
|
||||
const live = useMemo(() => rows.filter((r) => r.in_schema).map((r) => r.table_name), [rows])
|
||||
const scopes = useMemo(() => {
|
||||
const liveSet = new Set(live)
|
||||
const out: { key: Scope; label: string; hint?: string; tables: string[] | null }[] = []
|
||||
const sel = selected.filter((t) => liveSet.has(t))
|
||||
if (sel.length) out.push({ key: 'selected', label: PT.scopeSelected(sel.length), tables: sel })
|
||||
const vis = visible.filter((r) => r.in_schema).map((r) => r.table_name)
|
||||
if (filtered && vis.length && vis.length !== live.length) out.push({ key: 'filtered', label: PT.scopeFiltered(vis.length), tables: vis })
|
||||
out.push({ key: 'default', label: PT.scopeDefault, hint: PT.scopeDefaultHint(PROFILE_DEFAULT_TABLES), tables: null })
|
||||
return out
|
||||
}, [live, selected, visible, filtered])
|
||||
const [scope, setScope] = useState<Scope>(() => scopes[0].key)
|
||||
const picked = scopes.find((s) => s.key === scope) ?? scopes[0]
|
||||
const tooMany = (picked.tables?.length ?? 0) > PROFILE_MAX_TABLES
|
||||
const running = job?.phase === 'running'
|
||||
useTicker(1000, running)
|
||||
|
||||
let body: ReactNode
|
||||
let footer: ReactNode
|
||||
if (!job) {
|
||||
body = (
|
||||
<>
|
||||
<p className="text-xs leading-relaxed text-dim">{PT.intro}</p>
|
||||
{!settings.enabled && (
|
||||
<Notice tone="warn" attr={{ 'data-profile-disabled': '' }}>
|
||||
<p className="font-medium">{PT.disabledTitle}</p>
|
||||
<p className="mt-0.5 text-dim">{PT.disabledBody}</p>
|
||||
<button type="button" className="btn btn-sm mt-2" onClick={onSettings} data-profile-enable="">{PT.enable}</button>
|
||||
</Notice>
|
||||
)}
|
||||
<fieldset>
|
||||
<legend className="label">{PT.scope}</legend>
|
||||
<div className="space-y-1.5" role="radiogroup" aria-label={PT.scope}>
|
||||
{scopes.map((s) => {
|
||||
const over = (s.tables?.length ?? 0) > PROFILE_MAX_TABLES
|
||||
return (
|
||||
<label key={s.key} className={clsx('flex cursor-pointer items-start gap-2 rounded-md border px-3 py-2 text-xs',
|
||||
scope === s.key ? 'border-[var(--accent)] bg-accent-soft' : 'hover:bg-hover')} data-profile-scope={s.key}>
|
||||
<input type="radio" className="mt-0.5" name="catalog-profile-scope" checked={scope === s.key} onChange={() => setScope(s.key)} />
|
||||
<span className="min-w-0">
|
||||
<span>{s.label}</span>
|
||||
{(s.hint || over) && (
|
||||
<span className={clsx('mt-0.5 block text-2xs', over ? 'text-[var(--err)]' : 'text-faint')}>
|
||||
{over ? PT.scopeTooMany(PROFILE_MAX_TABLES) : s.hint}
|
||||
</span>
|
||||
)}
|
||||
</span>
|
||||
</label>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
</fieldset>
|
||||
{settings.enabled && <Budget settings={settings} sourceName={sourceName} onSettings={onSettings} />}
|
||||
<p className="flex items-start gap-1.5 text-2xs leading-relaxed text-faint">
|
||||
<ShieldAlert size={11} className="mt-px shrink-0" aria-hidden /> {PT.noCancel}
|
||||
</p>
|
||||
</>
|
||||
)
|
||||
footer = (
|
||||
<>
|
||||
<button type="button" className="btn" onClick={onClose}>{CT.cancel}</button>
|
||||
<button type="button" className="btn btn-primary" disabled={!settings.enabled || tooMany} onClick={() => onStart(picked.tables)}
|
||||
data-profile-start="">
|
||||
<ScanSearch size={12} aria-hidden /> {PT.start}
|
||||
</button>
|
||||
</>
|
||||
)
|
||||
} else if (job.phase === 'running') {
|
||||
const used = Date.now() - job.startedAt
|
||||
const max = job.settings.max_total_s * 1000
|
||||
body = (
|
||||
<div className="space-y-3" aria-live="polite">
|
||||
<div className="flex items-center gap-2 text-xs">
|
||||
<Spinner size={13} />
|
||||
<span className="font-medium" data-profile-title="">{PT.running}</span>
|
||||
<span className="flex-1" />
|
||||
<span className="tnum text-2xs text-faint" data-profile-elapsed={Math.floor(used / 1000)}>
|
||||
{PT.elapsed(secondsText(used), secondsText(max))}
|
||||
</span>
|
||||
</div>
|
||||
{/* 不是进度:服务端不报进度,这条只量已用时长占总时长上限的多少,剖析最晚在上限处停下 */}
|
||||
<div className="h-1 overflow-hidden rounded-full bg-hover" role="meter" aria-label={PT.elapsed(secondsText(used), secondsText(max))}
|
||||
aria-valuemin={0} aria-valuemax={job.settings.max_total_s} aria-valuenow={Math.min(job.settings.max_total_s, Math.floor(used / 1000))}>
|
||||
<div className="h-full rounded-full bg-[var(--border-strong)] transition-[width] duration-1000 ease-linear"
|
||||
style={{ width: `${Math.min(100, (used / max) * 100)}%` }} />
|
||||
</div>
|
||||
<Notice tone="info" attr={{ 'data-profile-no-cancel': '' }}>{PT.runningHint}</Notice>
|
||||
<p className="text-2xs leading-relaxed text-faint">{PT.leavePage}</p>
|
||||
</div>
|
||||
)
|
||||
footer = <button type="button" className="btn" onClick={onClose} data-autofocus data-profile-background="">{PT.background}</button>
|
||||
} else if (job.phase === 'done') {
|
||||
body = <Report report={job.report} ms={job.ms} onSettings={onSettings} onOpenTable={onOpenTable} onFillCodes={onFillCodes} />
|
||||
footer = (
|
||||
<>
|
||||
<button type="button" className="btn btn-ghost mr-auto" onClick={onReset} data-profile-again="">{PT.again}</button>
|
||||
<button type="button" className="btn btn-primary" onClick={onClose} data-autofocus data-profile-close="">{PT.close}</button>
|
||||
</>
|
||||
)
|
||||
} else if (job.phase === 'blocked') {
|
||||
const action = job.kind === 'disabled'
|
||||
? <button type="button" className="btn btn-sm btn-primary" onClick={onSettings} data-profile-enable="">{PT.enable}</button>
|
||||
: job.kind === 'busy' || job.kind === 'other'
|
||||
? <button type="button" className="btn btn-sm" onClick={() => onStart(job.tables)} data-profile-retry="">{PT.retry}</button>
|
||||
: <button type="button" className="btn btn-sm" onClick={onGoSource} data-profile-go-source="">{PT.goSource}</button>
|
||||
body = (
|
||||
<Notice tone={job.kind === 'busy' ? 'info' : 'warn'} attr={{ 'data-profile-blocked': job.kind }}>
|
||||
<p className="font-medium">{PT.blocked[job.kind]}</p>
|
||||
<p className="mt-0.5 text-dim">{job.message}</p>
|
||||
{job.kind !== 'other' && <p className="mt-0.5 text-dim">{PT.blockedNext[job.kind]}</p>}
|
||||
<div className="mt-2">{action}</div>
|
||||
</Notice>
|
||||
)
|
||||
footer = (
|
||||
<>
|
||||
<button type="button" className="btn btn-ghost mr-auto" onClick={onReset} data-profile-back="">{PT.backToSetup}</button>
|
||||
<button type="button" className="btn" onClick={onClose}>{PT.close}</button>
|
||||
</>
|
||||
)
|
||||
} else {
|
||||
body = (
|
||||
<div data-profile-failed="">
|
||||
<p className="mb-2 text-xs font-medium">{PT.failedTitle}</p>
|
||||
<ErrorState error={job.error} onRetry={() => onStart(job.tables)} />
|
||||
</div>
|
||||
)
|
||||
footer = (
|
||||
<>
|
||||
<button type="button" className="btn btn-ghost mr-auto" onClick={onReset}>{PT.backToSetup}</button>
|
||||
<button type="button" className="btn" onClick={onClose}>{PT.close}</button>
|
||||
</>
|
||||
)
|
||||
}
|
||||
|
||||
return (
|
||||
<Modal open onClose={onClose} title={PT.title} width={640} footer={footer}>
|
||||
<div className="space-y-4" data-profile-dialog={job?.phase ?? 'setup'}>{body}</div>
|
||||
</Modal>
|
||||
)
|
||||
}
|
||||
|
||||
/** 确认前写明预算:按这个数据源的剖析设置,最多发几条查询、单条和总时长、抽样规模、整表统计的门槛 */
|
||||
function Budget({ settings: s, sourceName, onSettings }: { settings: CatalogProfileSettings; sourceName: string; onSettings: () => void }) {
|
||||
const lines = [PT.budgetQueries(s.max_queries, s.query_timeout_s), PT.budgetTotal(s.max_total_s), PT.budgetSample(s.sample_size),
|
||||
PT.budgetScan(s.max_scan_rows)]
|
||||
return (
|
||||
<div className="rounded-lg border px-3 py-2.5" data-profile-budget={`${s.max_queries},${s.query_timeout_s},${s.max_total_s}`}>
|
||||
<div className="mb-1.5 flex items-center gap-2 text-2xs">
|
||||
<span className="font-medium text-dim">{PT.budget}</span>
|
||||
<span className="text-faint">{PT.budgetFrom(sourceName)}</span>
|
||||
<span className="flex-1" />
|
||||
<button type="button" className="rounded px-1 text-2xs text-dim underline decoration-dotted underline-offset-2 hover:text-fg"
|
||||
onClick={onSettings} data-profile-settings-open="">{PT.editSettings}</button>
|
||||
</div>
|
||||
<ul className="space-y-0.5 text-xs">
|
||||
{lines.map((l) => (
|
||||
<li key={l} className="flex items-start gap-1.5">
|
||||
<span className="mt-[7px] h-1 w-1 shrink-0 rounded-full bg-[var(--text-faint)]" aria-hidden />
|
||||
<span className="tnum">{l}</span>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 报告
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function Report({ report: r, ms, onSettings, onOpenTable, onFillCodes }: {
|
||||
report: CatalogProfileOut
|
||||
ms: number
|
||||
onSettings: () => void
|
||||
onOpenTable: (table: string) => void
|
||||
onFillCodes: (table: string, column: string) => void
|
||||
}) {
|
||||
const changed = r.total.added + r.total.updated + r.total.removed
|
||||
const stop = r.stopped
|
||||
// 一项发现、一项跳过都没有的表合成一行,几十张表时报告不被空卡片撑长
|
||||
const quiet = r.tables.filter((t) => !t.error && !t.findings.length && !t.skipped.length && !t.date_ranges.length)
|
||||
const shown = r.tables.filter((t) => !quiet.includes(t))
|
||||
return (
|
||||
<div className="space-y-3" data-profile-report={stop ?? 'done'}>
|
||||
<div className="flex items-start gap-2">
|
||||
{stop
|
||||
? <TriangleAlert size={15} className="mt-px shrink-0" style={{ color: 'var(--st-waiting)' }} aria-hidden />
|
||||
: <CircleCheck size={15} className="mt-px shrink-0" style={{ color: 'var(--st-done)' }} aria-hidden />}
|
||||
<div className="min-w-0 flex-1">
|
||||
<p className="text-sm font-medium" data-profile-title="">{stop ? PT.stoppedTitle(PROFILE_STOP_LABEL[stop] ?? stop) : PT.done}</p>
|
||||
<p className="tnum mt-0.5 text-xs text-dim" data-profile-summary={`${r.queries_used}/${r.settings.max_queries}`}>
|
||||
{PT.summary(r.queries_used, r.settings.max_queries, r.tables.length)}
|
||||
<span className="text-faint"> · {secondsText(ms)}</span>
|
||||
</p>
|
||||
<p className="tnum mt-0.5 text-xs" data-profile-changes={`${r.total.added},${r.total.updated},${r.total.removed}`}>
|
||||
{changed ? PT.changes(r.total.added, r.total.updated, r.total.removed) : <span className="text-faint">{PT.noChange}</span>}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
{stop && (
|
||||
<Notice tone="warn" attr={{ 'data-profile-stopped': stop }}>
|
||||
<p>{PROFILE_STOP_NEXT[stop] ?? ''}</p>
|
||||
{stop !== 'failed' && (
|
||||
<button type="button" className="btn btn-sm mt-2" onClick={onSettings} data-profile-settings-open="">{PT.editSettings}</button>
|
||||
)}
|
||||
</Notice>
|
||||
)}
|
||||
{r.note && <Notice tone="info" attr={{ 'data-profile-note': '' }}>{r.note}</Notice>}
|
||||
{shown.map((t) => <TableReport key={t.table_name} t={t} onOpenTable={onOpenTable} onFillCodes={onFillCodes} />)}
|
||||
{quiet.length > 0 && (
|
||||
<p className="text-2xs leading-relaxed text-faint" data-profile-quiet={quiet.length}>
|
||||
{PT.noFindings}:<span className="mono">{quiet.map((t) => t.table_name).join('、')}</span>
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function sizeText(size: CatalogProfileSize | null): { text: string; hint?: string } | null {
|
||||
if (!size) return null
|
||||
if (size.rows != null) return { text: size.method === 'stats' ? PT.rowsStats(size.rows) : PT.rowsCount(size.rows), hint: PT.rowsHint[size.method] }
|
||||
if (size.at_least != null) return { text: PT.rowsAtLeast(size.at_least), hint: PT.rowsHint.count }
|
||||
return null
|
||||
}
|
||||
|
||||
function TableReport({ t, onOpenTable, onFillCodes }: {
|
||||
t: CatalogProfileTable
|
||||
onOpenTable: (table: string) => void
|
||||
onFillCodes: (table: string, column: string) => void
|
||||
}) {
|
||||
const relations = t.findings.filter((f): f is CatalogProfileRelationFinding => f.kind === 'relation')
|
||||
const codes = t.findings.filter((f): f is CatalogProfileCodesFinding => f.kind === 'codes')
|
||||
const date = t.findings.find((f): f is CatalogProfileDateFinding => f.kind === 'business_date')
|
||||
const size = sizeText(t.row_estimate)
|
||||
const changed = t.added + t.updated + t.removed
|
||||
return (
|
||||
<section className="rounded-lg border" data-profile-table={t.table_name} aria-label={t.table_name}>
|
||||
<header className="flex flex-wrap items-center gap-x-2 gap-y-0.5 border-b bg-elev px-3 py-1.5">
|
||||
<h3 className="mono min-w-0 text-xs font-medium [overflow-wrap:anywhere]">{t.table_name}</h3>
|
||||
<span className="tnum text-2xs text-faint">
|
||||
{PT.tableQueries(t.queries)}
|
||||
{size && <span title={size.hint}> · {size.text}</span>}
|
||||
{changed > 0 && <span> · {PT.changes(t.added, t.updated, t.removed)}</span>}
|
||||
</span>
|
||||
<span className="flex-1" />
|
||||
{!t.error && (
|
||||
<button type="button" className="inline-flex items-center gap-0.5 rounded px-1 text-2xs text-dim hover:bg-hover hover:text-fg"
|
||||
onClick={() => onOpenTable(t.table_name)} aria-label={PT.openTableLabel(t.table_name)} data-profile-open={t.table_name}>
|
||||
{PT.openTable} <ArrowUpRight size={10} aria-hidden />
|
||||
</button>
|
||||
)}
|
||||
</header>
|
||||
<div className="space-y-2.5 px-3 py-2">
|
||||
{t.error && (
|
||||
<p className="flex items-start gap-1.5 text-xs text-[var(--err)]" data-profile-table-error="">
|
||||
<OctagonX size={12} className="mt-0.5 shrink-0" aria-hidden /> <span>{t.error}</span>
|
||||
</p>
|
||||
)}
|
||||
{relations.length > 0 && (
|
||||
<Group title={PT.findingRelations}>
|
||||
{relations.map((f) => <RelationLine key={f.path} f={f} />)}
|
||||
</Group>
|
||||
)}
|
||||
{codes.length > 0 && (
|
||||
<Group title={PT.findingCodes}>
|
||||
{codes.map((f) => <CodesLine key={f.path} f={f} onFill={() => onFillCodes(t.table_name, f.column)} />)}
|
||||
</Group>
|
||||
)}
|
||||
{date && (
|
||||
<Group title={PT.findingDate}>
|
||||
<li className="text-xs" data-profile-finding="business_date">
|
||||
<StatusChip status={date.status} className="mr-1.5 align-middle" />
|
||||
{PT.dateProposal(date.column)}
|
||||
<span className="tnum text-dim">,{PT.dateSpan(date.min.slice(0, 19), date.max.slice(0, 19))}</span>
|
||||
</li>
|
||||
</Group>
|
||||
)}
|
||||
{!date && t.date_ranges.length > 0 && (
|
||||
<Group title={PT.dateRanges}>
|
||||
{t.date_ranges.map((d) => (
|
||||
<li key={d.column} className="tnum text-xs text-dim" data-profile-date-range={d.column}>
|
||||
<span className="mono text-fg">{d.column}</span>:{PT.dateSpan(d.min.slice(0, 19), d.max.slice(0, 19))}
|
||||
</li>
|
||||
))}
|
||||
</Group>
|
||||
)}
|
||||
{!t.error && !t.findings.length && !t.date_ranges.length && (
|
||||
<p className="text-2xs text-faint">{PT.noFindings}</p>
|
||||
)}
|
||||
{t.skipped.length > 0 && <Skipped list={t.skipped} />}
|
||||
</div>
|
||||
</section>
|
||||
)
|
||||
}
|
||||
|
||||
function Group({ title, children }: { title: string; children: ReactNode }) {
|
||||
return (
|
||||
<div>
|
||||
<div className="mb-1 text-2xs font-medium text-faint">{title}</div>
|
||||
<ul className="space-y-1.5">{children}</ul>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function RelationLine({ f }: { f: CatalogProfileRelationFinding }) {
|
||||
const outcome = f.confirmed
|
||||
? { text: PT.relationConfirmed, tone: 'var(--text-dim)' }
|
||||
: f.status === 'verified'
|
||||
? { text: PT.relationVerified, tone: STATUS_TONE.verified }
|
||||
: { text: `${PT.relationKept}:${f.coverage < 0.95 ? PT.relationLowCoverage : PT.relationNotUnique}`, tone: STATUS_TONE.proposed }
|
||||
return (
|
||||
<li className="text-xs" data-profile-finding="relation" data-status={f.status} data-path={f.path}>
|
||||
<div className="flex flex-wrap items-center gap-x-1.5 gap-y-0.5">
|
||||
<StatusChip status={f.status} />
|
||||
<span className="mono min-w-0 [overflow-wrap:anywhere]">{f.target}</span>
|
||||
</div>
|
||||
<div className="tnum mt-0.5 text-2xs text-dim">
|
||||
<span data-profile-coverage="">{PT.coverage(coverageText(f.coverage))}</span>
|
||||
{' · '}
|
||||
<span data-profile-cardinality="">{f.cardinality ? CATALOG_CARDINALITY_LABEL[f.cardinality] : PT.cardinalityUnknown}</span>
|
||||
{' · '}
|
||||
{PT.sampled(f.sample, f.matched)}
|
||||
</div>
|
||||
<div className="mt-0.5 text-2xs" style={{ color: outcome.tone }} data-profile-outcome="">{outcome.text}</div>
|
||||
</li>
|
||||
)
|
||||
}
|
||||
|
||||
function CodesLine({ f, onFill }: { f: CatalogProfileCodesFinding; onFill: () => void }) {
|
||||
// 服务端在 summary 里写了几个含义待填写(别的来源写过的含义会沿用);认不出时按都待填写
|
||||
const pending = Number(f.summary.match(/(\d+)\s*个含义待填写/)?.[1] ?? (f.summary.includes('待填写') ? f.values.length : 0))
|
||||
return (
|
||||
<li className="text-xs" data-profile-finding="codes" data-column={f.column}>
|
||||
<div className="flex flex-wrap items-center gap-x-1.5 gap-y-0.5">
|
||||
<StatusChip status={f.status} />
|
||||
<span className="mono font-medium">{f.column}</span>
|
||||
<span className="tnum text-2xs text-faint">{PT.codesSummary(f.values.length, f.rows)}</span>
|
||||
</div>
|
||||
<div className="mt-1 flex flex-wrap gap-1">
|
||||
{f.values.slice(0, CODES_SHOWN).map((v) => (
|
||||
<span key={v.value} className="inline-flex items-center gap-1 rounded border bg-bg px-1 text-2xs">
|
||||
<span className="mono">{v.value}</span>
|
||||
<span className="tnum text-faint">{PT.codeRows(v.rows)}</span>
|
||||
</span>
|
||||
))}
|
||||
{f.values.length > CODES_SHOWN && <span className="text-2xs text-faint">+{f.values.length - CODES_SHOWN}</span>}
|
||||
</div>
|
||||
<div className="mt-1 flex flex-wrap items-center gap-2">
|
||||
{pending > 0 && <span className="text-2xs" style={{ color: 'var(--st-waiting)' }} data-codes-pending={pending}>{PT.codesPendingN(pending)}</span>}
|
||||
<button type="button" className="btn btn-xs" onClick={onFill} aria-label={PT.fillMeaningsLabel(f.column)} data-profile-fill={f.column}>
|
||||
{PT.fillMeanings}
|
||||
</button>
|
||||
</div>
|
||||
</li>
|
||||
)
|
||||
}
|
||||
|
||||
/** 跳过的项:原因(短)+ 哪一项 + 服务端给的整句。多了折起来,每类停下的原因只列前几项 */
|
||||
function Skipped({ list }: { list: CatalogProfileSkip[] }) {
|
||||
const [all, setAll] = useState(false)
|
||||
const shown = all ? list : list.slice(0, SKIP_SHOWN)
|
||||
return (
|
||||
<div data-profile-skipped={list.length}>
|
||||
<div className="mb-1 text-2xs font-medium text-faint">{PT.skippedTitle(list.length)}</div>
|
||||
<ul className="space-y-1">
|
||||
{shown.map((s, i) => (
|
||||
<li key={`${s.kind}:${s.target}:${i}`} className="flex items-start gap-1.5 text-2xs" data-profile-skip={s.reason}>
|
||||
<span className="chip mt-px shrink-0 !px-1.5 !py-0" style={{ color: 'var(--text-dim)' }}>
|
||||
{PROFILE_SKIP_REASON_LABEL[s.reason] ?? s.reason}
|
||||
</span>
|
||||
<span className="min-w-0 leading-relaxed">
|
||||
<span className="text-faint">{PROFILE_SKIP_KIND_LABEL[s.kind] ?? s.kind} </span>
|
||||
<span className="mono text-dim [overflow-wrap:anywhere]">{s.target}</span>
|
||||
<span className="block text-dim">{s.detail}</span>
|
||||
</span>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
{list.length > SKIP_SHOWN && !all && (
|
||||
<button type="button" className="mt-1 rounded px-1 text-2xs text-dim underline decoration-dotted underline-offset-2 hover:text-fg"
|
||||
onClick={() => setAll(true)}>
|
||||
{PT.moreSkipped(list.length - SKIP_SHOWN)}
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
import { useState } from 'react'
|
||||
import { ShieldAlert } from 'lucide-react'
|
||||
import { ApiError, api } from '../../api/client'
|
||||
import type { CatalogProfileNumberKey } from '../../types'
|
||||
import { Field, Modal, Spinner, toast } from '../../components/ui'
|
||||
import { PROFILE_FIELD_HINT, PROFILE_FIELD_LABEL, PROFILE_FIELD_UNIT, PROFILE_TEXT as PT } from '../../lib/terms'
|
||||
import {
|
||||
PROFILE_DEFAULTS, PROFILE_LIMITS, PROFILE_NUMBER_KEYS, hasProfileOption, profileFieldOfRejection, profileFormOf, profileOptionOf,
|
||||
profileProblems, profileSettingsOfForm, sameProfileForm, withProfileOption,
|
||||
} from './profile'
|
||||
import type { ProfileForm } from './profile'
|
||||
|
||||
// ===========================================================================
|
||||
// 数据剖析的设置:开关、五项预算、风险说明。数据源编辑框里是其中一段;导入表格的源没有编辑框,数据目录页在
|
||||
// 剖析被拒(未开启)时也要就地打开它,所以另有一个只改这一项的弹窗(options 的其余键原样带上)。
|
||||
// 剖析会对业务库发查询,开关默认关着;开启后五项预算才展开,风险说明写的是此刻填的数。
|
||||
// ===========================================================================
|
||||
|
||||
/** 保存被拒时的落点:某一项,或者整段(认不出是哪一项) */
|
||||
export type ProfileErrorKey = 'enabled' | CatalogProfileNumberKey | 'section'
|
||||
|
||||
export function ProfileSettingsSection({ idPrefix, form, onChange, serverError }: {
|
||||
/** 输入框 id 的前缀:报错时按 `${idPrefix}-${键}` 聚焦 */
|
||||
idPrefix: string
|
||||
form: ProfileForm
|
||||
onChange: (next: ProfileForm) => void
|
||||
/** 服务端拒收(422)的那一句,落在对应的项上 */
|
||||
serverError?: { key: ProfileErrorKey; message: string } | null
|
||||
}) {
|
||||
const problems = profileProblems(form)
|
||||
const errorOf = (k: CatalogProfileNumberKey) => (serverError?.key === k ? serverError.message : problems[k])
|
||||
// 开关关着时预算收起;但填错的项(或服务端点了名的项)要看得见,不然保存被拦下却找不到原因
|
||||
const open = form.enabled || Object.keys(problems).length > 0
|
||||
|| (!!serverError && serverError.key !== 'enabled' && serverError.key !== 'section')
|
||||
const s = profileSettingsOfForm(form)
|
||||
const setValue = (k: CatalogProfileNumberKey, v: string) => onChange({ ...form, values: { ...form.values, [k]: v } })
|
||||
const switchError = serverError && (serverError.key === 'enabled' || serverError.key === 'section') ? serverError.message : null
|
||||
|
||||
return (
|
||||
<fieldset className="rounded-lg border" data-field="catalog_profile" data-profile-settings={form.enabled ? 'on' : 'off'}>
|
||||
<legend className="sr-only">{PT.section}</legend>
|
||||
<label className="flex cursor-pointer items-start gap-2 p-2.5">
|
||||
<input id={`${idPrefix}-enabled`} type="checkbox" className="mt-0.5" checked={form.enabled}
|
||||
aria-describedby={`${idPrefix}-enabled-hint`} aria-invalid={switchError ? true : undefined}
|
||||
onChange={(e) => onChange({ ...form, enabled: e.target.checked })} data-profile-enabled="" />
|
||||
<span className="min-w-0 text-xs">
|
||||
{PT.section}
|
||||
<span id={`${idPrefix}-enabled-hint`} className="ml-1.5 text-2xs leading-relaxed text-faint">{PT.enableHint}</span>
|
||||
</span>
|
||||
</label>
|
||||
{switchError && (
|
||||
<p className="px-2.5 pb-2 text-2xs leading-relaxed text-[var(--err)]" role="alert" data-profile-error="section">{switchError}</p>
|
||||
)}
|
||||
{open && (
|
||||
<div className="space-y-3 border-t px-2.5 py-2.5">
|
||||
{form.enabled && (
|
||||
<p className="flex items-start gap-1.5 rounded-md px-2 py-1.5 text-2xs leading-relaxed text-dim" data-profile-risk=""
|
||||
style={{ background: 'var(--st-waiting-soft)' }}>
|
||||
<ShieldAlert size={12} className="mt-px shrink-0" style={{ color: 'var(--st-waiting)' }} aria-hidden />
|
||||
<span>{PT.risk(s.max_queries, s.query_timeout_s, s.max_total_s)}</span>
|
||||
</p>
|
||||
)}
|
||||
<div className="grid gap-3 sm:grid-cols-2">
|
||||
{PROFILE_NUMBER_KEYS.map((k) => {
|
||||
const { min, max } = PROFILE_LIMITS[k]
|
||||
const unit = PROFILE_FIELD_UNIT[k]
|
||||
const err = errorOf(k)
|
||||
const id = `${idPrefix}-${k}`
|
||||
const hint = `${PT.range(min, max, unit, PROFILE_DEFAULTS[k])}${k === 'max_scan_rows' ? `,${PT.scanZero}` : ''}。${PROFILE_FIELD_HINT[k]}`
|
||||
return (
|
||||
<Field key={k} htmlFor={id} label={PROFILE_FIELD_LABEL[k]} hint={hint} error={err}>
|
||||
{(p) => (
|
||||
<div className="relative" data-profile-field={k}>
|
||||
<input {...p} className="field tnum pr-8" inputMode={PROFILE_LIMITS[k].integer ? 'numeric' : 'decimal'}
|
||||
autoComplete="off" spellCheck={false} value={form.values[k]}
|
||||
placeholder={PT.placeholder(PROFILE_DEFAULTS[k])}
|
||||
style={err ? { borderColor: 'var(--err)' } : undefined}
|
||||
onChange={(e) => setValue(k, e.target.value)} />
|
||||
<span className="pointer-events-none absolute right-2.5 top-1/2 -translate-y-1/2 text-2xs text-faint" aria-hidden>
|
||||
{unit}
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
</Field>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</fieldset>
|
||||
)
|
||||
}
|
||||
|
||||
/** 聚焦保存被拒的那一项(预算收着时先等它展开) */
|
||||
export function focusProfileField(idPrefix: string, key: ProfileErrorKey) {
|
||||
const id = key === 'section' ? `${idPrefix}-enabled` : `${idPrefix}-${key}`
|
||||
requestAnimationFrame(() => requestAnimationFrame(() => {
|
||||
const el = document.getElementById(id)
|
||||
el?.scrollIntoView({ block: 'nearest' })
|
||||
el?.focus({ preventScroll: true })
|
||||
}))
|
||||
}
|
||||
|
||||
/**
|
||||
* 只改数据剖析设置的弹窗:导入表格的源(没有编辑框)、数据目录页里剖析被拒时用。
|
||||
* 保存时 options 的其余键(遮罩列、schema……)原样带上,只换剖析设置这一项
|
||||
*/
|
||||
export function ProfileSettingsDialog({ row, onClose, onSaved }: {
|
||||
row: { id: string; name: string; options?: Record<string, unknown> | null }
|
||||
onClose: () => void
|
||||
onSaved: (row: any) => void
|
||||
}) {
|
||||
const [initial] = useState(() => profileFormOf(row.options))
|
||||
const [form, setForm] = useState<ProfileForm>(initial)
|
||||
const [saving, setSaving] = useState(false)
|
||||
const [serverError, setServerError] = useState<{ key: ProfileErrorKey; message: string } | null>(null)
|
||||
const dirty = !sameProfileForm(form, initial)
|
||||
const problems = Object.keys(profileProblems(form)).length
|
||||
const idPrefix = `profile-${row.id}`
|
||||
|
||||
const save = async () => {
|
||||
if (!dirty || problems) return
|
||||
setSaving(true)
|
||||
setServerError(null)
|
||||
try {
|
||||
const options = withProfileOption(row.options, profileOptionOf(form, hasProfileOption(row.options)))
|
||||
const next = await api.datasources.update(row.id, { options })
|
||||
onSaved(next)
|
||||
toast.ok(form.enabled === initial.enabled ? PT.saved(row.name) : form.enabled ? PT.savedOn(row.name) : PT.savedOff(row.name))
|
||||
} catch (e) {
|
||||
const key = e instanceof ApiError && e.status === 422 ? profileFieldOfRejection(e.message) : null
|
||||
if (key) {
|
||||
setServerError({ key, message: e instanceof ApiError ? e.message : '' })
|
||||
focusProfileField(idPrefix, key)
|
||||
} else {
|
||||
toast.error(e)
|
||||
}
|
||||
} finally {
|
||||
setSaving(false)
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<Modal open onClose={onClose} dirty={dirty} width={560} title={PT.dialogTitle(row.name)}
|
||||
footer={(
|
||||
<>
|
||||
{problems > 0 && <span className="mr-auto text-2xs text-[var(--err)]" data-profile-invalid={problems}>{PT.invalid(problems)}</span>}
|
||||
<button type="button" className="btn" onClick={onClose}>取消</button>
|
||||
<button type="button" className="btn btn-primary" disabled={saving || !dirty || problems > 0} onClick={() => void save()}
|
||||
data-profile-save="">
|
||||
{saving ? <Spinner size={11} /> : null} 保存
|
||||
</button>
|
||||
</>
|
||||
)}>
|
||||
<div className="space-y-3" data-profile-settings-dialog={row.id}>
|
||||
<p className="text-2xs leading-relaxed text-faint">{PT.dialogBody}</p>
|
||||
<ProfileSettingsSection idPrefix={idPrefix} form={form} serverError={serverError}
|
||||
onChange={(next) => { setForm(next); setServerError(null) }} />
|
||||
</div>
|
||||
</Modal>
|
||||
)
|
||||
}
|
||||
@@ -12,13 +12,15 @@ import { useLeaveGuard } from '../../lib/leave'
|
||||
import { formatDateTime, formatNumber } from '../../lib/format'
|
||||
import {
|
||||
CATALOG_CARDINALITY_LABEL, CATALOG_IMPACT_TEXT, CATALOG_KIND_HINT, CATALOG_KIND_LABEL, CATALOG_TABLE_FIELD_HINT,
|
||||
CATALOG_TABLE_FIELD_LABEL, CATALOG_TEXT as CT,
|
||||
CATALOG_TABLE_FIELD_LABEL, CATALOG_TEXT as CT, CODES_TEXT as KT,
|
||||
} from '../../lib/terms'
|
||||
import { CatalogImpactList } from '../../components/CatalogImpact'
|
||||
import { CountBadges, ItemMark, StatusChip, StatusLegend, SystemNotesNotice } from './parts'
|
||||
import type { ReviewTarget } from './parts'
|
||||
import { ColumnTable, Value } from './ColumnTable'
|
||||
import { ColumnTable, Value, pendingCodes } from './ColumnTable'
|
||||
import type { ColumnModel } from './ColumnTable'
|
||||
import { CodesDialog } from './CodesDialog'
|
||||
import { coverageText } from './profile'
|
||||
import {
|
||||
COLUMN_FIELDS, DATE_KEYS, TABLE_FIELDS, buildNotes, changeCount, columnPath, confirmProposed, countsOf, formFromNotes,
|
||||
relationDraft, relationPath, tablePath, unknownKeys, validateForm,
|
||||
@@ -49,13 +51,15 @@ const PROBLEM_TEXT = {
|
||||
type Editing = { initial: EditForm; form: EditForm }
|
||||
type Conflict = 'edit' | 'review'
|
||||
|
||||
export function TableDetail({ sourceId, table, rows, drafted, prev, next, onOpen, onBack, onDetail }: {
|
||||
export function TableDetail({ sourceId, table, rows, drafted, fillCodes, prev, next, onOpen, onBack, onDetail }: {
|
||||
sourceId: string
|
||||
table: string
|
||||
/** 全部表:关联关系的目标表能点就跳过去,编辑时给目标表做候选 */
|
||||
rows: CatalogTableRow[]
|
||||
/** 起草刚写进了这些表:正在看的表在里面、又没在编辑,就重新载入 */
|
||||
drafted: { seq: number; tables: Set<string> }
|
||||
/** 从剖析报告点「填写含义」过来:载入后打开这一列的码值弹窗(seq 让同一列连点两次也会再开) */
|
||||
fillCodes?: { table: string; column: string; seq: number } | null
|
||||
prev: string | null
|
||||
next: string | null
|
||||
onOpen: (table: string) => void
|
||||
@@ -73,6 +77,8 @@ export function TableDetail({ sourceId, table, rows, drafted, prev, next, onOpen
|
||||
const [live, setLive] = useState('')
|
||||
const [colQuery, setColQuery] = useState('')
|
||||
const [onlyPending, setOnlyPending] = useState(false)
|
||||
/** 正在填写哪一列的码值含义 */
|
||||
const [codesFor, setCodesFor] = useState<string | null>(null)
|
||||
const loadSeq = useRef(0)
|
||||
const onDetailRef = useRef(onDetail)
|
||||
onDetailRef.current = onDetail
|
||||
@@ -101,6 +107,7 @@ export function TableDetail({ sourceId, table, rows, drafted, prev, next, onOpen
|
||||
setColQuery('')
|
||||
setOnlyPending(false)
|
||||
setLive('')
|
||||
setCodesFor(null)
|
||||
void load()
|
||||
}, [load])
|
||||
|
||||
@@ -115,6 +122,15 @@ export function TableDetail({ sourceId, table, rows, drafted, prev, next, onOpen
|
||||
if (drafted.tables.has(table) && !editingRef.current) void load()
|
||||
}, [drafted, table, load])
|
||||
|
||||
// 剖析报告里点了「填写含义」:这张表载入后打开那一列的码值弹窗。编辑中不打开,免得同一项两处同时改
|
||||
const fillSeen = useRef(0)
|
||||
useEffect(() => {
|
||||
if (!fillCodes || fillCodes.table !== table || fillCodes.seq === fillSeen.current) return
|
||||
if (!detail || detail.table_name !== table || editingRef.current) return
|
||||
fillSeen.current = fillCodes.seq
|
||||
if (detail.notes.columns?.[fillCodes.column]?.codes) setCodesFor(fillCodes.column)
|
||||
}, [fillCodes, table, detail])
|
||||
|
||||
const changes = edit ? changeCount(edit.initial, edit.form) : 0
|
||||
const problems = useMemo(() => (edit ? validateForm(edit.form, PROBLEM_TEXT) : {}), [edit])
|
||||
const problemCount = Object.keys(problems).length
|
||||
@@ -163,6 +179,41 @@ export function TableDetail({ sourceId, table, rows, drafted, prev, next, onOpen
|
||||
if (done) setLive(CT.confirmedN(n))
|
||||
}, [sourceId, table, write])
|
||||
|
||||
const fillCodesOf = useCallback((col: string) => setCodesFor(col), [])
|
||||
|
||||
/**
|
||||
* 填好的码值含义整份提交:只换这一列码值的值,服务端记为人工填写、已确认。别人刚改过(409)时收起弹窗、挂冲突横幅;
|
||||
* 其余失败弹窗留着,填的字不丢
|
||||
*/
|
||||
const saveCodes = useCallback(async (col: string, value: Record<string, string>) => {
|
||||
const d = detailRef.current
|
||||
const item = d?.notes.columns?.[col]?.codes
|
||||
if (!d || !item) return
|
||||
const notes = structuredClone(d.notes)
|
||||
notes.columns![col] = { ...notes.columns![col], codes: { ...item, value } }
|
||||
setBusy(`codes:${col}`)
|
||||
setWriteError(null)
|
||||
try {
|
||||
const next = await api.dataCatalog.put(sourceId, table, notes, d.version)
|
||||
setDetail(next)
|
||||
setConflict(null)
|
||||
onDetailRef.current(next)
|
||||
setCodesFor(null)
|
||||
const filled = Math.max(0, pendingCodes(item.value) - pendingCodes(value))
|
||||
toast.ok(KT.saved(filled))
|
||||
setLive(KT.saved(filled))
|
||||
} catch (e) {
|
||||
if (e instanceof ApiError && e.status === 409) {
|
||||
setCodesFor(null)
|
||||
setConflict('review')
|
||||
} else {
|
||||
toast.error(e)
|
||||
}
|
||||
} finally {
|
||||
setBusy('')
|
||||
}
|
||||
}, [sourceId, table])
|
||||
|
||||
const setColumn = useCallback((col: string, f: ColumnField, v: string) => {
|
||||
setEdit((e) => e && ({ ...e, form: { ...e.form, columns: { ...e.form.columns, [col]: { ...e.form.columns[col], [f]: v } } } }))
|
||||
}, [])
|
||||
@@ -368,7 +419,7 @@ export function TableDetail({ sourceId, table, rows, drafted, prev, next, onOpen
|
||||
? (
|
||||
<ColumnTable columns={shownColumns} form={edit?.form.columns ?? null} initial={edit?.initial.columns ?? null}
|
||||
problems={problems} busy={!!busy} systemNotes={detail.system_notes}
|
||||
onReview={review} onConfirmColumn={confirmColumn} onChange={setColumn} />
|
||||
onReview={review} onConfirmColumn={confirmColumn} onFillCodes={fillCodesOf} onChange={setColumn} />
|
||||
)
|
||||
: <p className="rounded-lg border px-3 py-6 text-center text-xs text-faint">{CT.noColumns}</p>}
|
||||
</section>
|
||||
@@ -394,6 +445,11 @@ export function TableDetail({ sourceId, table, rows, drafted, prev, next, onOpen
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{codesFor && notes.columns?.[codesFor]?.codes && (
|
||||
<CodesDialog column={codesFor} item={notes.columns[codesFor]!.codes!} saving={busy === `codes:${codesFor}`}
|
||||
onSave={(v) => void saveCodes(codesFor, v)} onClose={() => setCodesFor(null)} />
|
||||
)}
|
||||
|
||||
{edit && (
|
||||
<footer className="flex flex-wrap items-center gap-2 border-t bg-panel px-4 py-2.5" data-catalog-edit-bar="">
|
||||
<span className="tnum text-xs text-dim" data-catalog-changes={changes}>{changes ? CT.changes(changes) : CT.noChanges}</span>
|
||||
@@ -597,7 +653,9 @@ function RelationList({ relations, busy, targetOf, onOpen, onReview }: {
|
||||
<td className={clsx('mono px-2 py-1.5 align-top', rejected && 'text-faint line-through')}>{r.to_columns.join(', ')}</td>
|
||||
<td className="px-2 py-1.5 align-top">{r.cardinality ? CATALOG_CARDINALITY_LABEL[r.cardinality] : <span className="text-faint">{CT.cardinalityNone}</span>}</td>
|
||||
<td className="tnum px-2 py-1.5 align-top">
|
||||
{typeof r.coverage === 'number' ? CT.coverageValue(Math.round(r.coverage * 100)) : <span className="text-faint">—</span>}
|
||||
{typeof r.coverage === 'number'
|
||||
? <span data-relation-coverage={r.coverage}>{coverageText(r.coverage)}</span>
|
||||
: <span className="text-faint">—</span>}
|
||||
</td>
|
||||
<td className="px-2 py-1.5 align-top">
|
||||
<ItemMark target={{ path: relationPath(r.id), where, source: r.source, status: r.status, note: r.note, updated_at: r.updated_at }}
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
import { memo, useCallback, useEffect, useRef } from 'react'
|
||||
import type { KeyboardEvent as ReactKeyboardEvent, MouseEvent as ReactMouseEvent } from 'react'
|
||||
import { AlertTriangle, Search, Sparkles, X } from 'lucide-react'
|
||||
import { AlertTriangle, ScanSearch, Search, Sparkles, X } from 'lucide-react'
|
||||
import clsx from 'clsx'
|
||||
import type { CatalogTableRow } from '../../types'
|
||||
import { useRadioGroup } from '../../components/ui'
|
||||
import { formatNumber } from '../../lib/format'
|
||||
import { CATALOG_KIND_LABEL, CATALOG_TEXT as CT } from '../../lib/terms'
|
||||
import { CATALOG_KIND_LABEL, CATALOG_TEXT as CT, PROFILE_TEXT as PT } from '../../lib/terms'
|
||||
import { CountBadges } from './parts'
|
||||
import { LIST_FILTERS, LIST_SORTS, progressOf } from './model'
|
||||
import type { ListFilter, ListSort } from './model'
|
||||
@@ -17,7 +17,7 @@ import type { ListFilter, ListSort } from './model'
|
||||
// ===========================================================================
|
||||
|
||||
export function TableIndex({
|
||||
rows, total, counts, query, onQuery, filter, onFilter, sort, onSort, selected, onSelected, active, onOpen, onDraft,
|
||||
rows, total, counts, query, onQuery, filter, onFilter, sort, onSort, selected, onSelected, active, onOpen, onDraft, onProfile,
|
||||
}: {
|
||||
/** 筛选、排序后的行 */
|
||||
rows: CatalogTableRow[]
|
||||
@@ -35,6 +35,8 @@ export function TableIndex({
|
||||
active: string | undefined
|
||||
onOpen: (table: string) => void
|
||||
onDraft: () => void
|
||||
/** 剖析所选(数据剖析弹窗,范围默认是已选的表) */
|
||||
onProfile?: () => void
|
||||
}) {
|
||||
const radio = useRadioGroup(LIST_FILTERS, filter, onFilter)
|
||||
const listRef = useRef<HTMLUListElement>(null)
|
||||
@@ -162,6 +164,11 @@ export function TableIndex({
|
||||
<button type="button" className="btn btn-primary btn-sm" onClick={onDraft} data-catalog-draft-selected="">
|
||||
<Sparkles size={11} aria-hidden /> {CT.draftSelected(selected.size)}
|
||||
</button>
|
||||
{onProfile && (
|
||||
<button type="button" className="btn btn-sm" onClick={onProfile} data-catalog-profile-selected="">
|
||||
<ScanSearch size={11} aria-hidden /> {PT.profileSelected(selected.size)}
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
@@ -205,15 +205,18 @@ function codesText(v: Record<string, string>): string {
|
||||
return Object.entries(v).map(([k, x]) => `${k}=${x}`).join('\n')
|
||||
}
|
||||
|
||||
/** 码值的文字:每行一个「码值=含义」(全角等号也认)。返回解析结果和第一处错误 */
|
||||
/**
|
||||
* 码值的文字:每行一个「码值=含义」(全角等号也认)。含义可以空着(「2=」):数据剖析只知道列里出现过哪些取值,
|
||||
* 含义等人填,服务端也收空含义。返回解析结果和第一处错误
|
||||
*/
|
||||
export function parseCodes(text: string): { value: Record<string, string>; error: { line: number } | { dup: string } | null } {
|
||||
const value: Record<string, string> = {}
|
||||
const lines = text.split('\n')
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i].trim()
|
||||
if (!line) continue
|
||||
const m = line.match(/^(.+?)\s*[==]\s*(.+)$/)
|
||||
if (!m) return { value, error: { line: i + 1 } }
|
||||
const m = line.match(/^(.+?)\s*[==]\s*(.*)$/)
|
||||
if (!m || !m[1].trim()) return { value, error: { line: i + 1 } }
|
||||
const code = m[1].trim()
|
||||
if (code in value) return { value, error: { dup: code } }
|
||||
value[code] = m[2].trim()
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
import type { CatalogProfileNumberKey, CatalogProfileSettings } from '../../types'
|
||||
import { PROFILE_FIELD_LABEL, PROFILE_TEXT } from '../../lib/terms'
|
||||
|
||||
// ===========================================================================
|
||||
// 数据剖析的设置:数据源 options.catalog_profile 和表单之间的来回转换、校验、保存被拒时认出是哪一项。
|
||||
// 规则跟着服务端(backend/app/data/catalog_profile.py 的 ProfileSettings、profile_settings、
|
||||
// profile_settings_problem)走:缺省即关闭;读的时候宽松(某一项不对就用缺省),写的时候严格(超出范围的拒收)。
|
||||
// ===========================================================================
|
||||
|
||||
/** 数据源 options 里剖析设置的键 */
|
||||
export const PROFILE_OPTION = 'catalog_profile'
|
||||
|
||||
export const PROFILE_NUMBER_KEYS: readonly CatalogProfileNumberKey[] = [
|
||||
'max_queries', 'query_timeout_s', 'sample_size', 'max_scan_rows', 'max_total_s',
|
||||
]
|
||||
|
||||
/** 缺省值(服务端 ProfileSettings 的默认) */
|
||||
export const PROFILE_DEFAULTS: CatalogProfileSettings = {
|
||||
enabled: false, max_queries: 60, query_timeout_s: 10, sample_size: 2000, max_scan_rows: 100_000, max_total_s: 120,
|
||||
}
|
||||
|
||||
/** 各数值项的范围(服务端 _NUMBER_FIELDS):下限、上限、是否必须是整数 */
|
||||
export const PROFILE_LIMITS: Record<CatalogProfileNumberKey, { min: number; max: number; integer: boolean }> = {
|
||||
max_queries: { min: 1, max: 500, integer: true },
|
||||
query_timeout_s: { min: 1, max: 60, integer: false },
|
||||
sample_size: { min: 10, max: 10_000, integer: true },
|
||||
max_scan_rows: { min: 0, max: 10_000_000, integer: true },
|
||||
max_total_s: { min: 10, max: 600, integer: false },
|
||||
}
|
||||
|
||||
/** 一个数:数字和数字写成的文字都认(同服务端 _number)。不合规返回 null */
|
||||
function numberIn(value: unknown, key: CatalogProfileNumberKey): number | null {
|
||||
if (value == null || typeof value === 'boolean') return null
|
||||
const text = String(value).trim().replace(/,/g, '')
|
||||
if (!text) return null
|
||||
const n = Number(text)
|
||||
const { min, max, integer } = PROFILE_LIMITS[key]
|
||||
if (!Number.isFinite(n) || n < min || n > max) return null
|
||||
if (integer && !Number.isInteger(n)) return null
|
||||
return n
|
||||
}
|
||||
|
||||
const rawOf = (options: Record<string, unknown> | null | undefined): Record<string, unknown> | null => {
|
||||
const raw = options?.[PROFILE_OPTION]
|
||||
return raw && typeof raw === 'object' && !Array.isArray(raw) ? raw as Record<string, unknown> : null
|
||||
}
|
||||
|
||||
/** options → 生效的设置。宽松:某一项不对用缺省;开关只认 true */
|
||||
export function profileSettingsOf(options: Record<string, unknown> | null | undefined): CatalogProfileSettings {
|
||||
const raw = rawOf(options)
|
||||
const out: CatalogProfileSettings = { ...PROFILE_DEFAULTS, enabled: raw?.enabled === true }
|
||||
if (!raw) return out
|
||||
for (const key of PROFILE_NUMBER_KEYS) {
|
||||
const n = numberIn(raw[key], key)
|
||||
if (n != null) out[key] = n
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 表单:数值项是文字(输入时允许写一半),空着表示用缺省
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface ProfileForm {
|
||||
enabled: boolean
|
||||
values: Record<CatalogProfileNumberKey, string>
|
||||
}
|
||||
|
||||
export function profileFormOf(options: Record<string, unknown> | null | undefined): ProfileForm {
|
||||
const raw = rawOf(options)
|
||||
const values = Object.fromEntries(PROFILE_NUMBER_KEYS.map((k) => {
|
||||
const v = raw?.[k]
|
||||
return [k, v == null || typeof v === 'object' ? '' : String(v)]
|
||||
})) as Record<CatalogProfileNumberKey, string>
|
||||
return { enabled: raw?.enabled === true, values }
|
||||
}
|
||||
|
||||
/** options 里已经有剖析设置这一项 */
|
||||
export const hasProfileOption = (options: Record<string, unknown> | null | undefined): boolean => !!rawOf(options)
|
||||
|
||||
/**
|
||||
* 表单 → 要写进 options 的设置。原来没有(stored 为假)、现在也没开、一项都没填的不写(不给 options 平白多一个键);
|
||||
* 其余写开关和填了的项。填的不是数时原样交上去,由服务端拒收并说明是哪一项(表单上已经先拦过一次)
|
||||
*/
|
||||
export function profileOptionOf(form: ProfileForm, stored: boolean): Record<string, unknown> | undefined {
|
||||
const filled = PROFILE_NUMBER_KEYS.filter((k) => form.values[k].trim())
|
||||
if (!stored && !form.enabled && !filled.length) return undefined
|
||||
const out: Record<string, unknown> = { enabled: form.enabled }
|
||||
for (const k of filled) {
|
||||
const text = form.values[k].trim().replace(/,/g, '')
|
||||
const n = Number(text)
|
||||
out[k] = Number.isFinite(n) ? n : form.values[k].trim()
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/** 表单的格式问题:键 → 说明。空对象表示可以保存 */
|
||||
export function profileProblems(form: ProfileForm): Partial<Record<CatalogProfileNumberKey, string>> {
|
||||
const out: Partial<Record<CatalogProfileNumberKey, string>> = {}
|
||||
for (const k of PROFILE_NUMBER_KEYS) {
|
||||
if (!form.values[k].trim()) continue
|
||||
if (numberIn(form.values[k], k) == null) {
|
||||
const { min, max, integer } = PROFILE_LIMITS[k]
|
||||
out[k] = PROFILE_TEXT.rangeError(min, max, integer)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/** 表单此刻对应的设置:没填、填错的项按缺省(风险说明据此写出「最多 N 条」) */
|
||||
export function profileSettingsOfForm(form: ProfileForm): CatalogProfileSettings {
|
||||
const out: CatalogProfileSettings = { ...PROFILE_DEFAULTS, enabled: form.enabled }
|
||||
for (const k of PROFILE_NUMBER_KEYS) {
|
||||
const n = numberIn(form.values[k], k)
|
||||
if (n != null) out[k] = n
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
export const sameProfileForm = (a: ProfileForm, b: ProfileForm): boolean =>
|
||||
a.enabled === b.enabled && PROFILE_NUMBER_KEYS.every((k) => a.values[k] === b.values[k])
|
||||
|
||||
/**
|
||||
* 服务端保存时拒收的 422(一句中文)说的是剖析设置的哪一项:
|
||||
* 「数据剖析的「查询次数上限」需要填写 1 到 500 之间的整数;当前为「0」」→ max_queries。
|
||||
* 说的是剖析设置、但认不出哪一项(格式不对、有认不出的项)返回 'section';说的不是剖析设置返回 null
|
||||
*/
|
||||
export function profileFieldOfRejection(message: string): 'enabled' | CatalogProfileNumberKey | 'section' | null {
|
||||
if (!message.includes('数据剖析')) return null
|
||||
const named = message.match(/「([^」]+)」/)?.[1] ?? ''
|
||||
for (const k of ['enabled', ...PROFILE_NUMBER_KEYS] as const) {
|
||||
if (named && named.startsWith(PROFILE_FIELD_LABEL[k])) return k
|
||||
}
|
||||
return 'section'
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 剖析:范围、报告里的读数、409 的分类
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** 不指定表时服务端剖析几张(PROFILE_DEFAULT_TABLES) */
|
||||
export const PROFILE_DEFAULT_TABLES = 10
|
||||
/** 一次最多指定几张表(接口 tables 的 max_length) */
|
||||
export const PROFILE_MAX_TABLES = 200
|
||||
|
||||
/** 覆盖率写成百分数:最多一位小数,不到 100% 的不进成 100%(同服务端 _percent) */
|
||||
export function coverageText(ratio: number): string {
|
||||
const text = (ratio * 100).toFixed(1).replace(/\.0$/, '')
|
||||
return text === '100' && ratio < 1 ? '99.9%' : `${text}%`
|
||||
}
|
||||
|
||||
/** 已用时长的读数:「不到 1 秒」「8 秒」「1 分 05 秒」 */
|
||||
export function secondsText(ms: number): string {
|
||||
const s = Math.max(0, Math.floor(ms / 1000))
|
||||
if (s < 1) return '不到 1 秒'
|
||||
return s < 60 ? `${s} 秒` : `${Math.floor(s / 60)} 分 ${String(s % 60).padStart(2, '0')} 秒`
|
||||
}
|
||||
|
||||
/** 剖析接口回 409 的几种情况(按服务端的原话认:catalog_profile.ensure_enabled、api/catalog.py、engine.SnapshotTampered) */
|
||||
export type ProfileBlock = 'disabled' | 'inactive' | 'noSchema' | 'busy' | 'tampered' | 'other'
|
||||
|
||||
export function profileBlockOf(message: string): ProfileBlock {
|
||||
if (message.includes('未开启数据剖析')) return 'disabled'
|
||||
if (message.includes('已停用')) return 'inactive'
|
||||
if (message.includes('正在进行数据剖析')) return 'busy'
|
||||
if (message.includes('探查结构')) return 'noSchema'
|
||||
if (/已拒绝(?:这次)?查询/.test(message)) return 'tampered'
|
||||
return 'other'
|
||||
}
|
||||
|
||||
/** 设置里有没有剖析设置这一项(options 的其余键原样带上) */
|
||||
export function withProfileOption(options: Record<string, unknown> | null | undefined, value: Record<string, unknown> | undefined): Record<string, unknown> {
|
||||
const { [PROFILE_OPTION]: _old, ...rest } = options ?? {}
|
||||
return value ? { ...rest, [PROFILE_OPTION]: value } : rest
|
||||
}
|
||||
@@ -14,13 +14,16 @@ import { splitShrunk } from '../canvas/copilotMerge'
|
||||
import { confirmDialog, IconButton, StatusBadge, toast } from '../components/ui'
|
||||
import { formatCost, formatLapse, formatOffset, formatSpan, formatTokens, shortId, NONE } from '../lib/format'
|
||||
import { statusLabel } from '../lib/status'
|
||||
import { runClassLabel } from '../lib/terms'
|
||||
import { runClassLabel, SQL_CHECK_TEXT } from '../lib/terms'
|
||||
import { sortSqlChecks } from '../lib/sqlcheck'
|
||||
import type { RunUsage, ToolChange } from '../types'
|
||||
import { AssistantStream, type StreamTurn } from './AssistantStream'
|
||||
import { Composer } from './Composer'
|
||||
import { SqlCheckBody } from './SqlChecks'
|
||||
import { EvidenceHostContext, type EvidenceHost } from './evidenceHost'
|
||||
import {
|
||||
COPILOT_PHASE_TEXT, copilotOutcome, decodeCopilot, decodeRun, exitLabels, issueLine, SELF_CHECK_ROUNDS, type CopilotIssue, type CopilotOutcome,
|
||||
COPILOT_PHASE_TEXT, copilotOutcome, decodeCopilot, decodeRun, exitLabels, issueLine, issueSqlCheck, SELF_CHECK_ROUNDS, type CopilotIssue,
|
||||
type CopilotOutcome,
|
||||
} from './decode'
|
||||
import { catalogPatchesOf } from './catalogPatch'
|
||||
import { ApprovalCard } from './RunPanel'
|
||||
@@ -289,11 +292,13 @@ function ChatView({ hasRun, onOpenRun }: { hasRun: boolean; onOpenRun: () => voi
|
||||
)
|
||||
}
|
||||
const issues = out.check?.status === 'failed' ? out.check.issues : []
|
||||
// 对照数据目录的 SQL 检查:不挡运行的提醒、提示(错误交回模型改过,没改好的在上面那组)
|
||||
const sqlNotes = sortSqlChecks(out.sqlChecks)
|
||||
const { unasked, asked } = toolLossOf(out)
|
||||
// 画布一处没变(unchanged)就没有可撤的:给了按钮,撤掉的是这一轮之前的那一步
|
||||
const undo = source.outcome === 'applied'
|
||||
|| (out.kind === 'built' && source.outcome !== 'reverted' && source.outcome !== 'unchanged')
|
||||
if (!issues.length && !out.skipped.length && !unasked.length && !asked.length) {
|
||||
if (!issues.length && !out.skipped.length && !unasked.length && !asked.length && !sqlNotes.length) {
|
||||
return undo ? (
|
||||
<button type="button" className="btn btn-xs btn-ghost" onClick={() => undoCopilotTurn(source.id)}>
|
||||
<Undo2 size={11} aria-hidden /> 撤销这次生成
|
||||
@@ -306,6 +311,10 @@ function ChatView({ hasRun, onOpenRun }: { hasRun: boolean; onOpenRun: () => voi
|
||||
<div className="w-full space-y-1.5">
|
||||
{/* 没修好的问题直接摊开,不折叠:这是此刻唯一要做的事 */}
|
||||
{issues.length > 0 && <IssueList issues={issues} onLocate={locateIssue} />}
|
||||
{sqlNotes.length > 0 && (
|
||||
<IssueList issues={sqlNotes} onLocate={locateIssue} title={SQL_CHECK_TEXT.title}
|
||||
plain={sqlNotes.every((x) => x.level === 'info')} />
|
||||
)}
|
||||
{(unasked.length > 0 || asked.length > 0) && <ToolLossList unasked={unasked} asked={asked} onLocate={locateTools} />}
|
||||
<div className="flex flex-wrap items-center gap-1.5">
|
||||
{repairable && (
|
||||
@@ -380,28 +389,40 @@ function ChatView({ hasRun, onOpenRun }: { hasRun: boolean; onOpenRun: () => voi
|
||||
)
|
||||
}
|
||||
|
||||
/** 自查没修好的问题,逐条带「定位」。问题落在哪个节点上认得出来才给;写节点名,认不出才写 id */
|
||||
function IssueList({ issues, onLocate }: { issues: CopilotIssue[]; onLocate: (x: CopilotIssue) => void }) {
|
||||
/**
|
||||
* 自查没修好的问题,逐条带「定位」。问题落在哪个节点上认得出来才给;写节点名,认不出才写 id。
|
||||
* 对照数据目录的 SQL 检查另画级别、中文规则名、涉及的表和列、SQL 片段(和证据面板同一套),「定位」落到 SQL 那一项。
|
||||
* title 有值时是不挡运行的那一组(SQL 检查的提醒、提示):全是提示时不用提醒色
|
||||
*/
|
||||
function IssueList({ issues, onLocate, title, plain = false }: {
|
||||
issues: CopilotIssue[]; onLocate: (x: CopilotIssue) => void; title?: string; plain?: boolean
|
||||
}) {
|
||||
return (
|
||||
<ul data-issue-list="" className="space-y-0.5 rounded border px-2 py-1.5 text-2xs leading-relaxed"
|
||||
style={{ borderColor: 'color-mix(in srgb, var(--st-waiting) 45%, var(--border))',
|
||||
background: 'var(--st-waiting-soft)' }}>
|
||||
{issues.map((x, i) => (
|
||||
<li key={i} className="flex items-start gap-1.5" data-issue-node={x.nodeId} data-issue-field={x.field}>
|
||||
<span className="min-w-0 flex-1 [overflow-wrap:anywhere]" title={issueLine(x)}>
|
||||
{x.nodeId && <span className={clsx('text-dim', !x.label && 'mono')}>「{x.label ?? x.nodeId}」</span>}{x.message}
|
||||
</span>
|
||||
{x.nodeId && (
|
||||
<button type="button"
|
||||
className="inline-flex shrink-0 items-center gap-0.5 rounded px-1 text-dim transition-colors hover:bg-hover hover:text-fg"
|
||||
title={x.field ? '打开该节点的设置,定位到出问题的配置项' : '在画布上定位这个节点'}
|
||||
onClick={() => onLocate(x)}>
|
||||
<Crosshair size={10} aria-hidden /> 定位
|
||||
</button>
|
||||
)}
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
<div data-issue-list={title ? 'sql' : ''} className="rounded border px-2 py-1.5 text-2xs leading-relaxed"
|
||||
style={plain ? undefined : { borderColor: 'color-mix(in srgb, var(--st-waiting) 45%, var(--border))', background: 'var(--st-waiting-soft)' }}>
|
||||
{title && <div className="mb-1 font-medium text-fg">{title}</div>}
|
||||
<ul className="space-y-1">
|
||||
{issues.map((x, i) => {
|
||||
const sql = issueSqlCheck(x)
|
||||
return (
|
||||
<li key={i} className="flex items-start gap-1.5" data-issue-node={x.nodeId} data-issue-field={x.field} data-issue-code={x.code}>
|
||||
<div className="min-w-0 flex-1 [overflow-wrap:anywhere]" title={issueLine(x)}>
|
||||
{x.nodeId && <span className={clsx('text-dim', !x.label && 'mono')}>「{x.label ?? x.nodeId}」</span>}
|
||||
{sql ? <div className="mt-0.5"><SqlCheckBody check={sql} /></div> : x.message}
|
||||
</div>
|
||||
{x.nodeId && (
|
||||
<button type="button"
|
||||
className="inline-flex shrink-0 items-center gap-0.5 rounded px-1 text-dim transition-colors hover:bg-hover hover:text-fg"
|
||||
title={x.field ? '打开该节点的设置,定位到出问题的配置项' : '在画布上定位这个节点'}
|
||||
onClick={() => onLocate(x)}>
|
||||
<Crosshair size={10} aria-hidden /> 定位
|
||||
</button>
|
||||
)}
|
||||
</li>
|
||||
)
|
||||
})}
|
||||
</ul>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ import { CatalogImpactList } from '../components/CatalogImpact'
|
||||
import { CATALOG_PATCH_TEXT as T, CATALOG_STATUS_LABEL } from '../lib/terms'
|
||||
import type { CatalogPatchChange } from '../types'
|
||||
import {
|
||||
catalogTableHref, patchSubmit, patchTarget, patchValueText, patchWhere, type CatalogPatch,
|
||||
catalogTableHref, codeMeaning, patchSubmit, patchTarget, patchValueText, patchWhere, type CatalogPatch,
|
||||
} from './catalogPatch'
|
||||
|
||||
// ===========================================================================
|
||||
@@ -211,8 +211,9 @@ function CodesAfter({ before, after }: { before: unknown; after: unknown }) {
|
||||
{entries.map(([k, v], i) => (
|
||||
<span key={k}>
|
||||
{i > 0 && '、'}
|
||||
<span className={clsx(prev[k] !== v && 'font-medium text-[var(--accent)]')} data-code-changed={prev[k] !== v ? k : undefined}>
|
||||
{k}={v}
|
||||
<span className={clsx(prev[k] !== v && 'font-medium text-[var(--accent)]', !String(v ?? '').trim() && 'text-faint')}
|
||||
data-code-changed={prev[k] !== v ? k : undefined}>
|
||||
{k}={codeMeaning(v)}
|
||||
</span>
|
||||
</span>
|
||||
))}
|
||||
|
||||
@@ -18,7 +18,7 @@ import {
|
||||
} from '../lib/evidence'
|
||||
import { humanizeError } from '../lib/errors'
|
||||
import { formatDateTime, formatNumber, NONE, shortId } from '../lib/format'
|
||||
import { EVIDENCE_TEXT, JUDGE_TEXT, MERGE_TEXT, NODE_TYPE_LABEL, nodeTypeLabel } from '../lib/terms'
|
||||
import { EVIDENCE_TEXT, JUDGE_TEXT, MERGE_TEXT, NODE_TYPE_LABEL, SQL_CHECK_TEXT, nodeTypeLabel } from '../lib/terms'
|
||||
import { useCatalog } from '../store/catalog'
|
||||
import { askKey, segmentKey, useEvidence, useExplore, useVerdicts } from '../store/evidence'
|
||||
import type {
|
||||
@@ -29,6 +29,7 @@ import { ArtifactViewer, ResultTable } from './AssistantStream'
|
||||
import { useEvidenceHost } from './evidenceHost'
|
||||
import { EvidenceProvenance } from './EvidenceProvenance'
|
||||
import { CopyChip } from './Markdown'
|
||||
import { SqlCheckList } from './SqlChecks'
|
||||
|
||||
/**
|
||||
* 证据面板:点开报告里的一个片段,看它从哪来。
|
||||
@@ -1109,6 +1110,9 @@ function MetricPart({ step, entry, inputs, runId, seal, pending, failed, resolve
|
||||
// 拿截断的查询结果整组算出来的:值是真的算出来的,但只算到了取回的那部分,出具已按缺口降档
|
||||
const incomplete = step?.incomplete === true || entry?.incomplete === true
|
||||
const incompleteReason = step?.incomplete_reason || entry?.incomplete_reason || EVIDENCE_TEXT.incompleteFallback
|
||||
// 所依据的查询对照数据目录查出了错误级的问题:值照算,结果不可靠,出具同样按缺口降档。问题和改法在查询步骤里
|
||||
const sqlFailed = step?.sql_check_failed === true || entry?.sql_check_failed === true
|
||||
const sqlReason = step?.sql_check_reason || entry?.sql_check_reason || EVIDENCE_TEXT.sqlCheckFallback
|
||||
|
||||
return (
|
||||
<Part title={EVIDENCE_TEXT.metric} data-ev-metric="">
|
||||
@@ -1121,6 +1125,12 @@ function MetricPart({ step, entry, inputs, runId, seal, pending, failed, resolve
|
||||
{EVIDENCE_TEXT.incomplete}
|
||||
</span>
|
||||
)}
|
||||
{sqlFailed && (
|
||||
<span className="chip" data-ev-sql-check-failed="" title={sqlReason}
|
||||
style={{ color: 'var(--st-waiting)', borderColor: 'var(--st-waiting)' }}>
|
||||
{EVIDENCE_TEXT.sqlCheckFailed}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
{(caliber || version || artifact || from) && (
|
||||
<div className="mt-0.5 flex flex-wrap items-center gap-x-2 text-2xs text-dim">
|
||||
@@ -1139,6 +1149,9 @@ function MetricPart({ step, entry, inputs, runId, seal, pending, failed, resolve
|
||||
{incomplete && (
|
||||
<p className="mt-1" style={{ color: 'var(--st-waiting)' }} data-ev-incomplete-reason="">{incompleteReason}</p>
|
||||
)}
|
||||
{sqlFailed && (
|
||||
<p className="mt-1" style={{ color: 'var(--st-waiting)' }} data-ev-sql-check-reason="">{sqlReason}</p>
|
||||
)}
|
||||
{unshowable && (
|
||||
<p className="mt-1" style={{ color: 'var(--st-waiting)' }} data-ev-unshowable="">
|
||||
{EVIDENCE_TEXT.unshowable(valueText(value))}
|
||||
@@ -1366,6 +1379,13 @@ function QueryPart({ id, step, seal, masked }: { id: string; step: EvidenceStep;
|
||||
{step.sql}
|
||||
</pre>
|
||||
)}
|
||||
{!!step.checks?.length && (
|
||||
// 这次查询对照数据目录查出的问题:说明、涉及的表和列、SQL 片段。没查出问题时接口不给这个键
|
||||
<div className="mb-1.5" data-ev-sql-checks="">
|
||||
<div className="mb-0.5 text-2xs text-faint">{SQL_CHECK_TEXT.title}</div>
|
||||
<SqlCheckList checks={step.checks} />
|
||||
</div>
|
||||
)}
|
||||
{!!step.merge?.warnings.length && (
|
||||
<div className="mb-1.5 text-2xs" data-ev-merge-warnings="">
|
||||
<div className="mb-0.5 text-faint">{MERGE_TEXT.executionWarnings}</div>
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
import { Info, OctagonX, TriangleAlert } from 'lucide-react'
|
||||
import type { LucideIcon } from 'lucide-react'
|
||||
import clsx from 'clsx'
|
||||
import type { SqlCheckItem, SqlCheckLevel } from '../types'
|
||||
import { SQL_CHECK_LEVEL_HINT, SQL_CHECK_LEVEL_LABEL, SQL_CHECK_RULE_HINT, SQL_CHECK_TEXT } from '../lib/terms'
|
||||
import { SQL_CHECK_TONE, isSqlCheckCode, sortSqlChecks, sqlCheckLevelOf, sqlCheckWhere, sqlRuleLabel } from '../lib/sqlcheck'
|
||||
|
||||
// ===========================================================================
|
||||
// SQL 检查的结果:级别标识(错误、提醒、提示,四通道:图标、文字、颜色、边框)、中文规则名、给人看的说明、涉及的表和列、
|
||||
// SQL 片段。证据面板的查询步骤和助手面板共用这一份。
|
||||
// ===========================================================================
|
||||
|
||||
const LEVEL_ICON: Record<SqlCheckLevel, LucideIcon> = { error: OctagonX, warning: TriangleAlert, info: Info }
|
||||
|
||||
/** 级别标识:图标 + 「错误 / 提醒 / 提示」,悬停写明依据的确证程度 */
|
||||
export function SqlLevelBadge({ level: raw, className }: { level: unknown; className?: string }) {
|
||||
const level = sqlCheckLevelOf(raw)
|
||||
const Icon = LEVEL_ICON[level]
|
||||
const tone = SQL_CHECK_TONE[level]
|
||||
return (
|
||||
<span className={clsx('inline-flex shrink-0 items-center gap-0.5 whitespace-nowrap rounded-full border px-1.5 text-2xs leading-[16px]', className)}
|
||||
style={{ color: tone, borderColor: `color-mix(in srgb, ${tone} 45%, var(--border))`, background: `color-mix(in srgb, ${tone} 8%, transparent)` }}
|
||||
title={SQL_CHECK_LEVEL_HINT[level]} data-sql-level={level}>
|
||||
<Icon size={10} aria-hidden />
|
||||
{SQL_CHECK_LEVEL_LABEL[level]}
|
||||
</span>
|
||||
)
|
||||
}
|
||||
|
||||
/** 一条检查的正文:规则名、说明、涉及的表和列、SQL 片段 */
|
||||
export function SqlCheckBody({ check: c, excerpt = true }: { check: SqlCheckItem; excerpt?: boolean }) {
|
||||
const where = sqlCheckWhere(c)
|
||||
return (
|
||||
<>
|
||||
<div className="flex min-w-0 flex-wrap items-center gap-x-1.5 gap-y-0.5">
|
||||
<SqlLevelBadge level={c.level} />
|
||||
<span className="font-medium text-fg" title={isSqlCheckCode(c.code) ? SQL_CHECK_RULE_HINT[c.code] : undefined} data-sql-rule="">
|
||||
{sqlRuleLabel(c.code)}
|
||||
</span>
|
||||
{where && (
|
||||
<span className="text-faint" data-sql-where={where}>
|
||||
{SQL_CHECK_TEXT.where} <span className="mono text-dim [overflow-wrap:anywhere]">{where}</span>
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
{c.message && <p className="mt-0.5 leading-relaxed text-dim [overflow-wrap:anywhere]" data-sql-message="">{c.message}</p>}
|
||||
{excerpt && c.sql_excerpt && (
|
||||
<pre className="mono mt-1 max-h-20 overflow-auto whitespace-pre-wrap rounded bg-bg px-1.5 py-1 leading-relaxed text-dim [overflow-wrap:anywhere]"
|
||||
aria-label={SQL_CHECK_TEXT.excerpt} title={SQL_CHECK_TEXT.excerpt} data-sql-excerpt="">
|
||||
{c.sql_excerpt}
|
||||
</pre>
|
||||
)}
|
||||
</>
|
||||
)
|
||||
}
|
||||
|
||||
/** 检查结果的清单:错误在前。每条一个淡色框,框的颜色跟级别走 */
|
||||
export function SqlCheckList({ checks, className }: { checks: SqlCheckItem[]; className?: string }) {
|
||||
const list = sortSqlChecks(checks)
|
||||
return (
|
||||
<ul className={clsx('space-y-1 text-2xs', className)} aria-label={SQL_CHECK_TEXT.listLabel(list.length)} data-sql-checks={list.length}>
|
||||
{list.map((c, i) => {
|
||||
const tone = SQL_CHECK_TONE[sqlCheckLevelOf(c.level)]
|
||||
return (
|
||||
<li key={`${c.code}:${c.table}:${c.column ?? ''}:${i}`} className="rounded border px-2 py-1.5"
|
||||
style={{ borderColor: `color-mix(in srgb, ${tone} 30%, var(--border))`, background: `color-mix(in srgb, ${tone} 5%, transparent)` }}
|
||||
data-sql-check={c.code} data-level={sqlCheckLevelOf(c.level)}>
|
||||
<SqlCheckBody check={c} />
|
||||
</li>
|
||||
)
|
||||
})}
|
||||
</ul>
|
||||
)
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
import type { CatalogBusinessDate, CatalogCardinality, CatalogMeasure, CatalogPatchChange, CatalogTableKind } from '../types'
|
||||
import {
|
||||
CATALOG_CARDINALITY_LABEL, CATALOG_COLUMN_FIELD_LABEL, CATALOG_KIND_LABEL, CATALOG_MEASURE_LABEL, CATALOG_PATCH_TEXT,
|
||||
CATALOG_TABLE_FIELD_LABEL, CATALOG_TEXT,
|
||||
CATALOG_TABLE_FIELD_LABEL, CATALOG_TEXT, CODES_TEXT,
|
||||
} from '../lib/terms'
|
||||
|
||||
// ===========================================================================
|
||||
@@ -108,10 +108,13 @@ export function relationText(v: unknown): string {
|
||||
return `${left} → ${right}${card ? `(${card})` : ''}`
|
||||
}
|
||||
|
||||
/** 码值写成一行:1=有效、9=作废 */
|
||||
/** 码值的含义:空串是数据剖析写进来的候选,含义还要人填,和目录页同一个说法 */
|
||||
export const codeMeaning = (x: unknown) => (String(x ?? '').trim() ? String(x) : CODES_TEXT.pending)
|
||||
|
||||
/** 码值写成一行:1=有效、9=作废、8=含义待填写 */
|
||||
export function codesText(v: unknown): string {
|
||||
if (!v || typeof v !== 'object') return String(v)
|
||||
return Object.entries(v as Record<string, string>).map(([k, x]) => `${k}=${x}`).join('、')
|
||||
return Object.entries(v as Record<string, string>).map(([k, x]) => `${k}=${codeMeaning(x)}`).join('、')
|
||||
}
|
||||
|
||||
/** 一项的值写成一行字(过程里的明细、卡片里的改前改后都用它)。没有值写「未填写」 */
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
import type { CatalogDriftTable, RunEvent, TeamMember, TeamRound, TeamRun, ToolChange } from '../types'
|
||||
import type { CatalogDriftTable, RunEvent, SqlCheckItem, TeamMember, TeamRound, TeamRun, ToolChange } from '../types'
|
||||
import type { NodeState, RunPhase } from './trace'
|
||||
import { formatDuration, formatNumber } from '../lib/format'
|
||||
import {
|
||||
CATALOG_DRIFT_TEXT, CATALOG_PATCH_TEXT, JUDGE_TEXT, MERGE_TEXT, TYPE_LABEL, claimTally, evidenceTally, issuanceLabel, nodeTypeLabel,
|
||||
CATALOG_DRIFT_TEXT, CATALOG_PATCH_TEXT, JUDGE_TEXT, MERGE_TEXT, SQL_CHECK_TEXT, TYPE_LABEL, claimTally, evidenceTally,
|
||||
issuanceLabel, nodeTypeLabel,
|
||||
} from '../lib/terms'
|
||||
import { catalogPatchOf, patchValueText, patchWhere } from './catalogPatch'
|
||||
import { claimCountsOf, isJudged, statsTally } from '../lib/evidence'
|
||||
import { isSqlCheckCode, sortSqlChecks, sqlCheckWhere, sqlRuleLabel } from '../lib/sqlcheck'
|
||||
|
||||
// 泳道数据画布也要用(supervisor 节点要展开成协作矩阵),所以类型放在
|
||||
// types.ts 里;这里再导出一遍,老引用不用改
|
||||
@@ -778,6 +780,20 @@ function explainLog(code: string | undefined, message: string): Pick<Step, 'titl
|
||||
fix: 'canvas',
|
||||
}
|
||||
}
|
||||
case 'metric_sql_check': {
|
||||
// 「指标「客单价」所依据的查询未通过 SQL 检查(「订单」关联「订单明细」是一对多,对「订单」的「订单金额」求和会
|
||||
// 重复计算),结果不可靠」;几处时是「所依据的查询有 2 处未通过 SQL 检查(…等),结果不可靠」(metrics.py
|
||||
// _sql_check_reason)。括号里的问题本身可能带括号:取到「,结果不可靠」前面最后一个右括号
|
||||
const name = message.match(/^指标「(.+?)」/)?.[1]
|
||||
const n = message.match(/有\s*(\d+)\s*处未通过 SQL 检查/)?.[1]
|
||||
const why = message.match(/未通过 SQL 检查(([\s\S]+))[,,]\s*结果不可靠\s*$/)?.[1]
|
||||
return {
|
||||
title: name ? SQL_CHECK_TEXT.metricTitle(name, n ? Number(n) : 1) : SQL_CHECK_TEXT.metricTitleAnon,
|
||||
sub: why,
|
||||
next: SQL_CHECK_TEXT.metricNext,
|
||||
fix: 'canvas',
|
||||
}
|
||||
}
|
||||
case 'judge_limit': {
|
||||
// 「结论句裁判已到上限(这份报告的裁判金额上限 $0.05):3 句没判,记为未裁判;已判的保留」(judge.py run_request)
|
||||
const n = message.match(/[::]\s*(\d+)\s*句(?:没判|未裁判)/)?.[1]
|
||||
@@ -2452,8 +2468,14 @@ export interface CopilotIssue {
|
||||
label?: string
|
||||
/** 落在节点的哪一项配置上(condition、tools、agents[1].tools):「定位」据此聚焦到检查器里那个输入框 */
|
||||
field?: string
|
||||
/** 机读代号:datasource_out_of_scope、tools_dropped…… */
|
||||
/** 机读代号:datasource_out_of_scope、tools_dropped……;SQL 检查是规则编号(fanout_sum……) */
|
||||
code?: string
|
||||
/** error / warning / info。老会话、老后端不给 */
|
||||
level?: 'error' | 'warning' | 'info'
|
||||
/** SQL 检查的问题另带:涉及的表和列、SQL 片段 */
|
||||
table?: string
|
||||
column?: string
|
||||
sqlExcerpt?: string
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -2468,11 +2490,16 @@ function parseIssue(v: unknown, labels?: (id: string) => string | undefined): Co
|
||||
}
|
||||
if (v && typeof v === 'object') {
|
||||
const o = v as Record<string, unknown>
|
||||
const str = (x: unknown) => (typeof x === 'string' && x ? x : undefined)
|
||||
return named({
|
||||
message: String(o.message ?? ''),
|
||||
...(o.node_id ? { nodeId: String(o.node_id) } : {}),
|
||||
...(typeof o.field === 'string' && o.field ? { field: o.field } : {}),
|
||||
...(typeof o.code === 'string' && o.code ? { code: o.code } : {}),
|
||||
...(str(o.field) ? { field: str(o.field) } : {}),
|
||||
...(str(o.code) ? { code: str(o.code) } : {}),
|
||||
...(o.level === 'error' || o.level === 'warning' || o.level === 'info' ? { level: o.level } : {}),
|
||||
...(str(o.table) ? { table: str(o.table) } : {}),
|
||||
...(str(o.column) ? { column: str(o.column) } : {}),
|
||||
...(str(o.sql_excerpt) ? { sqlExcerpt: str(o.sql_excerpt) } : {}),
|
||||
})
|
||||
}
|
||||
const text = String(v ?? '')
|
||||
@@ -2480,9 +2507,37 @@ function parseIssue(v: unknown, labels?: (id: string) => string | undefined): Co
|
||||
return named(m ? { nodeId: m[1], message: m[2] || text } : { message: text })
|
||||
}
|
||||
|
||||
/** 一条自查问题写成一行:带节点名(认得出时),认不出才退到 id */
|
||||
export const issueLine = (x: CopilotIssue): string =>
|
||||
x.nodeId ? `「${x.label ?? x.nodeId}」${x.message}` : x.message
|
||||
/**
|
||||
* 一条自查问题写成一行:带节点名(认得出时),认不出才退到 id。SQL 检查的问题前面写中文规则名、后面写涉及的表和列,
|
||||
* 不露规则编号
|
||||
*/
|
||||
export const issueLine = (x: CopilotIssue): string => {
|
||||
const where = isSqlCheckCode(x.code) ? sqlCheckWhere({ table: x.table ?? '', column: x.column }) : ''
|
||||
const body = isSqlCheckCode(x.code)
|
||||
? `${sqlRuleLabel(x.code)}:${x.message}${where ? `(${SQL_CHECK_TEXT.where} ${where})` : ''}`
|
||||
: x.message
|
||||
return x.nodeId ? `「${x.label ?? x.nodeId}」${body}` : body
|
||||
}
|
||||
|
||||
/** 一条自查问题按 SQL 检查读(界面上画级别、规则名、表和列、SQL 片段);不是 SQL 检查的返回 null */
|
||||
export function issueSqlCheck(x: CopilotIssue): SqlCheckItem | null {
|
||||
if (!isSqlCheckCode(x.code)) return null
|
||||
return {
|
||||
code: x.code, level: x.level ?? 'error', message: x.message, table: x.table ?? '',
|
||||
...(x.column ? { column: x.column } : {}), ...(x.sqlExcerpt ? { sql_excerpt: x.sqlExcerpt } : {}),
|
||||
}
|
||||
}
|
||||
|
||||
/** final.issues 里对照数据目录的 SQL 检查结果(不挡运行的提醒、提示也在里面);自查没修好、已经列过的错误不再算 */
|
||||
function sqlIssuesOf(ops: CopilotOp[], labels: (id: string) => string | undefined): CopilotIssue[] {
|
||||
const final = [...ops].reverse().find((o) => o.op === 'final')
|
||||
const list: unknown[] = final && Array.isArray(final.issues) ? final.issues : []
|
||||
const failed = [...ops].reverse().find((o) => o.op === 'check' && o.status === 'failed')
|
||||
const listed = new Set((failed && Array.isArray(failed.issues) ? failed.issues : [])
|
||||
.map((x: unknown) => parseIssue(x)).map((x: CopilotIssue) => `${x.nodeId ?? ''}|${x.code ?? ''}|${x.message}`))
|
||||
return list.map((x) => parseIssue(x, labels))
|
||||
.filter((x) => isSqlCheckCode(x.code) && !listed.has(`${x.nodeId ?? ''}|${x.code ?? ''}|${x.message}`))
|
||||
}
|
||||
|
||||
/**
|
||||
* 节点 id → 名字。先认这一轮操作流里加过、改过的(问数据页没有画布,只有这一份),
|
||||
@@ -2817,6 +2872,23 @@ export function decodeCopilot(ops: CopilotOp[], opts?: {
|
||||
add({ id: `cm-${i}-${t}`, seq: i, kind: 'note', status: 'done', level: 'warn',
|
||||
title: clip(`已跳过一个步骤:节点类型「${t}」不存在`, 80) })
|
||||
}
|
||||
// 对照数据目录的 SQL 检查:错误交回模型改过(没改好的在自查那一行),不挡运行的提醒、提示照样要看得见
|
||||
const sql = sqlIssuesOf(ops.slice(0, i), labels)
|
||||
if (sql.length) {
|
||||
const n = { error: 0, warning: 0, info: 0 }
|
||||
for (const x of sql) n[x.level ?? 'error'] += 1
|
||||
const nodes = [...new Set(sql.map((x) => x.nodeId).filter(Boolean))]
|
||||
add({
|
||||
id: `cs-${i}`, seq: i, kind: 'note', status: 'done', code: 'sql_check',
|
||||
...(n.error || n.warning ? { level: 'warn' as const } : {}),
|
||||
title: SQL_CHECK_TEXT.assistantTitle(SQL_CHECK_TEXT.counts(n.error, n.warning, n.info)),
|
||||
detail: sortSqlChecks(sql).map(issueLine).join('\n'),
|
||||
// 问数据页没有画布:下一步说去画布里打开这个工作流
|
||||
next: n.error || n.warning ? (canvas ? SQL_CHECK_TEXT.assistantNext : SQL_CHECK_TEXT.assistantNextChat)
|
||||
: SQL_CHECK_TEXT.assistantInfoNext,
|
||||
...(nodes.length === 1 ? { nodeId: nodes[0], ...(canvas ? { fix: 'canvas' as const } : {}) } : {}),
|
||||
})
|
||||
}
|
||||
// 工具绑定变化单独成一行:改图回执以前只说「修改 1」,模型改提示词时漏写 tools、
|
||||
// 把查库的工具整个抹掉,要到运行结果不对才发现
|
||||
const changes = toolChangesOf(op)
|
||||
@@ -2913,6 +2985,8 @@ export interface CopilotOutcome {
|
||||
toolChanges: ToolChange[]
|
||||
/** 自查的「工具被去掉了」警告(tools_dropped):这一轮的要求里没提到去掉 */
|
||||
dropped: CopilotIssue[]
|
||||
/** 对照数据目录的 SQL 检查结果(final.issues),自查没修好、已经列在 check 里的不重复 */
|
||||
sqlChecks: CopilotIssue[]
|
||||
}
|
||||
|
||||
export function copilotOutcome(
|
||||
@@ -2920,7 +2994,7 @@ export function copilotOutcome(
|
||||
): CopilotOutcome {
|
||||
const res: CopilotOutcome = {
|
||||
kind: running ? 'running' : 'empty', skipped: [], added: 0, updated: 0, removed: 0,
|
||||
toolChanges: [], dropped: [],
|
||||
toolChanges: [], dropped: [], sqlChecks: [],
|
||||
}
|
||||
const labels = labelsOf(ops, labelOf)
|
||||
for (const op of ops) {
|
||||
@@ -2952,6 +3026,7 @@ export function copilotOutcome(
|
||||
res.toolChanges = toolChangesOf(op)
|
||||
// 同一批警告 final.issues 里也有一份:自查已经带回来了就不再算一遍
|
||||
if (!res.dropped.length) res.dropped = droppedOf(op)
|
||||
res.sqlChecks = sqlIssuesOf(ops, labels)
|
||||
if (!running) res.kind = 'built'
|
||||
break
|
||||
case 'error':
|
||||
|
||||
@@ -2147,6 +2147,12 @@ export interface EvidenceStep extends Omit<EvidenceInput, 'status'> {
|
||||
incomplete?: boolean
|
||||
/** metric:不完整的原因(人话,「基于被截断的查询结果计算(只取回了前 1000 行),结果不完整」) */
|
||||
incomplete_reason?: string
|
||||
/** metric:所依据的查询没通过 SQL 检查(有错误级的问题),值照算,出具已按缺口降档 */
|
||||
sql_check_failed?: boolean
|
||||
/** metric:没通过的原因(人话,「所依据的查询未通过 SQL 检查(…),结果不可靠」) */
|
||||
sql_check_reason?: string
|
||||
/** query:这次查询对照数据目录查出的问题。没查出问题时没有这个键 */
|
||||
checks?: SqlCheckItem[]
|
||||
inputs?: EvidenceInput[]
|
||||
/** 口径卡钉在哪个工作流的哪一版(metric;方案第 5 节的写法,接口实际给在 source 上) */
|
||||
caliber_from?: EvidenceCaliberSource | null
|
||||
@@ -2786,3 +2792,138 @@ export interface CatalogImpact {
|
||||
/** 直接引用的在前 */
|
||||
templates: CatalogImpactTemplate[]
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// 数据剖析(后端 app/data/catalog_profile.py):对业务库发少量只读查询,核实推断的关联关系、取码值候选、
|
||||
// 提议业务日期。默认关闭,按数据源在 options.catalog_profile 里开启。
|
||||
// ===========================================================================
|
||||
|
||||
/** 剖析设置里的数值项 */
|
||||
export type CatalogProfileNumberKey = 'max_queries' | 'query_timeout_s' | 'sample_size' | 'max_scan_rows' | 'max_total_s'
|
||||
|
||||
/** 一个数据源的剖析设置(服务端 ProfileSettings.to_dict) */
|
||||
export interface CatalogProfileSettings {
|
||||
enabled: boolean
|
||||
max_queries: number
|
||||
query_timeout_s: number
|
||||
sample_size: number
|
||||
/** 0 表示一律不做整表统计 */
|
||||
max_scan_rows: number
|
||||
max_total_s: number
|
||||
}
|
||||
|
||||
/** 整次剖析中途停下的原因:查询次数用完、总时长用完、连续多条查询失败 */
|
||||
export type CatalogProfileStop = 'budget' | 'deadline' | 'failed'
|
||||
|
||||
/** 一张表的行数:stats 数据库的统计信息(估算);count 数到上限为止;unknown 没能得到 */
|
||||
export interface CatalogProfileSize {
|
||||
rows: number | null
|
||||
method: 'stats' | 'count' | 'unknown' | (string & {})
|
||||
/** 数到上限也没数完:至少这么多行 */
|
||||
at_least: number | null
|
||||
}
|
||||
|
||||
/** 关系的核对结论:覆盖率、基数,升为已验证(verified)或保持推断(proposed);人工确认过的只补覆盖率和基数 */
|
||||
export interface CatalogProfileRelationFinding {
|
||||
kind: 'relation'
|
||||
path: string
|
||||
target: string
|
||||
columns: string[]
|
||||
to_table: string
|
||||
to_columns: string[]
|
||||
status: CatalogStatus
|
||||
confirmed: boolean
|
||||
coverage: number
|
||||
cardinality: CatalogCardinality | null
|
||||
/** 抽了几个不同的键值、在被指向表里对上几个 */
|
||||
sample: number
|
||||
matched: number
|
||||
summary: string
|
||||
}
|
||||
|
||||
/** 码值候选:观察到的取值和行数,含义留给人填 */
|
||||
export interface CatalogProfileCodesFinding {
|
||||
kind: 'codes'
|
||||
path: string
|
||||
column: string
|
||||
values: { value: string; rows: number }[]
|
||||
rows: number
|
||||
status: CatalogStatus
|
||||
summary: string
|
||||
}
|
||||
|
||||
/** 业务日期提议:表里只有一个日期类列 */
|
||||
export interface CatalogProfileDateFinding {
|
||||
kind: 'business_date'
|
||||
path: string
|
||||
column: string
|
||||
min: string
|
||||
max: string
|
||||
status: CatalogStatus
|
||||
summary: string
|
||||
}
|
||||
|
||||
export type CatalogProfileFinding = CatalogProfileRelationFinding | CatalogProfileCodesFinding | CatalogProfileDateFinding
|
||||
|
||||
/** 没做的一项。detail 是可以直接显示的整句 */
|
||||
export interface CatalogProfileSkip {
|
||||
/** relation / codes / date / row_estimate / table */
|
||||
kind: string
|
||||
target: string
|
||||
path: string | null
|
||||
/** budget / deadline / failed / timeout / error / rejected / masked / too_large / view / unsupported / no_data / missing / high_cardinality */
|
||||
reason: string
|
||||
detail: string
|
||||
}
|
||||
|
||||
export interface CatalogProfileTable {
|
||||
table_name: string
|
||||
queries: number
|
||||
row_estimate: CatalogProfileSize | null
|
||||
findings: CatalogProfileFinding[]
|
||||
skipped: CatalogProfileSkip[]
|
||||
/** 日期类列的取值范围 */
|
||||
date_ranges: { column: string; min: string; max: string }[]
|
||||
added: number
|
||||
updated: number
|
||||
removed: number
|
||||
version: number
|
||||
/** 这张表没剖析(表结构里没有)或结果没写进去(写入一直冲突) */
|
||||
error: string | null
|
||||
}
|
||||
|
||||
/** POST /datasources/{id}/catalog/profile */
|
||||
export interface CatalogProfileOut {
|
||||
profiled_at: string
|
||||
actor: string | null
|
||||
settings: CatalogProfileSettings
|
||||
queries_used: number
|
||||
stopped: CatalogProfileStop | null
|
||||
tables: CatalogProfileTable[]
|
||||
total: { added: number; updated: number; removed: number }
|
||||
/** 一张表都没有剖析时的说明 */
|
||||
note: string | null
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// 基于数据目录的 SQL 检查(后端 app/data/sqlcheck.py):七条规则,编号稳定
|
||||
// ===========================================================================
|
||||
|
||||
export type SqlCheckCode = 'fanout_sum' | 'stock_summed' | 'join_unconfirmed' | 'ratio_aggregated' | 'missing_valid_filter'
|
||||
| 'unknown_code' | 'wrong_date_column'
|
||||
|
||||
/** error:依据有确证、结果必然有误;warning:依据有确证、很可能有误;info:依据只是推断 */
|
||||
export type SqlCheckLevel = 'error' | 'warning' | 'info'
|
||||
|
||||
/** 一条检查结果(证据接口查询步骤里的 checks、助手自查和发布前检查的问题)。交回模型改写用的那句不上界面,这里不收 */
|
||||
export interface SqlCheckItem {
|
||||
code: SqlCheckCode | (string & {})
|
||||
level: SqlCheckLevel | (string & {})
|
||||
/** 给人看的说明 */
|
||||
message: string
|
||||
/** schema_cache 里的表名 */
|
||||
table: string
|
||||
column?: string
|
||||
relation_id?: string
|
||||
sql_excerpt?: string
|
||||
}
|
||||
|
||||
@@ -4,6 +4,8 @@
|
||||
// 所以这里查的是「请求里带没带读到的版本」「批量确认只动推断项、来源不变」「编辑时没动的项原样交回(被驳回的照旧
|
||||
// 驳回)」「别人刚改过时有没有停下来让人重新载入,而不是悄悄覆盖」「起草按批调用、能停、出错时说得清」——
|
||||
// 正常路径的页面检查一个字都不会说。
|
||||
// 数据剖析(阶段 4B)会对业务库发查询:剖析接口、保存数据源一律在浏览器层答掉,查的是「确认前写没写预算」「进行中是否
|
||||
// 明说停不下来」「报告说没说清用了几条查询、为什么停、跳过了什么」「各种 409 有没有下一步」「设置被拒时报错落没落到那一项」。
|
||||
//
|
||||
// 写法同 check-versions:沙箱里的数据源多半没探查过结构,目录也是空的,所以数据源列表、目录清单、单表目录都用
|
||||
// page.route 伪造(虚构的景区业务库和一个导入表格的源);写请求(整份提交、单项审阅、起草)也在浏览器层答掉,
|
||||
@@ -400,6 +402,14 @@ function catalogHandlers(state, replies = {}) {
|
||||
const name = parts(key)[4]
|
||||
return json({ table: name, templates: state.impact?.[name] ?? [] })(route)
|
||||
})],
|
||||
// 剖析没配答复的一律 503:检查脚本不能让剖析真的发到业务库
|
||||
[/^POST \/datasources\/check-cat-[a-z]+\/catalog\/profile$/, reply('profile', json({ detail: '检查脚本没有为这次剖析准备答复' }, 503))],
|
||||
// 保存数据源(剖析设置):照提交改假状态,回整条数据源
|
||||
[/^PATCH \/datasources\/check-cat-[a-z]+$/, reply('patch', (route, { key, body }) => {
|
||||
const id = parts(key)[2]
|
||||
state.sources = state.sources.map((s) => (s.id === id ? { ...s, ...body, password: undefined } : s))
|
||||
return json(state.sources.find((s) => s.id === id))(route)
|
||||
})],
|
||||
[/^GET \/datasources\/check-cat-[a-z]+\/catalog\/[^/]+$/, reply('get', (route, { key }) => {
|
||||
const { src, t } = findTable(state, key)
|
||||
return t ? json(detailOf(src, t))(route) : json({ detail: `数据源中没有表 ${parts(key)[4]},可能已被删除或尚未探查结构` }, 404)(route)
|
||||
@@ -1103,6 +1113,384 @@ await section('数据目录 · 空状态与导入表格', async () => {
|
||||
await close()
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 数据剖析(阶段 4B):剖析会对业务库发查询,这里一条都不能真的发出去——剖析接口和保存数据源都在浏览器层答掉,
|
||||
// 照服务端(data/catalog_profile.py)的结论改假状态
|
||||
// ---------------------------------------------------------------------------
|
||||
const GATE_REL = relId(['gate_id'], 'gates', ['id'])
|
||||
const MEMBER_REL = relId(['member_id'], 'members', ['id'])
|
||||
const PROFILE_DAY = '2026-10-03'
|
||||
const PROFILE_ON = { enabled: true, max_queries: 60, query_timeout_s: 10 }
|
||||
const withProfile = (state, src, value) => {
|
||||
state.sources = state.sources.map((s) => (s.id === src ? { ...s, options: { ...s.options, catalog_profile: value } } : s))
|
||||
}
|
||||
const BUDGET_DETAIL = '本次剖析的查询次数已用完(上限 60 条),未检查'
|
||||
|
||||
/**
|
||||
* 剖析的默认答复:入园记录的闸机关系核实成已验证(覆盖率 98.7%),会员关系覆盖率不够保持推断,状态列的码值候选多出一个
|
||||
* 没有含义的取值 2;订单只剖析了一半就用完了查询次数,订单明细一条没查。ms 是服务端「执行」多久
|
||||
*/
|
||||
function profileReply(state, { ms = 0 } = {}) {
|
||||
return async (route) => {
|
||||
if (ms) await sleep(ms)
|
||||
const visits = state.tables[S1].find((t) => t.name === 'visits')
|
||||
const notes = clone(visits.notes)
|
||||
const at = `${PROFILE_DAY}T03:00:00+00:00`
|
||||
notes.relations = notes.relations.map((r) => (r.id === GATE_REL ? {
|
||||
...r, source: 'profile', status: 'verified', coverage: 0.987, cardinality: 'many_to_one', updated_at: at,
|
||||
note: `数据剖析(${PROFILE_DAY}):子表抽样 300 个不同键值,父表对上 296 个,覆盖率 98.7%;被指向列是主键;子表一侧有重复值,多对一。`,
|
||||
} : r))
|
||||
notes.relations.push(rel(['member_id'], 'members', ['id'], 'profile', { coverage: 0.62, updated_at: at,
|
||||
note: `数据剖析(${PROFILE_DAY}):抽样覆盖率 62%,可能不是这条关系;子表抽样 200 个不同键值,父表对上 124 个;被指向列是主键。` }))
|
||||
notes.columns.status.codes = item({ 1: '已入园', 9: '已作废', 2: '' }, 'profile', 'proposed')
|
||||
notes.columns.status.codes.note = `数据剖析(${PROFILE_DAY}):统计全表,182000 行非空值共 3 个取值:1(94%)、9(5%)、2(1%)。`
|
||||
writeEntry(visits, notes)
|
||||
const tables = [
|
||||
{
|
||||
table_name: 'visits', queries: 38, row_estimate: { rows: 182000, method: 'stats', at_least: null },
|
||||
findings: [
|
||||
{ kind: 'relation', path: `relations.${GATE_REL}`, target: 'gate_id → gates.id', columns: ['gate_id'], to_table: 'gates', to_columns: ['id'],
|
||||
status: 'verified', confirmed: false, coverage: 0.987, cardinality: 'many_to_one', sample: 300, matched: 296, summary: '覆盖率 98.7%,升为有确证' },
|
||||
{ kind: 'relation', path: `relations.${MEMBER_REL}`, target: 'member_id → members.id', columns: ['member_id'], to_table: 'members',
|
||||
to_columns: ['id'], status: 'proposed', confirmed: false, coverage: 0.62, cardinality: 'many_to_one', sample: 200, matched: 124,
|
||||
summary: '抽样覆盖率 62%,保持推断' },
|
||||
{ kind: 'codes', path: 'columns.status.codes', column: 'status', values: [{ value: '1', rows: 171080 }, { value: '9', rows: 9100 },
|
||||
{ value: '2', rows: 1820 }], rows: 182000, status: 'proposed', summary: '3 个取值,1 个含义待填写' },
|
||||
],
|
||||
skipped: [
|
||||
{ kind: 'codes', target: 'member_level', path: 'columns.member_level.codes', reason: 'masked', detail: 'visits.member_level 在数据源设置中被遮罩,不取值' },
|
||||
{ kind: 'date', target: 'visit_time', path: 'business_date', reason: 'too_large',
|
||||
detail: 'visits 的行数超过整表统计的行数上限(100000 行),不取日期列的最小值和最大值' },
|
||||
],
|
||||
date_ranges: [], added: 1, updated: 2, removed: 0, version: visits.version, error: null,
|
||||
},
|
||||
{
|
||||
table_name: 'orders', queries: 22, row_estimate: { rows: null, method: 'count', at_least: 100001 }, findings: [],
|
||||
skipped: [{ kind: 'relation', target: 'member_id → members.id', path: 'relations.x', reason: 'budget', detail: BUDGET_DETAIL }],
|
||||
date_ranges: [], added: 0, updated: 0, removed: 0, version: 1, error: null,
|
||||
},
|
||||
{
|
||||
table_name: 'order_items', queries: 0, row_estimate: null, findings: [],
|
||||
skipped: [{ kind: 'table', target: 'order_items', path: null, reason: 'budget', detail: BUDGET_DETAIL }],
|
||||
date_ranges: [], added: 0, updated: 0, removed: 0, version: 1, error: null,
|
||||
},
|
||||
]
|
||||
return json({
|
||||
profiled_at: at, actor: ACTOR, settings: { ...PROFILE_ON, sample_size: 2000, max_scan_rows: 100000, max_total_s: 120 },
|
||||
queries_used: 60, stopped: 'budget', tables, total: { added: 1, updated: 2, removed: 0 }, note: null,
|
||||
})(route)
|
||||
}
|
||||
}
|
||||
const profileBox = (page) => page.locator('[data-profile-dialog]')
|
||||
const profilePhase = (page) => profileBox(page).getAttribute('data-profile-dialog').catch(() => null)
|
||||
/** 报告里不该露的机读码:停止原因、跳过原因、状态、基数的英文值 */
|
||||
const PROFILE_CODE_RE = /\b(?:budget|deadline|failed|masked|too_large|high_cardinality|no_data|unsupported|verified|proposed|many_to_one|row_estimate)\b/
|
||||
|
||||
await section('数据目录 · 数据剖析设置', async () => {
|
||||
const state = freshState()
|
||||
const replies = { patch: [] }
|
||||
const { page, sent, errors, natives, close } = await open('/data/databases', { handlers: catalogHandlers(state, replies) })
|
||||
// 手工登记的库:编辑框里有「数据剖析」一段,默认关闭,预算收着
|
||||
await page.locator('[data-source="zzcatscenic"]').getByRole('button', { name: '编辑', exact: true }).click()
|
||||
const box = page.locator('[data-profile-settings]')
|
||||
await box.waitFor()
|
||||
check('编辑框里有「数据剖析」,默认关闭、预算收着', (await box.getAttribute('data-profile-settings')) === 'off'
|
||||
&& await box.locator('[data-profile-field]').count() === 0 && (await box.innerText()).includes('默认关闭'))
|
||||
await box.locator('[data-profile-enabled]').check()
|
||||
const labels = await box.locator('label.label').allInnerTexts()
|
||||
check('开启后五项预算展开,中文标签', labels.join('、') === '查询次数上限、单条查询时限、抽样键值数、整表统计行数上限、总时长上限', labels.join('、'))
|
||||
const units = await box.locator('[data-profile-field] span').allInnerTexts()
|
||||
check('……每项带单位', units.join(',') === '条,秒,个,行,秒', units.join(','))
|
||||
const hints = await box.locator('[id$="-hint"]').allInnerTexts()
|
||||
check('……写明范围和默认值,整表统计写明 0 的含义', hints.some((h) => h.startsWith('范围 1–500 条,默认 60'))
|
||||
&& hints.some((h) => h.includes('范围 0–10,000,000 行') && h.includes('填 0 表示一律不做整表统计')), hints.join(' | '))
|
||||
const risk = () => box.locator('[data-profile-risk]').innerText().catch(() => '')
|
||||
check('……风险说明:会发出只读查询,每次最多 60 条、单条 10 秒(缺省)', (await risk()).includes('只读查询:最多 60 条,单条不超过 10 秒'), await risk())
|
||||
await page.locator('#ds-profile-max_queries').fill('80')
|
||||
check('……风险说明跟着填的数走', (await risk()).includes('最多 80 条'), await risk())
|
||||
await page.locator('#ds-profile-max_queries').fill('0')
|
||||
const err = await page.locator('#ds-profile-max_queries-error').innerText().catch(() => '')
|
||||
check('填错(0 条):就地写明范围,保存按钮禁用', err === '请填写 1 到 500 之间的整数' && await page.locator('[data-source-save]').isDisabled(), err)
|
||||
check('……输入框标出错(aria-invalid)', (await page.locator('#ds-profile-max_queries').getAttribute('aria-invalid')) === 'true')
|
||||
await page.locator('#ds-profile-max_queries').fill('80')
|
||||
await page.locator('#ds-profile-query_timeout_s').fill('20')
|
||||
// 服务端按字段拒收(422):报错落在那一项下面,光标放进去,不只弹提示
|
||||
const REJECT = '数据剖析的「单条查询时限(秒)」需要填写 1 到 60 之间的数;当前为「20」'
|
||||
replies.patch.push(json({ detail: REJECT }, 422))
|
||||
await page.locator('[data-source-save]').click()
|
||||
check('保存被拒(422):服务端原话写在「单条查询时限」下面', await until(async () =>
|
||||
(await page.locator('#ds-profile-query_timeout_s-error').innerText().catch(() => '')) === REJECT),
|
||||
await page.locator('#ds-profile-query_timeout_s-error').innerText().catch(() => ''))
|
||||
check('……光标落在那一项上,框标出错', await until(async () => page.evaluate(() => document.activeElement?.id === 'ds-profile-query_timeout_s'))
|
||||
&& (await page.locator('#ds-profile-query_timeout_s').getAttribute('aria-invalid')) === 'true')
|
||||
check('……没有落到「高级连接参数」的查询时限上', await page.locator('#ds-adv-query_timeout_s-error').count() === 0)
|
||||
if (SHOTS) await sleep(350)
|
||||
await shot(page, 'catalog-profile-settings-422')
|
||||
await page.locator('#ds-profile-query_timeout_s').fill('15')
|
||||
check('改了这一项,报错收起', await page.locator('#ds-profile-query_timeout_s-error').count() === 0)
|
||||
await page.locator('[data-source-save]').click()
|
||||
await until(async () => (await page.locator('[data-profile-settings]').count()) === 0)
|
||||
const saved = writes(sent, /^PATCH /).at(-1)?.body
|
||||
check('保存:options 里写剖析设置(开关和填了的项,数字)', JSON.stringify(saved?.options?.catalog_profile)
|
||||
=== JSON.stringify({ enabled: true, max_queries: 80, query_timeout_s: 15 }), JSON.stringify(saved?.options))
|
||||
check('……卡片上标「数据剖析已开启」,悬停写明预算', await until(async () => (await page.locator('[data-source="zzcatscenic"] [data-profile-on]').count()) === 1)
|
||||
&& (await page.locator('[data-source="zzcatscenic"] [data-profile-on]').getAttribute('title')) === '数据剖析已开启:每次最多 80 条只读查询,单条不超过 15 秒')
|
||||
// 再打开:剖析设置回显,没有摊进「高级连接参数」变成一串文字
|
||||
await page.locator('[data-source="zzcatscenic"]').getByRole('button', { name: '编辑', exact: true }).click()
|
||||
await page.locator('[data-profile-settings="on"]').waitFor()
|
||||
check('再打开编辑框:开关和填的数回显', await page.locator('#ds-profile-max_queries').inputValue() === '80'
|
||||
&& await page.locator('#ds-profile-query_timeout_s').inputValue() === '15')
|
||||
const advanced = await page.locator('[data-profile-settings] ~ details, details').allInnerTexts()
|
||||
check('……剖析设置没有摊进「高级连接参数」', !advanced.join(' ').includes('catalog_profile') && !advanced.join(' ').includes('[object Object]'))
|
||||
await page.getByRole('button', { name: '取消', exact: true }).click()
|
||||
|
||||
// 导入表格的源没有编辑框:卡片上单独设置,options 的其余键原样带上
|
||||
withProfile(state, S2, undefined)
|
||||
state.sources = state.sources.map((s) => (s.id === S2 ? { ...s, options: { mask_columns: ['联系电话'] } } : s))
|
||||
await goto(page, '/data/tables')
|
||||
await page.locator('[data-source="zzcatupload"] [data-edit-profile]').click()
|
||||
const dialog = page.locator('[data-profile-settings-dialog]')
|
||||
await dialog.waitFor()
|
||||
check('导入表格的源:卡片上「设置数据剖析」打开只改这一项的弹窗', (await page.locator('[role="dialog"]').last().innerText()).includes('「zzcatupload」的数据剖析设置'))
|
||||
check('……没改时保存禁用', await page.locator('[data-profile-save]').isDisabled())
|
||||
await dialog.locator('[data-profile-enabled]').check()
|
||||
await page.locator('[data-profile-save]').click()
|
||||
await until(async () => (await dialog.count()) === 0)
|
||||
const body = writes(sent, /^PATCH \/datasources\/check-cat-upload$/).at(-1)?.body
|
||||
check('……保存只换剖析设置,遮罩列原样带上', JSON.stringify(body) === JSON.stringify({ options: { mask_columns: ['联系电话'], catalog_profile: { enabled: true } } }),
|
||||
JSON.stringify(body))
|
||||
check('……卡片上标「数据剖析已开启」', await until(async () => (await page.locator('[data-source="zzcatupload"] [data-profile-on]').count()) === 1))
|
||||
check('写请求只有保存数据源', writes(sent, /./).every((s) => /^PATCH \/datasources\/check-cat-/.test(s.key)),
|
||||
JSON.stringify(writes(sent, /./).map((s) => s.key)))
|
||||
check('没有原生对话框', natives.length === 0, natives.join(' | '))
|
||||
check('没有运行时报错', errors.length === 0, errors[0] ?? '')
|
||||
await close()
|
||||
})
|
||||
|
||||
await section('数据目录 · 数据剖析', async () => {
|
||||
const state = freshState()
|
||||
withProfile(state, S1, PROFILE_ON)
|
||||
const replies = { profile: [] }
|
||||
const { page, sent, errors, natives, close } = await open(`/data/catalog/${S1}`, { handlers: catalogHandlers(state, replies) })
|
||||
await page.locator('[data-catalog-row]').first().waitFor()
|
||||
const head = page.locator('[data-catalog-profile]')
|
||||
check('页头有「数据剖析」', (await head.getAttribute('aria-label')) === '数据剖析')
|
||||
await head.click()
|
||||
await profileBox(page).waitFor()
|
||||
const setup = await profileBox(page).innerText()
|
||||
check('没选表、没筛选:范围只有「使用次数最多、有待核实关联关系的表」,写明至多 10 张',
|
||||
(await profileBox(page).locator('[data-profile-scope]').evaluateAll((els) => els.map((e) => e.getAttribute('data-profile-scope')))).join(',') === 'default'
|
||||
&& setup.includes('使用次数最多、有待核实关联关系的表') && setup.includes('至多 10 张'))
|
||||
const budget = await profileBox(page).locator('[data-profile-budget]').innerText()
|
||||
check('确认前写明预算:查询条数、单条和总时长、抽样规模、整表统计门槛,按哪个源的设置',
|
||||
['最多 60 条只读查询,单条不超过 10 秒', '总时长不超过 120 秒', '抽样 2,000 个键值', '行数超过 100,000 的表不做整表统计', '按「zzcatscenic」的数据剖析设置']
|
||||
.every((s) => budget.includes(s)), budget.replace(/\s+/g, ' '))
|
||||
check('……写明开始后无法中途停止', setup.includes('开始后无法中途停止'))
|
||||
check('……还没开始就没有写请求', writes(sent, /./).length === 0)
|
||||
if (SHOTS) await sleep(350)
|
||||
await shot(page, 'catalog-profile-setup')
|
||||
await page.keyboard.press('Escape')
|
||||
await until(async () => (await profileBox(page).count()) === 0)
|
||||
|
||||
// 打开入园记录的详情,选两张表剖析:完成后清单和正在看的详情都要重新载入
|
||||
await openDetail(page, S1, 'visits')
|
||||
const v0 = await version(page)
|
||||
check('剖析前:闸机关系是「推断 · 命名推断」,没有覆盖率', (await markStatus(page, `relations.${GATE_REL}`)) === 'proposed'
|
||||
&& (await page.locator(`[data-relation="${GATE_REL}"]`).innerText()).includes('—'))
|
||||
await page.locator('[data-catalog-select="visits"]').click()
|
||||
await page.locator('[data-catalog-select="orders"]').click()
|
||||
await page.locator('[data-catalog-profile-selected]').click()
|
||||
await profileBox(page).waitFor()
|
||||
check('剖析所选:范围默认是「已选的 2 张表」', await profileBox(page).locator('[data-profile-scope="selected"] input').isChecked()
|
||||
&& (await profileBox(page).locator('[data-profile-scope="selected"]').innerText()).includes('已选的 2 张表'))
|
||||
replies.profile.push(profileReply(state, { ms: 2600 }))
|
||||
const lists0 = count(sent, /^GET \/datasources\/check-cat-scenic\/catalog$/)
|
||||
await page.locator('[data-profile-start]').click()
|
||||
check('进行中:写「正在剖析…」和已用时长、总时长上限', await until(async () => (await profilePhase(page)) === 'running')
|
||||
&& (await profileBox(page).locator('[data-profile-title]').innerText()) === '正在剖析…'
|
||||
&& (await profileBox(page).locator('[data-profile-elapsed]').innerText()).includes('(总时长上限 2 分 00 秒)'))
|
||||
check('……时长在走(不是一直「不到 1 秒」)', await until(async () => Number(await profileBox(page).locator('[data-profile-elapsed]').getAttribute('data-profile-elapsed')) >= 1, 2500))
|
||||
check('……不给「停止」,明说无法中途停止', await profileBox(page).getByRole('button', { name: /停止|取消/ }).count() === 0
|
||||
&& (await profileBox(page).locator('[data-profile-no-cancel]').innerText()).includes('无法中途停止'))
|
||||
const req = writes(sent, /catalog\/profile$/).at(-1)
|
||||
check('……请求带所选的两张表和本机署名', JSON.stringify([...(req?.body?.tables ?? [])].sort()) === JSON.stringify(['orders', 'visits'])
|
||||
&& decodeURIComponent(req?.actor ?? '') === ACTOR, JSON.stringify(req?.body))
|
||||
if (SHOTS) await sleep(350)
|
||||
await shot(page, 'catalog-profile-running')
|
||||
// 关掉窗口剖析照常进行:页头换成计时,完成后提示并可以打开报告
|
||||
await page.locator('[data-profile-background]').click()
|
||||
check('「在后台继续」:窗口关掉,页头换成「正在剖析」', await until(async () => (await profileBox(page).count()) === 0)
|
||||
&& (await page.locator('[data-catalog-profile="running"]').innerText()).includes('正在剖析'))
|
||||
const toastBox = page.locator('[data-toast]').filter({ hasText: '数据剖析已停止' })
|
||||
check('完成:提示「数据剖析已停止:查询次数已用完」,带「查看报告」', await until(async () => (await toastBox.count()) > 0, 6000),
|
||||
await page.locator('[data-toast]').allInnerTexts().then((t) => t.join(' | ')).catch(() => ''))
|
||||
check('……页头回到「数据剖析」', await page.locator('[data-catalog-profile="running"]').count() === 0)
|
||||
check('……写进去的表重新载入:清单重取、正在看的详情版本更新', await until(async () => count(sent, /^GET \/datasources\/check-cat-scenic\/catalog$/) > lists0)
|
||||
&& await until(async () => (await version(page)) === v0 + 1))
|
||||
await toastBox.getByRole('button', { name: '查看报告' }).click()
|
||||
await profileBox(page).waitFor()
|
||||
const report = profileBox(page).locator('[data-profile-report]')
|
||||
check('报告:停止原因说中文', (await report.locator('[data-profile-title]').innerText()) === '剖析已停止:查询次数已用完')
|
||||
check('……用了多少条查询、剖析几张表', (await report.locator('[data-profile-summary]').innerText()).startsWith('用了 60 / 60 条查询,剖析 3 张表'))
|
||||
check('……目录的变化', (await report.locator('[data-profile-changes]').innerText()) === '新增 1 项、更新 2 项')
|
||||
check('……停下之后怎么办:调高查询次数上限或缩小范围,给「修改设置」', (await report.locator('[data-profile-stopped="budget"]').innerText()).includes('调高查询次数上限')
|
||||
&& await report.locator('[data-profile-stopped] [data-profile-settings-open]').count() === 1)
|
||||
const visitsCard = report.locator('[data-profile-table="visits"]')
|
||||
const gate = visitsCard.locator(`[data-profile-finding="relation"][data-path="relations.${GATE_REL}"]`)
|
||||
check('关系核实:覆盖率、基数、抽样规模,升为已验证', (await gate.locator('[data-profile-coverage]').innerText()) === '覆盖率 98.7%'
|
||||
&& (await gate.locator('[data-profile-cardinality]').innerText()) === '多对一' && (await gate.innerText()).includes('抽样 300 个键值,对上 296 个')
|
||||
&& (await gate.locator('[data-profile-outcome]').innerText()) === '升为已验证' && (await gate.getAttribute('data-status')) === 'verified',
|
||||
(await gate.innerText()).replace(/\s+/g, ' '))
|
||||
const member = visitsCard.locator(`[data-profile-finding="relation"][data-path="relations.${MEMBER_REL}"]`)
|
||||
check('……覆盖率不够的保持推断,写明原因', (await member.locator('[data-profile-outcome]').innerText()) === '保持推断:覆盖率不足,可能不是这条关系'
|
||||
&& (await member.locator('[data-profile-coverage]').innerText()) === '覆盖率 62%')
|
||||
const codes = visitsCard.locator('[data-profile-finding="codes"]')
|
||||
check('码值候选:列、各取值的行数、几个含义待填写,给「填写含义」', (await codes.getAttribute('data-column')) === 'status'
|
||||
&& (await codes.innerText()).includes('3 个取值,共 182,000 行') && (await codes.innerText()).includes('171,080 行')
|
||||
&& (await codes.locator('[data-codes-pending]').innerText()) === '1 个含义待填写' && await codes.locator('[data-profile-fill="status"]').count() === 1,
|
||||
(await codes.innerText()).replace(/\s+/g, ' '))
|
||||
const skipped = visitsCard.locator('[data-profile-skipped]')
|
||||
check('跳过的项:原因说中文,附服务端的整句', (await skipped.innerText()).includes('列已遮罩') && (await skipped.innerText()).includes('visits.member_level 在数据源设置中被遮罩,不取值')
|
||||
&& (await skipped.innerText()).includes('表太大'), (await skipped.innerText()).replace(/\s+/g, ' '))
|
||||
check('……行数估算写成人话:统计信息约 182,000 行、数到上限的超过 100,000 行',
|
||||
(await visitsCard.locator('header').innerText()).includes('约 182,000 行') && (await report.locator('[data-profile-table="orders"] header').innerText()).includes('超过 100,000 行'))
|
||||
check('……一条没查的表写明「查询次数用完」', (await report.locator('[data-profile-table="order_items"] [data-profile-skip="budget"]').innerText()).includes('查询次数用完'))
|
||||
const reportText = await report.innerText()
|
||||
check('报告不露机读码', !PROFILE_CODE_RE.test(reportText), reportText.match(PROFILE_CODE_RE)?.[0] ?? '')
|
||||
if (SHOTS) await sleep(350)
|
||||
await shot(page, 'catalog-profile-report')
|
||||
await page.locator('[data-profile-close]').click()
|
||||
await until(async () => (await profileBox(page).count()) === 0)
|
||||
|
||||
// 表详情:关系的来源写「数据剖析」,带覆盖率和基数;码值的含义空着写「含义待填写」
|
||||
const gateRow = page.locator(`[data-relation="${GATE_REL}"]`)
|
||||
check('表详情:闸机关系「已验证 · 数据剖析」,覆盖率 98.7%、多对一', (await markStatus(page, `relations.${GATE_REL}`)) === 'verified'
|
||||
&& (await gateRow.innerText()).includes('数据剖析') && (await gateRow.innerText()).includes('98.7%') && (await gateRow.innerText()).includes('多对一'),
|
||||
(await gateRow.innerText().catch(() => '')).replace(/\s+/g, ' '))
|
||||
await openMark(page, `relations.${GATE_REL}`)
|
||||
check('……点开来源:写明抽样规模和覆盖率的剖析说明', (await panel(page).innerText()).includes('子表抽样 300 个不同键值,父表对上 296 个'))
|
||||
await page.keyboard.press('Escape')
|
||||
const statusCodes = page.locator('[data-column="status"] [data-cell="codes"]')
|
||||
check('……码值的含义空着写「含义待填写」,给「填写含义」', (await statusCodes.locator('[data-code="2"] [data-code-pending]').innerText()) === '含义待填写'
|
||||
&& (await statusCodes.locator('[data-codes-fill]').innerText()).includes('填写含义'))
|
||||
// 从报告点「填写含义」:窗口关掉,打开这张表那一列的码值弹窗,光标在空着的那一个上
|
||||
await page.locator('[data-catalog-profile]').click()
|
||||
await profileBox(page).waitFor()
|
||||
check('页头再点:上一次的报告看过了就回到选范围', (await profilePhase(page)) === 'setup')
|
||||
await page.keyboard.press('Escape')
|
||||
await statusCodes.locator('[data-codes-fill]').click()
|
||||
const codesBox = page.locator('[data-codes-dialog="status"]')
|
||||
await codesBox.waitFor()
|
||||
check('码值弹窗:每个码值一个输入框,空着的写「含义待填写」,光标在它上面', await codesBox.locator('input[data-code]').count() === 3
|
||||
&& (await codesBox.locator('input[data-code="2"]').getAttribute('placeholder')) === '含义待填写'
|
||||
&& await until(async () => page.evaluate(() => document.activeElement?.getAttribute('data-code') === '2')))
|
||||
check('……写明剖析说明(各取值的占比)', (await codesBox.innerText()).includes('1(94%)、9(5%)、2(1%)'))
|
||||
check('……没填时保存禁用', await page.locator('[data-codes-save]').isDisabled())
|
||||
await codesBox.locator('input[data-code="2"]').fill('已退票')
|
||||
if (SHOTS) await sleep(350)
|
||||
await shot(page, 'catalog-codes-fill')
|
||||
const v1 = await version(page)
|
||||
await page.keyboard.press('Enter')
|
||||
check('回车保存:整份提交,只换这一列码值的含义,带读到的版本', await until(async () => (await codesBox.count()) === 0)
|
||||
&& JSON.stringify(lastBody(sent, /^PUT .*catalog\/visits$/)?.notes?.columns?.status?.codes?.value) === JSON.stringify({ 1: '已入园', 9: '已作废', 2: '已退票' })
|
||||
&& lastBody(sent, /^PUT .*catalog\/visits$/)?.if_version === v1, JSON.stringify(lastBody(sent, /^PUT .*catalog\/visits$/)?.notes?.columns?.status))
|
||||
check('……保存后不再有「含义待填写」,这一项记为人工确认', await until(async () => (await statusCodes.locator('[data-code-pending]').count()) === 0)
|
||||
&& (await statusCodes.innerText()).includes('已退票') && (await markStatus(page, 'columns.status.codes')) === 'confirmed')
|
||||
// 报告里的「填写含义」直达
|
||||
await page.locator('[data-catalog-profile]').click()
|
||||
replies.profile.push(profileReply(state))
|
||||
await page.locator('[data-profile-start]').click()
|
||||
await until(async () => (await profilePhase(page)) === 'done')
|
||||
check('窗口开着时完成:直接看报告,不另弹提示', (await toastBox.count()) <= 1)
|
||||
await openDetail(page, S1, 'orders')
|
||||
await page.locator('[data-catalog-profile]').click()
|
||||
await profileBox(page).waitFor()
|
||||
check('换了一张表、再点页头:回到选范围(报告已看过)', (await profilePhase(page)) === 'setup')
|
||||
await page.keyboard.press('Escape')
|
||||
|
||||
check('写请求只有剖析和整份提交', writes(sent, /./).every((s) => /catalog\/profile$|^PUT .*catalog\/visits$/.test(s.key)),
|
||||
JSON.stringify(writes(sent, /./).map((s) => s.key)))
|
||||
check('没有原生对话框', natives.length === 0, natives.join(' | '))
|
||||
check('没有运行时报错', errors.length === 0, errors[0] ?? '')
|
||||
await close()
|
||||
})
|
||||
|
||||
await section('数据目录 · 数据剖析开始不了(409)', async () => {
|
||||
const state = freshState()
|
||||
const replies = { profile: [], patch: [] }
|
||||
const { page, sent, errors, natives, close } = await open(`/data/catalog/${S1}`, { handlers: catalogHandlers(state, replies) })
|
||||
await page.locator('[data-catalog-row]').first().waitFor()
|
||||
// 数据源列表里就没开:不让开始,就地开启
|
||||
await page.locator('[data-catalog-profile]').click()
|
||||
await profileBox(page).waitFor()
|
||||
check('没开启:写明需要先开启,「开始剖析」禁用,没有预算', (await profileBox(page).locator('[data-profile-disabled]').innerText()).includes('未开启数据剖析')
|
||||
&& await page.locator('[data-profile-start]').isDisabled() && await profileBox(page).locator('[data-profile-budget]').count() === 0)
|
||||
await profileBox(page).locator('[data-profile-disabled] [data-profile-enable]').click()
|
||||
const settings = page.locator('[data-profile-settings-dialog]')
|
||||
await settings.waitFor()
|
||||
await settings.locator('[data-profile-enabled]').check()
|
||||
await page.locator(`#profile-${S1}-max_queries`).fill('40')
|
||||
await page.locator('[data-profile-save]').click()
|
||||
check('就地开启:保存后回到剖析,预算按刚存的设置', await until(async () => (await settings.count()) === 0)
|
||||
&& await until(async () => (await profileBox(page).locator('[data-profile-budget]').innerText().catch(() => '')).includes('最多 40 条只读查询'))
|
||||
&& !(await page.locator('[data-profile-start]').isDisabled()))
|
||||
check('……保存的是这个源的剖析设置', JSON.stringify(writes(sent, /^PATCH /).at(-1)?.body) === JSON.stringify({ options: { catalog_profile: { enabled: true, max_queries: 40 } } }),
|
||||
JSON.stringify(writes(sent, /^PATCH /).at(-1)?.body))
|
||||
|
||||
const cases = [
|
||||
['disabled', '数据源「zzcatscenic」未开启数据剖析。剖析会对业务库发查询,请先在数据源设置中开启数据剖析', '未开启数据剖析', 'data-profile-enable'],
|
||||
['inactive', '数据源「zzcatscenic」已停用,无法进行数据剖析。请先在数据源设置中启用', '数据源已停用', 'data-profile-go-source'],
|
||||
['noSchema', '尚未探查结构,无法进行数据剖析。请先在数据源卡片上点「探查结构」', '还没有表结构', 'data-profile-go-source'],
|
||||
['busy', '数据源「zzcatscenic」正在进行数据剖析,请等待完成后再试', '这个数据源正在剖析', 'data-profile-retry'],
|
||||
['tampered', '数据文件与登记的版本不一致,可能被修改过,已拒绝查询', '数据文件核对未通过', 'data-profile-go-source'],
|
||||
]
|
||||
for (const [kind, detail, title, action] of cases) {
|
||||
replies.profile.push(json({ detail }, 409))
|
||||
await page.locator('[data-profile-start]').click()
|
||||
const blocked = profileBox(page).locator(`[data-profile-blocked="${kind}"]`)
|
||||
const ok = await until(async () => (await blocked.count()) === 1)
|
||||
const text = ok ? await blocked.innerText() : ''
|
||||
check(`409(${title}):写明是哪种情况、服务端原话和下一步,给对应的按钮`, ok && text.includes(title) && text.includes(detail)
|
||||
&& await blocked.locator(`[${action}]`).count() === 1, text.replace(/\s+/g, ' ').slice(0, 140))
|
||||
if (kind === 'busy') {
|
||||
// 稍后重试:同一个范围再发一次,这次成功
|
||||
replies.profile.push(profileReply(state))
|
||||
await blocked.locator('[data-profile-retry]').click()
|
||||
check('……「重试」再发一次,成功后看到报告', await until(async () => (await profilePhase(page)) === 'done'))
|
||||
}
|
||||
if (kind === 'disabled') {
|
||||
// 服务端说没开(列表里的设置过时了):同样就地开启,开了回到选范围
|
||||
await blocked.locator('[data-profile-enable]').click()
|
||||
await settings.waitFor()
|
||||
await page.locator(`#profile-${S1}-max_queries`).fill('45')
|
||||
await page.locator('[data-profile-save]').click()
|
||||
check('……就地开启后回到选范围', await until(async () => (await profilePhase(page)) === 'setup'))
|
||||
continue
|
||||
}
|
||||
// 底栏的按钮不在弹窗正文里
|
||||
await page.locator(kind === 'busy' ? '[data-profile-again]' : '[data-profile-back]').click()
|
||||
await until(async () => (await profilePhase(page)) === 'setup')
|
||||
}
|
||||
if (SHOTS) await sleep(350)
|
||||
await shot(page, 'catalog-profile-blocked')
|
||||
// 不是 409 的失败:照常的报错,能重试
|
||||
replies.profile.push(json({ detail: '服务端处理剖析时出错,请稍后重试' }, 500))
|
||||
await page.locator('[data-profile-start]').click()
|
||||
check('其余失败:写「剖析未能完成」和原因,能重试', await until(async () => (await profileBox(page).locator('[data-profile-failed]').count()) === 1)
|
||||
&& (await profileBox(page).locator('[data-profile-failed]').innerText()).includes('剖析未能完成'))
|
||||
// 去数据源:停用、没有表结构、快照被改动时
|
||||
replies.profile.push(json({ detail: cases[1][1] }, 409))
|
||||
await profileBox(page).locator('[data-profile-failed]').getByRole('button', { name: /重试/ }).click()
|
||||
await profileBox(page).locator('[data-profile-go-source]').click()
|
||||
await page.waitForURL('**/data/databases', { timeout: 5000 }).catch(() => {})
|
||||
check('「前往数据源」回到数据源卡片', new URL(page.url()).pathname === '/data/databases', page.url())
|
||||
check('写请求只有剖析和保存剖析设置', writes(sent, /./).every((s) => /catalog\/profile$|^PATCH \/datasources\/check-cat-scenic$/.test(s.key)),
|
||||
JSON.stringify(writes(sent, /./).map((s) => s.key)))
|
||||
check('没有原生对话框', natives.length === 0, natives.join(' | '))
|
||||
check('没有运行时报错', errors.length === 0, errors[0] ?? '')
|
||||
await close()
|
||||
})
|
||||
|
||||
await section('数据目录 · 窄屏', async () => {
|
||||
const state = freshState()
|
||||
const { page, errors, close } = await open(`/data/catalog/${S1}`, { handlers: catalogHandlers(state), viewport: { width: 390, height: 844 } })
|
||||
@@ -1130,6 +1518,13 @@ await section('数据目录 · 窄屏', async () => {
|
||||
await page.locator('[data-catalog-detail-back]').click()
|
||||
check('……「返回表清单」回到清单', await until(async () => page.locator('[data-catalog-list]').isVisible()))
|
||||
await shot(page, 'catalog-narrow')
|
||||
// 数据剖析:页头只留图标,弹窗在视口里、不横向滚动
|
||||
await page.locator('[data-catalog-profile]').click()
|
||||
await profileBox(page).waitFor()
|
||||
const dlg = await page.locator('[role="dialog"]').last().boundingBox()
|
||||
const inner = await profileBox(page).evaluate((el) => el.scrollWidth - el.clientWidth)
|
||||
check('……数据剖析弹窗在视口里,正文不横向滚动', !!dlg && dlg.x >= 0 && dlg.x + dlg.width <= 390 && inner <= 1, `${JSON.stringify(dlg)} ${inner}`)
|
||||
await page.keyboard.press('Escape')
|
||||
check('没有运行时报错', errors.length === 0, errors[0] ?? '')
|
||||
await close()
|
||||
})
|
||||
|
||||
@@ -714,6 +714,7 @@ await section('Copilot:建议更新数据目录(catalog_patch 操作)', as
|
||||
cardinality: 'many_to_one' }) === 'visit_id → visits.id(多对一)')
|
||||
check('表类型、度量类型写界面上的叫法', cp.patchValueText('kind', 'fact') === '明细表' && cp.patchValueText('columns.a.measure', 'stock') === '存量')
|
||||
check('没有值写「未填写」', cp.patchValueText('grain', null) === '未填写')
|
||||
check('码值含义为空(剖析的候选)写「含义待填写」', cp.patchValueText('columns.status.codes', { 1: '有效', 9: '' }) === '1=有效、9=含义待填写')
|
||||
check('列名里有句点照样认', JSON.stringify(cp.patchTarget('columns.a.b.unit')) === '{"kind":"column","column":"a.b","field":"unit"}')
|
||||
check('形状不对不出卡片、不出行', cp.catalogPatchesOf([
|
||||
{ op: 'catalog_patch' }, { op: 'catalog_patch', source: 's', source_id: 'i', table: 't', changes: [{ path: 'nope', after: 1 }] },
|
||||
@@ -1732,5 +1733,89 @@ await section('按后端原文匹配的地方:后端现在的原文、整改
|
||||
sub.join(' | '))
|
||||
})
|
||||
|
||||
await section('SQL 检查(数据目录阶段 4B):指标所依据的查询未通过检查说成一行;助手自查和交付的问题写中文规则名', async () => {
|
||||
const E = (seq, type, node_id, data = {}) => ({ seq, type, node_id, ts: 1790000000 + seq, data })
|
||||
// 原话和 backend/app/engine/nodes/metrics.py 的 _sql_check_reason 同一个句式。先核对后端源码还是这么说的
|
||||
const metricsPy = readFileSync(`${root}backend/app/engine/nodes/metrics.py`, 'utf8')
|
||||
check('后端发指标未通过 SQL 检查警告的格式还是夹具里这几种', metricsPy.includes('code="metric_sql_check"')
|
||||
&& metricsPy.includes('message=f"指标「{m[\'name\']}」{m[\'sql_check_reason\']}"')
|
||||
&& metricsPy.includes('"所依据的查询未通过 SQL 检查,结果不可靠"')
|
||||
&& metricsPy.includes('f"所依据的查询未通过 SQL 检查({problems[0]}),结果不可靠"')
|
||||
&& metricsPy.includes('f"所依据的查询有 {len(problems)} 处未通过 SQL 检查({problems[0]}等),结果不可靠"'),
|
||||
'改了 metrics.py 的说法就同步改 decode.ts 的 explainLog 和这里的夹具')
|
||||
const CODES = /fanout_sum|stock_summed|join_unconfirmed|ratio_aggregated|missing_valid_filter|unknown_code|wrong_date_column/
|
||||
const decodeOne = (message) => {
|
||||
const all = flatten(mod.decodeRun([
|
||||
E(1, 'node.started', 'card', { node_type: 'metrics' }),
|
||||
E(2, 'log', 'card', { level: 'warn', code: 'metric_sql_check', metric: 'aov', message }),
|
||||
E(3, 'node.finished', 'card', { duration_ms: 3 }),
|
||||
]))
|
||||
return { step: all.find((s) => s.code === 'metric_sql_check'), node: all.find((s) => s.kind === 'node') }
|
||||
}
|
||||
// 问题本身带括号(列的中文名里写了单位):副标题要取到「,结果不可靠」前面最后一个右括号
|
||||
const ONE = '指标「客单价」所依据的查询未通过 SQL 检查(「订单」关联「订单明细」是一对多,对「订单」的「订单金额(元)」求和会重复计算),结果不可靠'
|
||||
const one = decodeOne(ONE)
|
||||
check('metric_sql_check 一行:标题点名哪个指标', one.step?.title === '指标「客单价」所依据的查询未通过 SQL 检查', one.step?.title)
|
||||
check('……原因放副标题(问题里的括号不截断),原话留在展开区',
|
||||
one.step?.sub === '「订单」关联「订单明细」是一对多,对「订单」的「订单金额(元)」求和会重复计算' && one.step?.detail === ONE,
|
||||
`${one.step?.sub} / ${one.step?.detail}`)
|
||||
check('……是提醒(warn),下一步指到 SQL 和数据目录,节点本身照样成功', one.step?.level === 'warn'
|
||||
&& !!one.step?.next?.includes('SQL') && !!one.step?.next?.includes('数据目录') && one.step?.fix === 'canvas'
|
||||
&& one.node?.status === 'done', `${one.step?.level} ${one.step?.next}`)
|
||||
const two = decodeOne('指标「客单价」所依据的查询有 2 处未通过 SQL 检查(「订单」关联「订单明细」是一对多,对「订单」的「订单金额」求和会重复计算等),结果不可靠')
|
||||
check('几处问题:标题写处数,副标题是第一处', two.step?.title === '指标「客单价」所依据的查询有 2 处未通过 SQL 检查'
|
||||
&& two.step?.sub === '「订单」关联「订单明细」是一对多,对「订单」的「订单金额」求和会重复计算等', `${two.step?.title} / ${two.step?.sub}`)
|
||||
const bare = decodeOne('指标「客单价」所依据的查询未通过 SQL 检查,结果不可靠')
|
||||
check('没有括号里的问题:只有标题,不编副标题', bare.step?.title === '指标「客单价」所依据的查询未通过 SQL 检查' && !bare.step?.sub,
|
||||
`${bare.step?.title} / ${bare.step?.sub}`)
|
||||
|
||||
// 助手自查(check)和交付(final.issues)里的 SQL 检查问题:code 是规则编号,field 是 args.sql,另带表、列和 SQL 片段
|
||||
const sqlIssue = (level, code, message, extra = {}) => ({ level, node_id: 'q', edge_id: null, field: 'args.sql', code, message,
|
||||
table: 'orders', ...extra })
|
||||
const ERR = sqlIssue('error', 'fanout_sum', '「订单」关联「订单明细」是一对多,对「订单」的「订单金额」求和会重复计算。请先按订单汇总明细再关联',
|
||||
{ column: 'total_amount', sql_excerpt: 'SUM(o.total_amount)' })
|
||||
const WARN = sqlIssue('warning', 'missing_valid_filter', '「订单」定义了有效记录条件「status <> 9」,查询中没有按它筛选')
|
||||
const INFO = sqlIssue('info', 'join_unconfirmed', '「订单」与「门店」按推断的关系关联(o.store_id = s.id),这条关系尚未确认', { table: 'stores' })
|
||||
const ops = [
|
||||
{ op: 'add_node', node: { id: 'q', type: 'tool', data: { label: '订单汇总' } } },
|
||||
{ op: 'done', explanation: '' },
|
||||
{ op: 'check', status: 'repairing', round: 1, issues: [ERR] },
|
||||
{ op: 'check', status: 'failed', issues: [ERR] },
|
||||
{ op: 'final', graph: { nodes: [{ id: 'q' }] }, issues: [ERR, INFO, WARN,
|
||||
{ level: 'warning', node_id: 'q', edge_id: null, field: null, code: 'tools_dropped', message: '工具被移除' }] },
|
||||
]
|
||||
const steps = mod.decodeCopilot(ops, { context: 'canvas' })
|
||||
const repairing = steps.find((s) => s.title.startsWith('自查发现'))
|
||||
check('自查交回去改的那一行:节点名 + 中文规则名 + 涉及的表和列,不露规则编号',
|
||||
repairing?.detail === '「订单汇总」一对多关联后重复计算:「订单」关联「订单明细」是一对多,对「订单」的「订单金额」求和会重复计算。'
|
||||
+ '请先按订单汇总明细再关联(涉及 orders.total_amount)', repairing?.detail)
|
||||
const failedRow = steps.find((s) => s.kind === 'error')
|
||||
check('没修好的那一行同样写规则名', !!failedRow?.detail?.includes('一对多关联后重复计算') && !CODES.test(failedRow.detail), failedRow?.detail)
|
||||
const sqlRow = steps.find((s) => s.code === 'sql_check')
|
||||
check('交付时的 SQL 检查单独一行:数提醒、提示,没修好的错误已经列过不重复', sqlRow?.title === 'SQL 检查:1 处提醒、1 处提示', sqlRow?.title)
|
||||
check('……提醒在前、提示在后,每条写规则名和表', sqlRow?.detail === [
|
||||
'「订单汇总」未筛选有效记录:「订单」定义了有效记录条件「status <> 9」,查询中没有按它筛选(涉及 orders)',
|
||||
'「订单汇总」关联条件未经确认:「订单」与「门店」按推断的关系关联(o.store_id = s.id),这条关系尚未确认(涉及 stores)',
|
||||
].join('\n'), sqlRow?.detail)
|
||||
check('……有提醒时是 warn,下一步指到「参数」里的 SQL,单个节点时给打开设置', sqlRow?.level === 'warn' && !!sqlRow.next?.includes('参数')
|
||||
&& sqlRow.nodeId === 'q' && sqlRow.fix === 'canvas', `${sqlRow?.level} ${sqlRow?.next} ${sqlRow?.nodeId}`)
|
||||
check('整个助手流里不露规则编号', !steps.some((s) => CODES.test(`${s.title}\n${s.detail ?? ''}\n${s.sub ?? ''}`)))
|
||||
const chatRow = mod.decodeCopilot(ops, { context: 'chat' }).find((s) => s.code === 'sql_check')
|
||||
check('问数据页没有画布:下一步说去画布里打开这个工作流,不给「打开设置」', !!chatRow?.next?.includes('在画布中打开') && !chatRow.fix,
|
||||
`${chatRow?.next} ${chatRow?.fix}`)
|
||||
const out = mod.copilotOutcome(ops)
|
||||
check('结局里带着不挡运行的 SQL 检查(没修好的错误在 check 里,不重复)', out.sqlChecks.length === 2
|
||||
&& out.sqlChecks.map((x) => x.code).join(',') === 'join_unconfirmed,missing_valid_filter'
|
||||
&& out.check?.issues.length === 1 && out.check.issues[0].code === 'fanout_sum', JSON.stringify(out.sqlChecks.map((x) => x.code)))
|
||||
const asCheck = mod.issueSqlCheck(out.check.issues[0])
|
||||
check('问题按 SQL 检查读得出级别、表、列、SQL 片段', asCheck?.level === 'error' && asCheck.table === 'orders' && asCheck.column === 'total_amount'
|
||||
&& asCheck.sql_excerpt === 'SUM(o.total_amount)' && out.check.issues[0].field === 'args.sql', JSON.stringify(asCheck))
|
||||
check('不是 SQL 检查的问题不按它读', mod.issueSqlCheck({ message: 'x', code: 'tools_dropped' }) === null)
|
||||
const infoOnly = mod.decodeCopilot([{ op: 'final', graph: { nodes: [{ id: 'q' }] }, issues: [INFO] }], { context: 'canvas' })
|
||||
.find((s) => s.code === 'sql_check')
|
||||
check('只有提示:不用提醒色,下一步说依据尚未确认', !!infoOnly && !infoOnly.level && !!infoOnly.next?.includes('尚未确认'),
|
||||
`${infoOnly?.level} ${infoOnly?.next}`)
|
||||
})
|
||||
|
||||
console.log(failed ? `\n✗ ${failed} 项未通过` : '\n✓ 解码器全部通过')
|
||||
process.exit(failed ? 1 : 0)
|
||||
|
||||
@@ -1036,6 +1036,58 @@ await section('incomplete', '截断的查询结果整组算出来的指标:标
|
||||
await cut.ctx.close()
|
||||
})
|
||||
|
||||
await section('sqlcheck', 'SQL 检查(数据目录阶段 4B):查询步骤列出级别、规则名、说明、表和列、SQL 片段;指标未通过时标出来并写明原因', async () => {
|
||||
await openQ(page, '1,288')
|
||||
check('没查出问题的查询步骤不出 SQL 检查', await panel(page).locator('[data-ev-sql-checks]').count() === 0)
|
||||
await page.keyboard.press('Escape')
|
||||
await openQ(page, '45,678.5元', { wait: '[data-ev-expression]' })
|
||||
check('通过的指标不出「SQL 检查未通过」', await panel(page).locator('[data-ev-sql-check-failed], [data-ev-sql-check-reason]').count() === 0)
|
||||
await page.keyboard.press('Escape')
|
||||
// 接口的形状:backend/app/api/evidence.py 的查询步骤带 checks(去掉了给模型的那句),指标步骤带 sql_check_failed / sql_check_reason
|
||||
const CHECKS = [
|
||||
{ code: 'join_unconfirmed', level: 'info', message: '「订单」与「地区」按推断的关系关联(o.region = r.code),这条关系尚未确认。', table: 'orders' },
|
||||
{ code: 'fanout_sum', level: 'error', table: 'orders', column: 'amount', sql_excerpt: 'SUM(o.amount)',
|
||||
message: '「订单」关联「订单明细」是一对多,对「订单」的「订单金额」求和会重复计算。请先按订单汇总明细再关联。' },
|
||||
{ code: 'missing_valid_filter', level: 'warning', message: '「订单」定义了有效记录条件「status <> 9」,查询中没有按它筛选。', table: 'orders' },
|
||||
]
|
||||
const REASON = '所依据的查询未通过 SQL 检查(「订单」关联「订单明细」是一对多,对「订单」的「订单金额」求和会重复计算),结果不可靠'
|
||||
const bad = await probeQ((id, body) => ({ ...body, chain: (body.chain ?? []).map((st) => (st.step === 'query'
|
||||
? { ...st, checks: CHECKS } : st.step === 'metric' ? { ...st, sql_check_failed: true, sql_check_reason: REASON } : st)) }))
|
||||
await openQ(bad.page, '1,288', { wait: '[data-ev-sql-checks]' })
|
||||
const p = bad.page.locator('[data-evidence-panel]')
|
||||
const items = p.locator('[data-ev-sql-checks] [data-sql-check]')
|
||||
const levels = await items.evaluateAll((els) => els.map((e) => e.getAttribute('data-level')))
|
||||
check('三条都列在 SQL 下面,错误在前、提示在后', levels.join(',') === 'error,warning,info', levels.join(','))
|
||||
const first = await items.first().innerText().catch(() => '')
|
||||
check('一条里有级别、中文规则名、说明、涉及的表和列、SQL 片段', first.includes('错误') && first.includes('一对多关联后重复计算')
|
||||
&& first.includes('会重复计算') && first.includes('orders.amount') && first.includes('SUM(o.amount)'), first.replace(/\s+/g, ' '))
|
||||
check('级别四通道:图标 + 文字,三档文字各不相同', (await items.locator('[data-sql-level]').allInnerTexts()).map((t) => t.trim()).join(',')
|
||||
=== '错误,提醒,提示' && await items.locator('[data-sql-level] svg').count() === 3)
|
||||
const tones = await items.locator('[data-sql-level]').evaluateAll((els) => els.map((e) => getComputedStyle(e).color))
|
||||
const want = [await tokenColor(bad.page, 'color', 'var(--st-failed)'), await tokenColor(bad.page, 'color', 'var(--st-waiting)'),
|
||||
await tokenColor(bad.page, 'color', 'var(--accent)')]
|
||||
check('级别的颜色走令牌:错误 --st-failed、提醒 --st-waiting、提示 --accent', tones.join('|') === want.join('|'), tones.join('|'))
|
||||
check('悬停写明依据的确证程度', (await items.first().locator('[data-sql-level]').getAttribute('title')) === '依据已确认,结果必然有误')
|
||||
check('不露规则编号', !/fanout_sum|missing_valid_filter|join_unconfirmed/.test(await p.innerText()))
|
||||
await bad.page.keyboard.press('Escape')
|
||||
await openQ(bad.page, '45,678.5元', { wait: '[data-ev-sql-check-failed]' })
|
||||
const chip = p.locator('[data-ev-sql-check-failed]')
|
||||
check('指标名旁有「SQL 检查未通过」标识,悬停看原因', (await chip.innerText().catch(() => '')) === 'SQL 检查未通过'
|
||||
&& await chip.getAttribute('title') === REASON, await chip.innerText().catch(() => ''))
|
||||
check('原因写在指标下面', (await p.locator('[data-ev-sql-check-reason]').innerText().catch(() => '')) === REASON)
|
||||
check('和「结果不完整」同一种样式(提醒色的标识)', await chip.evaluate((el) => getComputedStyle(el).color)
|
||||
=== await tokenColor(bad.page, 'color', 'var(--st-waiting)') && (await chip.getAttribute('class'))?.includes('chip'))
|
||||
if (SHOTS) {
|
||||
for (const theme of ['dark', 'light']) {
|
||||
await bad.page.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme)
|
||||
await bad.page.waitForTimeout(200)
|
||||
await bad.page.screenshot({ path: `${SHOTS}/evidence-sqlcheck-metric-${theme}.png` })
|
||||
}
|
||||
}
|
||||
check('没有运行时报错', bad.errors.length === 0, bad.errors[0] ?? '')
|
||||
await bad.ctx.close()
|
||||
})
|
||||
|
||||
await section('q-narrow', '查询步骤在 360px:栏内展开、底部抽屉都不横向滚动;减少动效', async () => {
|
||||
const overflow = () => page.evaluate((sel) => {
|
||||
const el = document.querySelector(sel)
|
||||
|
||||
@@ -148,6 +148,20 @@ const ASK = {
|
||||
edges: chain('in', 'fetch', 'done'),
|
||||
}
|
||||
|
||||
/**
|
||||
* 调用工具节点里写死的 SQL 对照数据目录查出了问题(数据目录阶段 4B):门禁的问题 code 是规则编号,field 是 args.sql。
|
||||
* 参数里 sql 不在第一行,定位要在 JSON 里找到这个键
|
||||
*/
|
||||
const SQLWF = {
|
||||
nodes: [
|
||||
node('in', 'input', 0, '入口', { fields: [{ name: 'week', required: true }] }),
|
||||
node('sum', 'tool', 240, '订单汇总', { tool: 'db_query__shop', args: { limit: 500, timeout: 30,
|
||||
sql: 'SELECT o.region, SUM(o.amount) AS amt\nFROM orders o JOIN order_items i ON i.order_id = o.id\nGROUP BY o.region' } }),
|
||||
node('done', 'output', 480, '成果', { fields: [{ name: 'table', value: '{{ nodes.sum.output }}' }] }),
|
||||
],
|
||||
edges: chain('in', 'sum', 'done'),
|
||||
}
|
||||
|
||||
const wf = (id, name, graph, extra = {}) => ({
|
||||
id, name, description: '检查脚本伪造的工作流', graph, tags: [], version: 3, is_template: false,
|
||||
status: 'draft', published_version: null, published_by: null, run_count: 0,
|
||||
@@ -164,6 +178,7 @@ const FAKES = {
|
||||
'pf-defs': wf('pf-defs', '__publish_check_defaults__', { ...EASY, defaults: { approval: 'always', model: 'demo-model' } }),
|
||||
'pf-legacy': wf('pf-legacy', '__upgrade_check_legacy__', LEGACY, { status: 'published', published_version: 3 }),
|
||||
'pf-ask': wf('pf-ask', '__upgrade_check_ask__', ASK),
|
||||
'pf-sql': wf('pf-sql', '__publish_check_sql__', SQLWF),
|
||||
}
|
||||
|
||||
const clone = (x) => JSON.parse(JSON.stringify(x))
|
||||
@@ -199,6 +214,11 @@ function lint(graph, level, { followsNever = false } = {}) {
|
||||
}
|
||||
for (const n of graph.nodes) {
|
||||
const c = n.data.config ?? {}
|
||||
// 基于数据目录的 SQL 检查(governance._lint_sql):错误级在受管档是硬性问题,已发布档只提示;没有确定性的修复
|
||||
if (n.type === 'tool' && /JOIN order_items/i.test(c.args?.sql ?? '')) {
|
||||
issues.push({ level: hard ? 'error' : 'warning', node_id: n.id, code: 'fanout_sum', fix: null, field: 'args.sql',
|
||||
message: '「订单」关联「订单明细」是一对多,对「订单」的「订单金额」求和会重复计算。请先按订单汇总明细再关联' })
|
||||
}
|
||||
if (followsNever && n.type === 'agent' && !c.approval && (c.tools ?? []).length) {
|
||||
const id = 'governed.default_approval_never:graph'
|
||||
issues.push({ level: hard ? 'error' : 'warning', node_id: n.id, code: 'governed.default_approval_never', fix: id, field: 'approval',
|
||||
@@ -806,6 +826,44 @@ await section('问题面板的发布前检查:选档、同一套修法、点
|
||||
check('切回「校验」照旧是原来的问题清单', await page.locator('[data-preflight="panel"]').count() === 0)
|
||||
})
|
||||
|
||||
await section('SQL 检查(数据目录阶段 4B):发布前检查写中文规则名,点一条定位到调用工具参数里的 SQL', async () => {
|
||||
const { page, state, errors } = await open({ id: 'pf-sql' })
|
||||
await openDialog(page)
|
||||
const row = dialog(page).locator('[data-preflight-issue="fanout_sum"]')
|
||||
await row.waitFor()
|
||||
const text = await row.innerText()
|
||||
check('发布弹窗:SQL 检查的问题写节点名和中文规则名,不露规则编号', text.includes('「订单汇总」') && text.includes('一对多关联后重复计算:')
|
||||
&& text.includes('会重复计算') && !/fanout_sum|args\.sql/.test(text), text)
|
||||
await shoot(page, 'preflight-sqlcheck', dialog(page))
|
||||
await row.locator('button').first().click()
|
||||
const where = () => page.evaluate(() => {
|
||||
const box = document.querySelector('[data-inspector-sheet] [data-field="args"] textarea')
|
||||
const line = box ? box.value.slice(0, box.value.indexOf('"sql"')).split('\n').length - 1 : -1
|
||||
const height = box ? parseFloat(getComputedStyle(box).lineHeight) || 16 : 16
|
||||
return { sel: window.__studio.getState().selectedId, field: !!box, line, top: box?.scrollTop ?? -1, visible: box ? box.scrollTop <= line * height : false }
|
||||
})
|
||||
check('点一条:弹窗关掉,选中节点并翻到「参数」,框内滚到 sql 那一行', await waitFor(page, async () => {
|
||||
const w = await where()
|
||||
return w.sel === 'sum' && w.field && w.line > 0 && w.visible
|
||||
}) && await dialog(page).count() === 0, JSON.stringify(await where()))
|
||||
// 问题面板的发布前检查:同一个落点
|
||||
await page.evaluate(() => window.__studio.getState().select(null))
|
||||
await page.locator('.react-flow__pane').click({ position: { x: 30, y: 30 } })
|
||||
await page.keyboard.press('Alt+KeyP')
|
||||
await page.locator('#dock-problems').waitFor()
|
||||
await page.locator('[data-problems-mode="publish"]').click()
|
||||
const panelRow = page.locator('[data-preflight="panel"] [data-preflight-issue="fanout_sum"]')
|
||||
await panelRow.waitFor()
|
||||
check('问题面板的发布前检查:同样写中文规则名', (await panelRow.innerText()).includes('一对多关联后重复计算:'))
|
||||
await panelRow.locator('button').first().click()
|
||||
check('……点一条同样落到「参数」里的 SQL', await waitFor(page, async () => {
|
||||
const w = await where()
|
||||
return w.sel === 'sum' && w.field && w.visible
|
||||
}), JSON.stringify(await where()))
|
||||
check('没有发布、没有保存', state.publishes.length === 0 && state.patches.length === 0)
|
||||
check('没有运行时报错', errors.length === 0, errors[0] ?? '')
|
||||
})
|
||||
|
||||
await section('同一个修复只画一次:validate 和门禁各报一条、几条问题共用一个图级修复', async () => {
|
||||
const { page, state, errors } = await open({ id: 'pf-dup' })
|
||||
await openDialog(page)
|
||||
|
||||
@@ -1652,6 +1652,65 @@ await section('画布右栏:去审批、放弃、出具横幅、回执(REQ-1
|
||||
await page.evaluate(() => window.__studio.getState().select(null))
|
||||
await page.waitForTimeout(200)
|
||||
|
||||
// 对照数据目录的 SQL 检查(数据目录阶段 4B):自查没修好的错误、不挡运行的提醒各一组,写中文规则名、级别、表和列、
|
||||
// SQL 片段,不露规则编号;「定位」落到调用工具参数里的 SQL,光标放在 sql 的值开头
|
||||
const SQL = 'SELECT o.region, SUM(o.amount) AS amt\nFROM orders o JOIN order_items i ON i.order_id = o.id\nGROUP BY o.region'
|
||||
await page.evaluate((sql) => {
|
||||
const st = window.__studio.getState()
|
||||
window.__studio.setState({ nodes: [...st.nodes, { id: 'fetch', type: 'card', position: { x: 620, y: 200 },
|
||||
data: { nodeType: 'tool', label: '订单汇总', config: { tool: 'db_query__shop', args: { limit: 100, sql } } } }] })
|
||||
}, SQL)
|
||||
const sqlErr = { level: 'error', node_id: 'fetch', edge_id: null, field: 'args.sql', code: 'fanout_sum', table: 'orders', column: 'amount',
|
||||
sql_excerpt: 'SUM(o.amount)', message: '「订单」关联「订单明细」是一对多,对「订单」的「订单金额」求和会重复计算。请先按订单汇总明细再关联' }
|
||||
const sqlWarn = { level: 'warning', node_id: 'fetch', edge_id: null, field: 'args.sql', code: 'missing_valid_filter', table: 'orders',
|
||||
message: '「订单」定义了有效记录条件「status <> 9」,查询中没有按它筛选' }
|
||||
await page.evaluate(([err, warn]) => window.__studio.setState({ copilotTurns: [{ explanation: '', error: '', phase: 'done', id: 'u8',
|
||||
instruction: '按地区汇总订单金额', outcome: 'applied', diff: { added: ['fetch'], changed: [], removed: [], total: 1 },
|
||||
ops: [{ op: 'add_node', node: { id: 'fetch', type: 'tool', data: { label: '订单汇总' } } },
|
||||
{ op: 'check', status: 'repairing', round: 1, issues: [err] },
|
||||
{ op: 'check', status: 'failed', issues: [err] },
|
||||
{ op: 'final', graph: { nodes: [{ id: 'in' }, { id: 'fetch' }] }, issues: [err, warn] }] }] }), [sqlErr, sqlWarn])
|
||||
await page.waitForTimeout(400)
|
||||
const blocking = page.locator('[data-issue-list=""] [data-issue-code="fanout_sum"]')
|
||||
const blockingText = await blocking.innerText().catch(() => '')
|
||||
check('自查没修好的 SQL 问题:级别、中文规则名、涉及的表和列、说明、SQL 片段', blockingText.includes('错误') && blockingText.includes('一对多关联后重复计算')
|
||||
&& blockingText.includes('orders.amount') && blockingText.includes('会重复计算') && blockingText.includes('SUM(o.amount)')
|
||||
&& blockingText.includes('「订单汇总」'), blockingText.replace(/\s+/g, ' '))
|
||||
const notes = page.locator('[data-issue-list="sql"]')
|
||||
const notesText = await notes.innerText().catch(() => '')
|
||||
check('不挡运行的提醒单列一组「SQL 检查」,没修好的错误不重复', notesText.startsWith('SQL 检查') && notesText.includes('提醒')
|
||||
&& notesText.includes('未筛选有效记录') && !notesText.includes('一对多关联后重复计算'), notesText.replace(/\s+/g, ' '))
|
||||
const panelText = await page.locator('[data-assistant-panel]').innerText()
|
||||
check('助手面板不露规则编号', !/fanout_sum|missing_valid_filter|args\.sql/.test(panelText))
|
||||
const sqlRow = page.locator('[data-turn="u8"] [data-step-code="sql_check"]')
|
||||
check('过程里有一行「SQL 检查:1 处提醒」', (await sqlRow.innerText().catch(() => '')).includes('SQL 检查:1 处提醒'),
|
||||
await page.locator('[data-turn="u8"]').innerText().then((t) => t.replace(/\s+/g, ' ').slice(0, 240)))
|
||||
if (SHOTS) {
|
||||
for (const theme of ['dark', 'light']) {
|
||||
await page.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme)
|
||||
await page.waitForTimeout(200)
|
||||
await page.locator('[data-assistant-panel]').screenshot({ path: `${SHOTS}/stream-panel-sqlcheck-${theme}.png` })
|
||||
}
|
||||
await page.evaluate(() => document.documentElement.setAttribute('data-theme', 'dark'))
|
||||
}
|
||||
await blocking.getByRole('button', { name: /定位/ }).click()
|
||||
await page.waitForTimeout(600)
|
||||
const sqlFocus = await page.evaluate((sql) => {
|
||||
const a = document.activeElement
|
||||
const at = a instanceof HTMLTextAreaElement ? a.selectionStart : -1
|
||||
return { sel: window.__studio.getState().selectedId, tag: a?.tagName ?? null,
|
||||
field: a?.closest('[data-inspector-sheet] [data-field]')?.getAttribute('data-field') ?? null,
|
||||
atSql: at >= 0 && (a.value.slice(at).startsWith(sql.split('\n')[0]) || a.value.slice(at).startsWith(JSON.stringify(sql).slice(1, 30))) }
|
||||
}, SQL)
|
||||
check('定位 SQL 检查的问题:打开调用工具的设置,光标落在「参数」里 sql 的值开头', sqlFocus.sel === 'fetch' && sqlFocus.field === 'args'
|
||||
&& sqlFocus.tag === 'TEXTAREA' && sqlFocus.atSql, JSON.stringify(sqlFocus))
|
||||
await page.evaluate(() => {
|
||||
const st = window.__studio.getState()
|
||||
st.select(null)
|
||||
window.__studio.setState({ copilotTurns: [], nodes: st.nodes.filter((n) => n.id !== 'fetch') })
|
||||
})
|
||||
await page.waitForTimeout(200)
|
||||
|
||||
// 模型把工具调用写成文字、判失败:报错和提醒行给「打开『数据查询』的设置」,点了就是那个节点的属性面板
|
||||
const MARKUP_ERROR = '模型以文本形式输出了工具调用的原始标记,未实际调用工具,这一步没有查询到任何数据。常见原因:节点未绑定工具,或模型、服务不支持工具调用。请在画布中为该节点绑定所需工具;如已绑定仍出现此问题,请换用支持工具调用的模型'
|
||||
await page.evaluate((run) => {
|
||||
|
||||
Reference in New Issue
Block a user