合入集成分支(阶段 2A 数据剖析、4B 剖析与 SQL 检查界面及修正)到 4A:两边功能都保留

冲突的解法:
- 后端 api/catalog.py:同时导入 catalog_impact 和 catalog_profile、SnapshotTampered;data/catalog.py:__all__ 取两边的并集重排。
- 2A 放宽了码值校验(含义可以为空,剖析写进来的是「含义待填写」):目录修改提案的码值补充改为含义为空的码不覆盖已有含义、
  只在原来没有这个码时加进去,补了测试;卡片和过程里的码值把空含义写成「含义待填写」,和目录页同一个说法。
- 前端 types.ts、terms.ts:两边各在同一处追加了新段落,按顺序都留下(提案、影响面、目录版本在前,剖析、码值、SQL 检查在后),
  没有重复的键;client.ts、decode.ts、CatalogPage.tsx、TableDetail.tsx 的导入取并集。
- check-catalog.mjs:影响面和剖析、保存数据源三条假路由都留下,各节和最后核对真库数量的逻辑不变。
- 4A 的测试数据源改用 tests/fixtures/sources.py 的 drop_source 删除,顺带作废连接池。
This commit is contained in:
ThearchyHelios
2026-10-03 12:35:47 +08:00
51 changed files with 4803 additions and 147 deletions
+65 -2
View File
@@ -8,9 +8,10 @@
- POST /catalog/{table}/patch 保存目录修改提案(乐观锁),记为人工确认
- GET /catalog/{table}/impact 影响面:引用这张表的已发布、受管模板(data/catalog_impact.py)
- POST /catalog/draft 同步起草(注释、外键、命名推断,可选模型)
- POST /catalog/profile 同步剖析:对业务库发少量只读查询,核对关系、取码值候选(data/catalog_profile.py)
规则都在 data/catalog.py,这里只做取源、转换形状和把异常翻成状态码:版本不符 409,结构不合规、审阅路径
不对 422,源或表不存在 404。所有写操作记录请求头 X-Actor 的署名(自报、未认证)。
不对 422,源或表不存在 404。所有写操作记录请求头 X-Actor 的署名(自报、未认证)。剖析没开启、正在剖析 409。
上传表格的源按当前快照的表结构回答(table_versions.resolve_source),和查询实际用的库是同一个版本。
"""
@@ -23,7 +24,8 @@ from pydantic import BaseModel, Field
from sqlalchemy.ext.asyncio import AsyncSession
from app.api.runs import actor_of
from app.data import catalog, catalog_impact, introspect, table_versions
from app.data import catalog, catalog_impact, catalog_profile, introspect, table_versions
from app.data.engine import SnapshotTampered
from app.db.base import get_session
from app.db.models import DataSource
@@ -65,6 +67,11 @@ class CatalogDraftIn(BaseModel):
use_model: bool = False
class CatalogProfileIn(BaseModel):
#: 要剖析的表;不给或为空时按使用次数取前 10 张目录里有待核实关系的表
tables: list[str] | None = Field(default=None, max_length=200)
async def _resolved(session: AsyncSession, source_id: str) -> tuple[DataSource, Any]:
"""(数据源记录, 用来读表结构的源):上传源解析成绑定当前快照的视图。"""
row = await session.get(DataSource, source_id)
@@ -227,6 +234,62 @@ async def draft_catalog(source_id: str, payload: CatalogDraftIn,
}
def _profile_table_out(t: catalog_profile.TableProfile) -> dict[str, Any]:
return {
"table_name": t.table,
"queries": t.queries,
"row_estimate": t.row_estimate.to_dict() if t.row_estimate else None,
"findings": t.findings,
"skipped": [s.to_dict() for s in t.skipped],
"date_ranges": t.date_ranges,
"added": t.added,
"updated": t.updated,
"removed": t.removed,
"version": t.version,
"error": t.error,
}
@router.post("/{source_id}/catalog/profile")
async def profile_catalog_tables(source_id: str, payload: CatalogProfileIn,
x_actor: str | None = Header(default=None),
session: AsyncSession = Depends(get_session)) -> dict[str, Any]:
"""同步剖析。返回每张表用了几条查询、写进目录的发现、跳过了什么及原因。
默认关闭:数据源设置里没开启数据剖析、数据源停用,409;同一个数据源已经在剖析,409。
"""
row, source = await _resolved(session, source_id)
try:
catalog_profile.ensure_enabled(row)
except catalog_profile.ProfileDisabled as e:
raise HTTPException(409, str(e)) from e
cache = source.schema_cache or {}
if not cache.get("tables"):
raise HTTPException(409, f"{introspect.why_empty(cache)},无法进行数据剖析。请先在数据源卡片上点「探查结构」")
actor = actor_of(x_actor)
try:
report = await catalog_profile.profile_catalog(session, row, source, tables=payload.tables or None,
actor=actor)
except (catalog_profile.ProfileDisabled, catalog_profile.ProfileBusy) as e:
raise HTTPException(409, str(e)) from e
except SnapshotTampered as e:
raise HTTPException(409, str(e)) from e
rows = [_profile_table_out(t) for t in report.tables]
note = None
if not rows:
note = "目录里还没有待核实的关联关系,请先起草数据目录,或指定要剖析的表"
return {
"profiled_at": report.profiled_at,
"actor": actor,
"settings": report.settings.to_dict(),
"queries_used": report.queries_used,
"stopped": report.stopped,
"tables": rows,
"total": {k: sum(r[k] for r in rows) for k in ("added", "updated", "removed")},
"note": note,
}
@router.post("/{source_id}/catalog/{table}/review")
async def review_catalog_item(source_id: str, table: str, payload: CatalogReviewIn,
x_actor: str | None = Header(default=None),
+3 -1
View File
@@ -23,6 +23,7 @@ from app.core.errors import AUTH, NETWORK, TIMEOUT, classify, explain, raw
from app.core.config import settings
from app.core.crypto import encrypt, mask
from app.data import catalog as data_catalog
from app.data.catalog_profile import profile_settings_problem
from app.data import introspect as introspect_mod
from app.data import table_versions
from app.data.engine import (
@@ -336,7 +337,8 @@ _MASK_FIELD = {
def _refuse_bad_options(options: dict[str, Any] | None) -> None:
problem = query_timeout_problem(options) or mask_columns_problem(options)
problem = (query_timeout_problem(options) or mask_columns_problem(options)
or profile_settings_problem(options))
if problem:
raise HTTPException(422, problem)
+80 -18
View File
@@ -15,13 +15,14 @@ notes 结构(每个「项」都是 {value, source, status},可选 note、upd
"valid_filter": 项(str,SQL 条件片段), "dedup": 项(str),
"columns": {列名: {"label": 项, "meaning": 项, "unit": 项,
"measure": 项("flow"|"stock"|"ratio"|"identifier"|"status"|"attribute"),
"codes": 项(dict[str, str])}},
"codes": 项(dict[码值, 含义];含义可为空串,表示待填写)}},
"relations": [{"id", "columns", "to_table", "to_columns", "cardinality", "coverage",
"source", "status", "note"?}]}
来源(source):comment 数据库注释、fk 外键约束、name 命名推断、profile 数据剖析(阶段 2)、llm 模型起草、
human 人工填写。状态(status):proposed 推断(命名推断、模型起草、数据库注释)、verified 有确证(外键
约束;阶段 2 的数据剖析也写这个状态)、confirmed 人工确认、rejected 人工驳回。只有 confirmed 和 verified
来源(source):comment 数据库注释、fk 外键约束、name 命名推断、profile 数据剖析(data/catalog_profile.py)、
llm 模型起草、human 人工填写。状态(status):proposed 推断(命名推断、模型起草、数据库注释)、verified 有确证
(外键约束;数据剖析核实了覆盖率和父键唯一的关系也是,见 profile_relation_holds)、confirmed 人工确认、
rejected 人工驳回。只有 confirmed 和 verified
将来会触发错误级检查;给模型看时 proposed 的项标「推断,未确认」;rejected 的项留着(防止下次起草又提出
来),但模型和检查都看不到。
@@ -98,6 +99,33 @@ def initial_status(source: str) -> str:
return _INITIAL_STATUS.get(source, "proposed")
#: 数据剖析把关系升为有确证的抽样覆盖率下限(data/catalog_profile.py 判定,撤销审阅时按它判回剖析的结论)
PROFILE_VERIFY_COVERAGE = 0.95
def profile_relation_holds(rel: Mapping[str, Any]) -> bool:
"""一条数据剖析得出的关系站不站得住:抽样覆盖率不低于 PROFILE_VERIFY_COVERAGE,而且被指向列核实唯一。
剖析只在被指向列核实唯一时才写基数(多对一或一对一),所以「基数有值」就是「父键唯一」。结论全在关系
自己的字段里,撤销审阅时照原样判得回来,不必另存一份「审阅前的状态」。
"""
coverage = rel.get("coverage")
return (rel.get("source") == "profile" and isinstance(coverage, (int, float)) and not isinstance(coverage, bool)
and coverage >= PROFILE_VERIFY_COVERAGE and rel.get("cardinality") in ("many_to_one", "one_to_one"))
def unreviewed_status(item: Mapping[str, Any]) -> str:
"""撤销审阅后回到的状态:来源的初始状态。
数据剖析是例外:它的状态随数据而定(覆盖率够、父键唯一的关系是有确证,其余是推断),不能一律回到
initial_status("profile") 的推断——那样人工确认过再撤销,有确证就无故丢了,要等下一次剖析才回来。
剖析提的码值候选、业务日期永远是推断。
"""
if item.get("source") == "profile" and "to_table" in item:
return "verified" if profile_relation_holds(item) else "proposed"
return initial_status(item.get("source"))
def make_item(value: Any, source: str, status: str | None = None, *, note: str | None = None,
at: str | None = None) -> dict[str, Any]:
"""构造一个项。status 不给时按来源取初始状态(initial_status)。"""
@@ -153,8 +181,10 @@ def _business_date_ok(value: Any) -> bool:
def _codes_ok(value: Any) -> bool:
"""码值对照:{码值: 含义},都是文字。含义可以是空串——数据剖析只知道列里出现过哪些取值,含义要等人填;
以前要求含义非空,剖析得到的码值候选就写不进来。"""
return (isinstance(value, dict) and bool(value)
and all(isinstance(k, str) and isinstance(v, str) and v.strip() for k, v in value.items()))
and all(isinstance(k, str) and isinstance(v, str) for k, v in value.items()))
#: 表类型、度量类型在界面上的叫法(和 frontend/src/lib/terms.ts 的 CATALOG_KIND_LABEL / CATALOG_MEASURE_LABEL 一致)。
@@ -587,7 +617,7 @@ def review_item(notes: Mapping[str, Any] | None, path: str, action: str, *, at:
# 人工填写的项没有「推断时的样子」可回:撤销就是删掉,下次起草可以重新提出
del slots[key]
return _assemble(slots)
item["status"] = {"confirm": "confirmed", "reject": "rejected"}.get(action) or initial_status(item.get("source"))
item["status"] = {"confirm": "confirmed", "reject": "rejected"}.get(action) or unreviewed_status(item)
item["updated_at"] = at or now_iso()
return _assemble(slots)
@@ -652,7 +682,7 @@ def apply_human_edit(existing: Mapping[str, Any] | None, submitted: Mapping[str,
continue
kept = dict(prev)
status = item.get("status")
if status != prev.get("status") and status in ("confirmed", "rejected", initial_status(prev.get("source"))):
if status != prev.get("status") and status in ("confirmed", "rejected", unreviewed_status(prev)):
kept["status"] = status
kept["updated_at"] = at
if item.get("note") != prev.get("note"):
@@ -779,6 +809,17 @@ def _patch_relation(table: str, raw: Any, path_id: str | None, tables: Mapping[s
return value, None
def _merge_codes(before: Mapping[str, str], value: Mapping[str, str]) -> dict[str, str]:
"""码值的补充:给出的码加进去或改掉原来的含义。含义为空表示「只知道有这个码、含义待填写」(数据剖析的码值
候选就是这样),所以提案里含义为空的码不覆盖已有的含义,只在原来没有这个码时加进去。"""
out = dict(before)
for code, meaning in value.items():
if not str(meaning).strip() and str(out.get(code) or "").strip():
continue
out[code] = meaning
return out
def plan_patch(notes: Mapping[str, Any] | None, changes: Any, *, table: str,
tables: Mapping[str, Any] | None = None) -> PatchPlan:
"""逐项核对一条目录修改提案,算出改前、改后。不改入参、不抛异常:不合法的项写进 problems。
@@ -790,7 +831,8 @@ def plan_patch(notes: Mapping[str, Any] | None, changes: Any, *, table: str,
规则:
- 取值格式和 validate_notes 同一套(_TABLE_VALUE_RULES / _COLUMN_VALUE_RULES / _relation_problems)。
- 码值是补充:在现有(没被驳回的)码值上加或改给出的码,不整份替换——「status=9 表示作废」不该把已确认的
其余码值删掉。
其余码值删掉。含义为空的码(数据剖析写进来的「含义待填写」)可以被补上含义;提案里含义为空的码只在原来
没有这个码时加进去,不会把已有的含义抹掉(_merge_codes)。
- 看起来像计算公式的文字不收(_FORMULA):公式进口径卡。
- 同一个路径出现两次只认第一次。
"""
@@ -864,7 +906,7 @@ def plan_patch(notes: Mapping[str, Any] | None, changes: Any, *, table: str,
before = _value_of(key, live) if live is not None else None
after = value
if key[0] == "c" and key[2] == "codes" and isinstance(before, dict):
after = {**before, **value}
after = _merge_codes(before, value)
if before is not None and before == after:
state = "same" if live.get("status") == "confirmed" else "confirm"
else:
@@ -1094,6 +1136,15 @@ def infer_name_relations(schema_cache: Mapping[str, Any] | None, table: str, *,
candidates: set[str] = set()
for form in _singulars(words[-1]):
candidates |= index.get((prefix, form), set())
if not candidates:
# 直接对不上:库里的表常带业务前缀(store_tags),列名却只写词根(tagId)。
# 只认和引用它的表共用前缀的那张,从最长的共用前缀试起;仍然不止一张就不推
own = _words(table)
for k in range(len(own) - 1, 0, -1):
for form in _singulars(words[-1]):
candidates |= index.get(("".join(own[:k]) + prefix, form), set())
if candidates:
break
if len(candidates) != 1:
continue
target = next(iter(candidates))
@@ -1625,6 +1676,17 @@ def _date_text(value: Mapping[str, Any]) -> str:
return ";".join(parts)
def _codes_text(codes: Mapping[str, Any]) -> str:
"""码值对照的文字:有含义的写「1=有效」,含义待填写的单独说明——不能写成「1=」,模型会当成含义就是空。"""
filled = [f"{k}={v}" for k, v in codes.items() if str(v).strip()]
pending = [str(k) for k, v in codes.items() if not str(v).strip()]
if not pending:
return "、".join(filled)
if not filled:
return f"{'、'.join(pending)}(含义待填写)"
return f"{'、'.join(filled)},{'、'.join(pending)} 的含义待填写"
def _relation_text(rel: Mapping[str, Any]) -> str:
left = ", ".join(rel.get("columns") or [])
right = ", ".join(f"{rel.get('to_table')}.{c}" for c in rel.get("to_columns") or [])
@@ -1675,7 +1737,7 @@ def _notes_lines(notes: Mapping[str, Any] | None, *, meta: Mapping[str, Any] | N
if name == "measure":
text = MEASURE_LABEL.get(value, str(value))
elif name == "codes":
text = "、".join(f"{k}={v}" for k, v in value.items())
text = _codes_text(value)
else:
text = str(value)
# 第一段是中文名(没有中文名时是含义),不加前缀;其余写明是什么
@@ -1932,13 +1994,13 @@ __all__ = [
"CATALOG_DRAFT_SYSTEM", "COLUMN_FIELDS", "COLUMN_FIELD_LABEL", "COLUMN_MEASURES", "CatalogConflict",
"CatalogEntry", "CatalogInvalid", "CatalogModelUnavailable", "CatalogPathError", "DRAFT_DEFAULT_TABLES",
"DraftReport", "INFERRED_MARK", "ITEM_SOURCES", "ITEM_STATUSES", "JoinEdge", "KIND_LABEL", "MEASURE_LABEL",
"MergeStats", "NEW_RELATION_PATH", "PATCH_MAX_CHANGES", "PatchChange", "PatchPlan", "QUERY_SNAPSHOT_KIND",
"REVIEW_ACTIONS", "TABLE_FIELDS", "TABLE_FIELD_LABEL", "TABLE_KINDS", "TableDraft", "TableDraftResult",
"UI_KIND_LABEL", "UI_MEASURE_LABEL", "apply_human_edit", "apply_patch", "describe_slot", "draft_catalog",
"draft_structure", "draft_with_model", "fk_relations", "frozen_catalog", "infer_name_relations",
"MergeStats", "NEW_RELATION_PATH", "PATCH_MAX_CHANGES", "PROFILE_VERIFY_COVERAGE", "PatchChange", "PatchPlan",
"QUERY_SNAPSHOT_KIND", "REVIEW_ACTIONS", "TABLE_FIELDS", "TABLE_FIELD_LABEL", "TABLE_KINDS", "TableDraft",
"TableDraftResult", "UI_KIND_LABEL", "UI_MEASURE_LABEL", "apply_human_edit", "apply_patch", "describe_slot",
"draft_catalog", "draft_structure", "draft_with_model", "fk_relations", "frozen_catalog", "infer_name_relations",
"initial_status", "join_paths", "make_item", "merge_notes", "now_iso", "parse_path", "plan_patch",
"query_snapshot_meta", "read_catalog", "read_entry", "relation_graph", "relation_id", "render_table_index",
"render_table_notes", "resolve_draft_model", "resolve_table_name", "review_entry", "review_item", "same_notes",
"save_human_edit", "save_patch", "status_counts", "system_notes_source", "table_usage", "validate_notes",
"visible_notes", "write_entry",
"profile_relation_holds", "query_snapshot_meta", "read_catalog", "read_entry", "relation_graph", "relation_id",
"render_table_index", "render_table_notes", "resolve_draft_model", "resolve_table_name", "review_entry",
"review_item", "same_notes", "save_human_edit", "save_patch", "status_counts", "system_notes_source",
"table_usage", "unreviewed_status", "validate_notes", "visible_notes", "write_entry",
]
File diff suppressed because it is too large Load Diff
+6 -1
View File
@@ -58,11 +58,16 @@ MAX_QUERY_TIMEOUT_S = 600
#: 完整快照仍能按工件 id 取到,SQL 里给列起个别名也能绕开
MASK_COLUMNS_OPTION = "mask_columns"
#: 数据源 options 里的数据剖析设置(一个对象:开关、查询次数上限、单条时限……)。取值和校验见
#: data/catalog_profile.py;放在这里只为让拼连接串时摘掉它
CATALOG_PROFILE_OPTION = "catalog_profile"
# options 里这些 key 是 AgentLab 自己的配置,不是驱动参数,拼 URL 时要摘掉。
# schema:探查哪个 schema(企业库里只读账号名下常常什么都没有,数据在别处)
# query_timeout_s:查询时限,由数据层按语句下发给数据库,见 _server_deadline
# mask_columns:证据面板遮罩的列,驱动不认识它,拼进连接串会被当成未知参数拒掉
_NON_DRIVER_OPTIONS = frozenset({"schema", QUERY_TIMEOUT_OPTION, MASK_COLUMNS_OPTION})
# catalog_profile:数据剖析设置,同上;它还是个对象,拼进去连 URL 都不成形
_NON_DRIVER_OPTIONS = frozenset({"schema", QUERY_TIMEOUT_OPTION, MASK_COLUMNS_OPTION, CATALOG_PROFILE_OPTION})
_MASK_SPLIT = re.compile(r"[,,、;;\n]+")
+42 -2
View File
@@ -169,13 +169,17 @@ class SqlChecker:
和助手挑表看到的是同一张图(catalog.relation_graph):目录里的关系,加上没起草过的表按外键、命名现推的;
被驳回的关系不进图,也不会被现推回来。
"""
return [e for e in self.edges_from(a) if e.to_table.lower() == b.lower()]
def edges_from(self, a: str) -> list[catalog.JoinEdge]:
"""从 a 出发的全部边。关系图第一次用到时才建,之后复用。"""
if self._edges is None:
graph = catalog.relation_graph(self._raw, schema_cache=self.schema_cache)
index: dict[str, list[catalog.JoinEdge]] = {}
for table, edges in graph.items():
index.setdefault(table.lower(), []).extend(edges)
self._edges = index
return [e for e in self._edges.get(a.lower(), []) if e.to_table.lower() == b.lower()]
return list(self._edges.get(a.lower(), []))
# ---- 检查
@@ -557,10 +561,46 @@ class _ScopeCheck:
continue # 两张表都没写目录:没有可对照的,不判
condition = " AND ".join(filter(None, (self.excerpt(c) for c in group["conds"])))
if best is None:
self._join_checks.append(self._join_unknown(a, b, candidates, condition))
parent = self._shared_parent(a, b, pairs)
self._join_checks.append(self._join_sibling(a, b, parent, condition) if parent
else self._join_unknown(a, b, candidates, condition))
elif best.status not in _STRONG:
self._join_checks.append(self._join_proposed(a, b, best, condition))
def _shared_parent(self, a: _Table, b: _Table, pairs: set[tuple[str, str]]) -> str | None:
"""兄弟关联:每一对连接列都是两边各自指向同一张父表的同一组列(a.member_id → members.id ← b.member_id)。
返回父表名;不是这种连法返回 None。
目录里不会记 a 和 b 之间的关系,但这不是「连错了列」:比如评价表和标签关联表各自按门店编号指向门店、
彼此按门店编号相连。照「对不上任何关系」报警告是误报。
"""
def parents(table: _Table, column: str) -> set[tuple[str, tuple[str, ...]]]:
return {(e.to_table.lower(), tuple(c.lower() for c in e.to_columns))
for e in self.checker.edges_from(table.name)
if tuple(c.lower() for c in e.from_columns) == (column,)
and e.cardinality in (None, "many_to_one", "one_to_one")}
shared: set[tuple[str, tuple[str, ...]]] | None = None
for left, right in pairs:
common = parents(a, left) & parents(b, right)
shared = common if shared is None else shared & common
if not shared:
return None
names = {table for table, _ in shared or ()}
return next(iter(names)) if len(names) == 1 else None
def _join_sibling(self, a: _Table, b: _Table, parent: str, condition: str) -> SqlCheck:
item = (self.checker.notes_of(parent) or {}).get("label")
label = str(item["value"]) if isinstance(item, dict) and isinstance(item.get("value"), str) \
and item["value"].strip() else parent
return SqlCheck(
code="join_unconfirmed", level="info", table=b.name, sql_excerpt=condition or None,
message=(f"「{a.label()}」与「{b.label()}」之间没有直接的关系,这里经共同的「{label}」相连。"
"关联后行数可能成倍增加:确认其中一侧每个键只有一行,或先各自汇总再关联"),
for_model=(f"{a.name} 和 {b.name} 按 {condition or '这组条件'} 关联:两边都指向 {parent},彼此是一对多对多。"
"如果其中一侧每个键不止一行,关联后另一侧的行会被重复,求和、计数会放大;"
"先把一侧筛到每个键一行(例如只取主记录),或分别汇总到这个键再关联"))
def _join_unknown(self, a: _Table, b: _Table, candidates: list[catalog.JoinEdge], condition: str) -> SqlCheck:
known = ";".join(dict.fromkeys(e.condition() for e in candidates))
hint = f"目录里这两张表的关系是:{known}。照它改关联条件" if known else \
+5 -3
View File
@@ -9,8 +9,9 @@
从 visits 到 channels 要经过它,正好是两跳的连接路径。group_visits 同理(旅行团与入园记录)。
- **命名推断必须放过的列**(NOT_INFERRED):自引用(employees.employee_id 是工号,不是指向自己)、
多个候选(guide 和 guides 两张表都在,tour_assignments.guide_id 不知道指哪张)、没有对应的表
(parking_records.lot_id、operation_logs.operator_id、categories.parentId)、类型对不上
(feedback_forms.store_id 是文字,stores.id 是整数)。
(operation_logs.operator_id、categories.parentId)、类型对不上(feedback_forms.store_id 是文字,stores.id 是整数)。
parking_records.lot_id 没有直接对上的 lot / lots,但和引用它的表共用 parking 前缀的 parking_lots 只有一张,
推得出来(在 NAME_RELATIONS 里)。
- **唯一约束 / 唯一索引。** orders.order_no、channels.channel_code 是列上的 UNIQUE;members.member_no、
visits.ticket_no 是单独建的唯一索引。
@@ -188,6 +189,8 @@ NAME_RELATIONS = frozenset({
("tour_groups", ("agency_id",), "agencies", ("id",)),
("tour_assignments", ("tour_group_id",), "tour_groups", ("id",)),
("lost_and_found", ("park_id",), "parks", ("id",)),
# 没有 lot / lots,按引用表共用的 parking 前缀对上 parking_lots
("parking_records", ("lot_id",), "parking_lots", ("id",)),
# 视图没有约束,按列名同样推得出来:按日汇总的入园人数照样能按 park_id 连到景区
("v_daily_visits", ("park_id",), "parks", ("id",)),
})
@@ -196,7 +199,6 @@ NAME_RELATIONS = frozenset({
NOT_INFERRED = {
("employees", "employee_id"): "自引用:工号,不是指向本表",
("tour_assignments", "guide_id"): "多个候选:guide 与 guides",
("parking_records", "lot_id"): "没有名为 lot / lots 的表",
("operation_logs", "operator_id"): "没有名为 operator / operators 的表",
("categories", "parentId"): "没有名为 parent / parents 的表",
("feedback_forms", "store_id"): "类型对不上:文字列指向整数主键",
+26
View File
@@ -0,0 +1,26 @@
"""测试里建的数据源,用完就删。
数据源表在每个测试进程的测试库里是共用的:留下来的源会出现在助手的数据源清单里(50 张表的合成库还会
触发挑表、多一次模型调用),别的用例的结果就跟着执行顺序变。阶段 1B 就是这样被绊过一次。
删数据源时它的数据目录随外键级联删除(测试库开着 PRAGMA foreign_keys=ON)。
"""
from __future__ import annotations
from sqlalchemy import delete, select
from app.data.engine import engines
from app.db.base import SessionLocal
from app.db.models import DataSource
async def drop_source(source_id: str | None = None, *, name: str | None = None) -> None:
"""按 id 或名字删一个数据源,并作废它的连接池。不存在就什么都不做。"""
async with SessionLocal() as session:
if source_id is None:
source_id = (await session.execute(
select(DataSource.id).where(DataSource.name == name))).scalar_one_or_none()
if source_id is None:
return
await session.execute(delete(DataSource).where(DataSource.id == source_id))
await session.commit()
await engines.invalidate(source_id)
+8 -5
View File
@@ -11,12 +11,12 @@ import pytest
from httpx import ASGITransport, AsyncClient
from app.data import catalog
from app.data.engine import engines
from app.db.base import SessionLocal
from app.db.models import DataSource
from app.main import app as _fastapi_app
from tests.fixtures.catalog import scenic
from tests.fixtures.catalog.fake_model import FakeModel
from tests.fixtures.sources import drop_source
ACTOR = {"X-Actor": quote("王敏")}
@@ -38,7 +38,7 @@ async def source_id(client, scenic_db):
r = await client.post(f"/api/datasources/{sid}/introspect")
assert r.status_code == 200, r.text
yield sid
await engines.invalidate(sid)
await drop_source(sid)
def _base(sid: str) -> str:
@@ -184,9 +184,12 @@ async def test_draft_without_schema_is_refused(client):
session.add(row)
await session.commit()
sid = row.id
r = await client.post(f"{_base(sid)}/draft", json={})
assert r.status_code == 409 and "探查结构" in r.json()["detail"]
assert (await client.get(_base(sid))).json()["tables"] == []
try:
r = await client.post(f"{_base(sid)}/draft", json={})
assert r.status_code == 409 and "探查结构" in r.json()["detail"]
assert (await client.get(_base(sid))).json()["tables"] == []
finally:
await drop_source(sid)
async def test_schema_endpoint_detail_carries_catalog(client, source_id):
+37
View File
@@ -15,6 +15,7 @@ from app.data.catalog import make_item
from app.db.base import SessionLocal
from app.db.models import DataSource
from tests.fixtures.catalog.fake_model import FakeModel
from tests.fixtures.sources import drop_source
def _col(name: str, type_: str = "INTEGER", comment: str | None = None, nullable: bool = True) -> dict:
@@ -156,6 +157,31 @@ def test_name_inference_negative(tables, why):
assert _infer(tables, next(iter(tables))) == set(), why
def test_name_inference_falls_back_to_a_table_sharing_the_prefix():
"""库里没有 tags,只有带前缀的 store_tags 和 menu_tag:和引用它的表共用 store 前缀的那张才是它要指的。
以前这种命名推不出关系,SQL 检查跟着误报「对不上任何关系」。"""
tables = {
"store_tag_links": _table(_col("id"), _col("tagId"), _col("storeID")),
"store_tags": _table(_col("id")),
"menu_tag": _table(_col("id")),
"store": _table(_col("id")),
}
assert _infer(tables, "store_tag_links") == {
(("tagId",), "store_tags", ("id",)), (("storeID",), "store", ("id",))}
@pytest.mark.parametrize("tables, why", [
# 直接对得上就用直接的,不去找带前缀的
({"shop_category_links": _table(_col("id"), _col("category_id")), "categories": _table(_col("id")),
"shop_categories": _table(_col("id"))}, {(("category_id",), "categories", ("id",))}),
# 引用它的表没有可共用的前缀:带前缀的表不止一张时照旧不推
({"links": _table(_col("id"), _col("tag_id")), "store_tags": _table(_col("id")),
"menu_tags": _table(_col("id"))}, set()),
])
def test_prefix_fallback_only_when_direct_match_is_missing(tables, why):
assert _infer(tables, next(iter(tables))) == why
def test_name_inference_one_to_one_when_column_is_own_key():
tables = {"member_profiles": _table(_col("member_id"), pk=("member_id",)), "members": _table(_col("id"))}
rel = catalog.infer_name_relations(_source(tables).schema_cache, "member_profiles")[0]
@@ -255,6 +281,16 @@ async def test_model_draft_respects_system_notes_of_imported_tables():
# ---------------------------------------------------------------- 起草并写入
_CREATED: list[str] = []
@pytest.fixture(autouse=True)
async def _drop_created_sources():
yield
while _CREATED:
await drop_source(_CREATED.pop())
async def _db_source(tables: dict, **kw) -> DataSource:
for name, meta in tables.items():
meta["qualified"] = name
@@ -263,6 +299,7 @@ async def _db_source(tables: dict, **kw) -> DataSource:
schema_cache={"tables": tables}, **kw)
session.add(row)
await session.commit()
_CREATED.append(row.id)
return row
+3 -3
View File
@@ -19,6 +19,7 @@ from app.data.engine import engines
from app.data.introspect import introspect
# 模型要在建表之前注册进 metadata:conftest 的建表只导入了 app.db.base,这个文件单独跑时没有别的模块先导入它们
from app.db import models # noqa: F401
from tests.fixtures.sources import drop_source
_CACHE: dict[str, dict] = {}
@@ -106,10 +107,9 @@ async def world(scenic_db):
for wid in created:
if (wf := await session.get(Workflow, wid)) is not None:
await session.delete(wf)
for sid in (ids.source, ids.other):
if (row := await session.get(DataSource, sid)) is not None:
await session.delete(row)
await session.commit()
for sid in (ids.source, ids.other):
await drop_source(sid)
async def _impact(source_id: str, table: str) -> dict:
+28 -5
View File
@@ -25,6 +25,7 @@ from app.data import catalog
from app.data.engine import engines
from app.data.introspect import introspect
from tests.fixtures.catalog import scenic_notes
from tests.fixtures.sources import drop_source
_CACHE: dict[str, dict] = {}
@@ -76,6 +77,32 @@ async def test_codes_are_supplemented_not_replaced(scenic_db):
assert change.value == {"9": "作废"}
async def test_codes_with_pending_meanings(scenic_db):
"""数据剖析写进来的码值候选含义为空(界面上写「含义待填写」):提案可以补上含义;提案里含义为空的码只在原来
没有时加进去,不抹掉已有的含义;含义为空的码合法,保存不报错。"""
tables = (await _schema(scenic_db))["tables"]
profiled = _visits()
profiled["columns"]["status"]["codes"] = catalog.make_item({"1": "有效", "0": "", "9": ""}, "profile", "proposed")
plan = catalog.plan_patch(profiled, [{"path": "columns.status.codes", "value": {"9": "作废", "1": ""}}],
table="visits", tables=tables)
assert plan.problems == []
[change] = plan.changes
assert change.after == {"1": "有效", "0": "", "9": "作废"} and change.state == "change"
# 只给了含义为空的新码:照样加进去,含义待人填
plan = catalog.plan_patch(profiled, [{"path": "columns.status.codes", "value": {"7": ""}}], table="visits",
tables=tables)
assert plan.changes[0].after == {"1": "有效", "0": "", "9": "", "7": ""}
# 只给了含义为空的已有码:什么都没变,这一项是「保存即确认」(原来是推断)
plan = catalog.plan_patch(profiled, [{"path": "columns.status.codes", "value": {"1": ""}}], table="visits",
tables=tables)
assert plan.changes[0].state == "confirm"
out = catalog.apply_patch(profiled, [{"path": "columns.status.codes", "value": {"9": "作废"}}], table="visits",
tables=tables)
codes = out["columns"]["status"]["codes"]
assert codes["value"] == {"1": "有效", "0": "", "9": "作废"} and (codes["source"], codes["status"]) == (
"human", "confirmed")
async def test_new_relation_gets_its_id_from_both_ends(scenic_db):
tables = (await _schema(scenic_db))["tables"]
value = {"columns": ["visit_id"], "to_table": "visits", "to_columns": ["id"], "cardinality": "many_to_one"}
@@ -190,11 +217,7 @@ async def scenic_source(scenic_db):
await catalog.write_entry(session, row.id, table, notes, if_version=0, actor="王敏")
source_id = row.id
yield SimpleNamespace(id=source_id, name=name)
async with SessionLocal() as session:
row = await session.get(DataSource, source_id)
if row is not None:
await session.delete(row)
await session.commit()
await drop_source(source_id)
async def _stream(monkeypatch, model, source, base_graph=None):
+430
View File
@@ -0,0 +1,430 @@
"""数据剖析接口:POST /api/datasources/{id}/catalog/profile(阶段 2A)。
剖析是数据目录里唯一对业务库发查询的地方,这里逐条钉住方案定的约束:默认关闭、只读、每条查询都过守卫、
查询次数和单条时限、总时长、遮罩的列不取样、大表不做整表统计、同一个数据源同时只有一次剖析;
以及剖析的结论怎么进目录:命名推断对上数据的升为有确证(来源记为数据剖析,和外键分开)、对不上的写明
覆盖率、人工确认过的只补覆盖率和基数、状态列得到码值候选、只有一个日期列的表提议业务日期。
所有测试都在合成的景区库上做(tests/fixtures/catalog/scenic.py),数据和名称全部虚构。
"""
from __future__ import annotations
import asyncio
import sqlite3
import uuid
from pathlib import Path
from types import SimpleNamespace
from urllib.parse import quote
import pytest
from httpx import ASGITransport, AsyncClient
from sqlalchemy import event
from sqlalchemy.engine import Engine
from app.data import catalog, catalog_profile
from app.data import engine as data_engine
from app.data.catalog_profile import PROFILE_OPTION, ProfileSettings
from app.db.base import SessionLocal
from app.db.models import DataSource
from app.main import app as _fastapi_app
from tests.fixtures.catalog import scenic
from tests.fixtures.sources import drop_source
ACTOR = {"X-Actor": quote("周宁")}
@pytest.fixture
async def client():
async with AsyncClient(transport=ASGITransport(app=_fastapi_app), base_url="http://test") as c:
yield c
def _on(**settings) -> dict:
return {PROFILE_OPTION: {"enabled": True, **settings}}
@pytest.fixture
async def make_source(client):
"""登记一个接上 path 的数据源并走真实的探查接口;draft 给了就先起草这些表的目录。"""
made: list[str] = []
async def _make(path: str, *, options: dict | None = None, draft: list[str] | None = None) -> str:
async with SessionLocal() as session:
row = DataSource(name=f"prof_{uuid.uuid4().hex[:10]}", kind="sqlite", database=str(path),
options=options or {})
session.add(row)
await session.commit()
sid = row.id
made.append(sid)
r = await client.post(f"/api/datasources/{sid}/introspect")
assert r.status_code == 200, r.text
if draft is not None:
r = await client.post(f"/api/datasources/{sid}/catalog/draft", json={"tables": draft})
assert r.status_code == 200, r.text
return sid
yield _make
for sid in made:
await drop_source(sid)
class _Spy:
"""记下两样东西:守卫放行(guard.check 返回)的语句,和真正发到这个业务库的语句。
后者挂在 SQLAlchemy 的 before_cursor_execute 上,任何一条到达驱动的语句都逃不掉——包括绕开查询层
另开连接发的。断言「发到库的 ⊆ 守卫放行的」就是「所有查询都经过守卫」。
"""
def __init__(self, db_name: str) -> None:
self.db_name = db_name
self.checked: list[str] = []
self.executed: list[str] = []
def clear(self) -> None:
self.checked.clear()
self.executed.clear()
def all_guarded(self) -> bool:
return bool(self.executed) and all(s in self.checked for s in self.executed)
def mentions(self, word: str) -> list[str]:
return [s for s in self.executed if word.lower() in s.lower()]
@pytest.fixture
def spy(monkeypatch, scenic_db):
holder = SimpleNamespace(spy=None)
def watch(path: str) -> _Spy:
holder.spy = _Spy(Path(path).name)
return holder.spy
real_check = data_engine.check
def checking(sql, **kw):
out = real_check(sql, **kw)
if holder.spy is not None:
holder.spy.checked.append(out)
return out
def on_execute(conn, cursor, statement, parameters, context, executemany):
if holder.spy is not None and holder.spy.db_name in str(conn.engine.url):
holder.spy.executed.append(statement)
monkeypatch.setattr(data_engine, "check", checking)
event.listen(Engine, "before_cursor_execute", on_execute)
yield watch
event.remove(Engine, "before_cursor_execute", on_execute)
def _url(sid: str) -> str:
return f"/api/datasources/{sid}/catalog/profile"
async def _notes(client, sid: str, table: str) -> dict:
r = await client.get(f"/api/datasources/{sid}/catalog/{table}")
assert r.status_code == 200, r.text
return r.json()
def _relation(notes: dict, column: str) -> dict:
return next(r for r in notes.get("relations") or [] if r["columns"] == [column])
def _table(body: dict, name: str) -> dict:
return next(t for t in body["tables"] if t["table_name"] == name)
def _reasons(table: dict) -> set[str]:
return {s["reason"] for s in table["skipped"]}
# ---------------------------------------------------------------- 开关与设置
async def test_profile_is_off_by_default_and_says_where_to_turn_it_on(client, make_source, scenic_db, spy):
sid = await make_source(scenic_db, draft=["visits"])
watch = spy(scenic_db)
watch.clear()
r = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR)
assert r.status_code == 409
detail = r.json()["detail"]
assert "数据源设置" in detail and "开启数据剖析" in detail
assert watch.executed == [] # 一条查询都没发
async def test_settings_are_saved_and_validated_through_datasource_update(client, make_source, scenic_db):
sid = await make_source(scenic_db)
r = await client.patch(f"/api/datasources/{sid}", json={"options": _on(max_queries=0)})
assert r.status_code == 422 and "查询次数上限" in r.json()["detail"]
r = await client.patch(f"/api/datasources/{sid}", json={"options": _on(max_queries=40, sample_size=500)})
assert r.status_code == 200, r.text
assert r.json()["options"][PROFILE_OPTION] == {"enabled": True, "max_queries": 40, "sample_size": 500}
# 设好以后就能剖析;返回里带着这次实际用的设置
r = await client.post(_url(sid), json={"tables": ["channels"]}, headers=ACTOR)
assert r.status_code == 200, r.text
assert r.json()["settings"]["max_queries"] == 40 and r.json()["settings"]["sample_size"] == 500
# ---------------------------------------------------------------- 剖析的结论
async def test_profile_verifies_name_relations_and_proposes_codes_and_business_date(client, make_source, scenic_db,
spy):
sid = await make_source(scenic_db, options=_on(), draft=["visits", "channels"])
watch = spy(scenic_db)
watch.clear()
r = await client.post(_url(sid), json={"tables": ["visits", "channels"]}, headers=ACTOR)
assert r.status_code == 200, r.text
body = r.json()
# 所有发到业务库的语句都经过了守卫
assert watch.all_guarded(), watch.executed
assert body["queries_used"] == len(watch.executed) <= body["settings"]["max_queries"]
assert body["queries_used"] == sum(t["queries"] for t in body["tables"])
assert body["stopped"] is None and body["actor"] == "周宁" and body["profiled_at"]
notes = (await _notes(client, sid, "visits"))
assert notes["updated_by"] == "周宁"
notes = notes["notes"]
# 命名推断出来的两条关系:数据对得上、父表主键唯一 → 有确证,来源记为数据剖析
for column, parent in (("gate_id", "gates"), ("member_id", "members")):
rel = _relation(notes, column)
assert rel["to_table"] == parent
assert (rel["source"], rel["status"]) == ("profile", "verified")
assert rel["coverage"] == 1.0 and rel["cardinality"] == "many_to_one"
assert "数据剖析" in rel["note"] and "抽样" in rel["note"]
# 外键约束来的关系不动:外键是外键,剖析是剖析
park = _relation(notes, "park_id")
assert (park["source"], park["status"], park["coverage"]) == ("fk", "verified", None)
# 状态列:码值候选,值有了、含义留空,推断
codes = notes["columns"]["status"]["codes"]
assert codes["value"] == {"1": "", "0": ""}
assert (codes["source"], codes["status"]) == ("profile", "proposed") and "1500" in codes["note"]
# 只有一个日期类列:提议为业务日期,备注里写明取值范围
bdate = notes["business_date"]
assert bdate["value"] == {"column": "visit_time"} and (bdate["source"], bdate["status"]) == ("profile", "proposed")
assert "2026-07" in bdate["note"] and "2026-09" in bdate["note"]
channels = (await _notes(client, sid, "channels"))["notes"]
assert set(channels["columns"]["channel_type"]["codes"]["value"]) == {"线上直销", "线下", "分销"}
assert "channel_code" not in channels.get("columns", {}) # 唯一约束列不是码值
visits = _table(body, "visits")
kinds = sorted(f["kind"] for f in visits["findings"])
assert kinds == ["business_date", "codes", "relation", "relation"]
gate = next(f for f in visits["findings"] if f["kind"] == "relation" and f["columns"] == ["gate_id"])
assert gate["status"] == "verified" and gate["path"] == f"relations.{_relation(notes, 'gate_id')['id']}"
assert gate["sample"] == 8 and gate["matched"] == 8 and gate["summary"]
assert visits["row_estimate"] == {"rows": 1500, "method": "count", "at_least": None}
assert visits["added"] == 2 and visits["updated"] == 2 and visits["error"] is None
async def test_wrong_name_relation_stays_proposed_with_coverage_in_note(client, make_source, tmp_path):
"""推错的命名关系:parking_records.lot_id 按名字对上了新建的 lots 表,但停车记录里的区号大多不在里面。"""
path = scenic.build(tmp_path / "lots.db")
db = sqlite3.connect(path)
try:
db.execute("CREATE TABLE lots (id INTEGER PRIMARY KEY, name TEXT NOT NULL)")
db.executemany("INSERT INTO lots VALUES (?, ?)", [(i, f"{i} 号停车区") for i in (1, 2, 3)])
db.executemany("INSERT INTO parking_records VALUES (?, ?, ?, ?, ?, ?)",
[(i, 1 + i % 20, None, "2026-08-01 09:00:00", None, 10.0) for i in range(1, 101)])
db.commit()
finally:
db.close()
sid = await make_source(path, options=_on(), draft=["parking_records"])
before = _relation((await _notes(client, sid, "parking_records"))["notes"], "lot_id")
assert (before["to_table"], before["source"], before["status"]) == ("lots", "name", "proposed")
r = await client.post(_url(sid), json={"tables": ["parking_records"]}, headers=ACTOR)
assert r.status_code == 200, r.text
rel = _relation((await _notes(client, sid, "parking_records"))["notes"], "lot_id")
assert rel["status"] == "proposed" and rel["source"] == "profile"
assert rel["coverage"] == 0.15
assert "抽样覆盖率 15%" in rel["note"] and "可能不是这条关系" in rel["note"] and "20 个" in rel["note"]
finding = _table(r.json(), "parking_records")["findings"][0]
assert finding["status"] == "proposed" and finding["coverage"] == 0.15
# 两个日期类列:不提议业务日期,取值范围只在返回里
ranges = {d["column"] for d in _table(r.json(), "parking_records")["date_ranges"]}
assert ranges == {"entered_at"} # exited_at 全是空值
assert "business_date" not in (await _notes(client, sid, "parking_records"))["notes"]
async def test_confirmed_relation_only_gets_coverage_and_cardinality(client, make_source, scenic_db):
sid = await make_source(scenic_db, options=_on(), draft=["visits"])
detail = await _notes(client, sid, "visits")
gate = _relation(detail["notes"], "gate_id")
r = await client.post(f"/api/datasources/{sid}/catalog/visits/review",
json={"path": f"relations.{gate['id']}", "action": "confirm",
"if_version": detail["version"]})
assert r.status_code == 200, r.text
r = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR)
assert r.status_code == 200, r.text
rel = _relation((await _notes(client, sid, "visits"))["notes"], "gate_id")
assert (rel["source"], rel["status"]) == ("name", "confirmed")
assert rel["coverage"] == 1.0 and rel["cardinality"] == "many_to_one"
finding = next(f for f in _table(r.json(), "visits")["findings"] if f.get("columns") == ["gate_id"])
assert finding["status"] == "confirmed" and finding["confirmed"] is True
async def test_rerun_with_same_data_changes_nothing(client, make_source, scenic_db):
"""同一天、数据没变:第二次剖析不改目录、不升版本(冻结进快照的目录不会无故变)。"""
sid = await make_source(scenic_db, options=_on(), draft=["visits"])
first = (await client.post(_url(sid), json={"tables": ["visits"]})).json()
second = (await client.post(_url(sid), json={"tables": ["visits"]})).json()
a, b = _table(first, "visits"), _table(second, "visits")
assert (b["added"], b["updated"], b["removed"]) == (0, 0, 0)
assert b["version"] == a["version"]
# ---------------------------------------------------------------- 安全约束
async def test_masked_columns_are_never_queried(client, make_source, scenic_db, spy):
sid = await make_source(scenic_db, options={**_on(), "mask_columns": "member_id, Channel_Type"},
draft=["visits", "channels"])
watch = spy(scenic_db)
watch.clear()
r = await client.post(_url(sid), json={"tables": ["visits", "channels"]}, headers=ACTOR)
assert r.status_code == 200, r.text
assert watch.all_guarded()
assert watch.mentions("member_id") == [] and watch.mentions("channel_type") == []
visits, channels = _table(r.json(), "visits"), _table(r.json(), "channels")
masked = [s for s in visits["skipped"] + channels["skipped"] if s["reason"] == "masked"]
assert {s["target"] for s in masked} == {"member_id → members.id", "channel_type"}
assert all("遮罩" in s["detail"] for s in masked)
notes = (await _notes(client, sid, "visits"))["notes"]
assert _relation(notes, "member_id")["source"] == "name" # 没动
assert _relation(notes, "gate_id")["status"] == "verified" # 其余照常
assert "channel_type" not in ((await _notes(client, sid, "channels"))["notes"].get("columns") or {})
async def test_query_budget_stops_profiling_and_reports_the_rest(client, make_source, scenic_db, spy):
sid = await make_source(scenic_db, options=_on(max_queries=3), draft=["visits"])
watch = spy(scenic_db)
watch.clear()
r = await client.post(_url(sid), json={"tables": ["visits", "channels"]}, headers=ACTOR)
assert r.status_code == 200, r.text
body = r.json()
assert body["queries_used"] == 3 == len(watch.executed) and body["stopped"] == "budget"
visits, channels = _table(body, "visits"), _table(body, "channels")
assert "budget" in _reasons(visits)
assert channels["queries"] == 0 and _reasons(channels) == {"budget"}
assert all("查询次数" in s["detail"] for s in visits["skipped"] + channels["skipped"] if s["reason"] == "budget")
# 没查完的关系不改:原来的命名推断原样留着
rel = _relation((await _notes(client, sid, "visits"))["notes"], "member_id")
assert (rel["source"], rel["status"]) == ("name", "proposed")
async def test_single_query_timeout_is_recorded_and_profiling_continues(client, make_source, scenic_db, monkeypatch):
"""闸口的抽样查询换成一条停不下来的递归查询:1 秒时限到点由数据库掐断,记为超时,其余照常。"""
sid = await make_source(scenic_db, options=_on(query_timeout_s=1), draft=["visits"])
real = data_engine.run_query
endless = "WITH RECURSIVE r(x) AS (SELECT 1 UNION ALL SELECT x + 1 FROM r) SELECT COUNT(*) FROM r"
async def slow_gate(source, sql, *, limits=None):
if "gate_id" in sql and "DISTINCT" in sql and "COUNT" not in sql:
assert limits is not None and limits.timeout_seconds == 1
return await real(source, endless, limits=limits)
return await real(source, sql, limits=limits)
monkeypatch.setattr(data_engine, "run_query", slow_gate)
r = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR)
assert r.status_code == 200, r.text
visits = _table(r.json(), "visits")
timeout = [s for s in visits["skipped"] if s["reason"] == "timeout"]
assert [s["target"] for s in timeout] == ["gate_id → gates.id"] and "超时" in timeout[0]["detail"]
assert r.json()["stopped"] is None
notes = (await _notes(client, sid, "visits"))["notes"]
assert _relation(notes, "gate_id")["source"] == "name" # 超时的不改
assert _relation(notes, "member_id")["status"] == "verified" # 其余照常
async def test_total_deadline_stops_profiling(make_source, scenic_db, monkeypatch):
sid = await make_source(scenic_db, draft=["visits"])
real = data_engine.run_query
async def slow(source, sql, *, limits=None):
await asyncio.sleep(0.5)
return await real(source, sql, limits=limits)
monkeypatch.setattr(data_engine, "run_query", slow)
async with SessionLocal() as session:
row = await session.get(DataSource, sid)
settings = ProfileSettings(enabled=True, max_total_s=1.4)
report = await catalog_profile.profile_catalog(session, row, row, tables=["visits"], actor=None,
settings=settings)
assert report.stopped == "deadline"
assert 1 <= report.queries_used <= 3
reasons = {s.reason for t in report.tables for s in t.skipped}
assert "deadline" in reasons
async def test_large_tables_get_no_full_scans(client, make_source, scenic_db, spy):
"""行数超过整表统计上限(这里设成 100 行):不做 COUNT(DISTINCT)、MIN/MAX,抽样和取值分布都带上限。"""
sid = await make_source(scenic_db, options=_on(max_scan_rows=100), draft=["visits"])
watch = spy(scenic_db)
watch.clear()
r = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR)
assert r.status_code == 200, r.text
visits = _table(r.json(), "visits")
assert visits["row_estimate"] == {"rows": None, "method": "count", "at_least": 101}
on_visits = [s for s in watch.executed if "from visits" in s.lower()]
assert on_visits and all("LIMIT" in s for s in on_visits), on_visits
assert not any("MIN(" in s for s in on_visits)
assert "too_large" in _reasons(visits)
assert any("太大" in s["detail"] or "上限" in s["detail"] for s in visits["skipped"] if s["reason"] == "too_large")
# 父表有主键,覆盖率又够:照样有确证;子表一侧没核对唯一,按多对一记
rel = _relation((await _notes(client, sid, "visits"))["notes"], "gate_id")
assert rel["status"] == "verified" and rel["cardinality"] == "many_to_one"
# ---------------------------------------------------------------- 接口
async def test_concurrent_profile_on_same_source_returns_409(client, make_source, scenic_db, monkeypatch):
sid = await make_source(scenic_db, options=_on(), draft=["visits"])
real = data_engine.run_query
started, release = asyncio.Event(), asyncio.Event()
async def held(source, sql, *, limits=None):
started.set()
await release.wait()
return await real(source, sql, limits=limits)
monkeypatch.setattr(data_engine, "run_query", held)
first = asyncio.create_task(client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR))
await asyncio.wait_for(started.wait(), 10)
second = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR)
assert second.status_code == 409 and "正在进行数据剖析" in second.json()["detail"]
release.set()
done = await asyncio.wait_for(first, 30)
assert done.status_code == 200, done.text
# 做完以后可以再来
again = await client.post(_url(sid), json={"tables": ["channels"]}, headers=ACTOR)
assert again.status_code == 200, again.text
async def test_default_tables_are_used_tables_with_relation_candidates(client, make_source, scenic_db):
sid = await make_source(scenic_db, options=_on(), draft=[])
r = await client.post(_url(sid), json={}, headers=ACTOR)
assert r.status_code == 200, r.text
names = [t["table_name"] for t in r.json()["tables"]]
assert names and len(names) <= catalog_profile.PROFILE_DEFAULT_TABLES
async with SessionLocal() as session:
entries = await catalog.read_catalog(session, sid)
for name in names:
rels = entries[name].notes.get("relations") or []
assert any(rel["source"] not in ("fk",) for rel in rels), name
assert "parks" not in names and "channels" not in names # 没有待核实关系的表不选
async def test_unknown_table_and_missing_source(client, make_source, scenic_db):
sid = await make_source(scenic_db, options=_on(), draft=["visits"])
r = await client.post(_url(sid), json={"tables": ["nope"]}, headers=ACTOR)
assert r.status_code == 200
assert _table(r.json(), "nope")["error"] and r.json()["queries_used"] == 0
r = await client.post(_url("0" * 32), json={}, headers=ACTOR)
assert r.status_code == 404
+205
View File
@@ -0,0 +1,205 @@
"""数据剖析(阶段 2A)里不连库的部分:设置的解析与校验、SQL 字面量、各方言的剖析 SQL 都过得了守卫、
目录对剖析结果的容纳(待填含义的码值、撤销审阅回到剖析的结论)。"""
from __future__ import annotations
from types import SimpleNamespace
import pytest
from app.data import catalog, catalog_profile, guard
from app.data.catalog import make_item
from app.data.catalog_profile import PROFILE_OPTION, ProfileSettings, profile_settings, profile_settings_problem
from app.data.engine import build_url
AT = "2026-10-03T08:00:00+00:00"
# ---------------------------------------------------------------- 设置
def test_settings_default_off_and_parse_leniently():
s = profile_settings({})
assert s == ProfileSettings()
assert s.enabled is False and s.max_queries == 60 and s.query_timeout_s == 10 and s.sample_size == 2000
assert s.max_scan_rows == 100_000 and s.max_total_s == 120
on = profile_settings({PROFILE_OPTION: {"enabled": True, "max_queries": "30", "sample_size": 500}})
assert on.enabled is True and on.max_queries == 30 and on.sample_size == 500
# 库里存着的值不对(老数据、手改过):那一项用缺省,不因此开启,也不报错
odd = profile_settings({PROFILE_OPTION: {"enabled": "yes", "max_queries": 0, "query_timeout_s": "x"}})
assert odd.enabled is False and odd.max_queries == 60 and odd.query_timeout_s == 10
assert profile_settings({PROFILE_OPTION: "on"}) == ProfileSettings()
@pytest.mark.parametrize("value, label", [
({"max_queries": 0}, "查询次数上限"),
({"max_queries": 501}, "查询次数上限"),
({"max_queries": 2.5}, "查询次数上限"),
({"query_timeout_s": 61}, "单条查询时限"),
({"sample_size": 5}, "抽样键值数"),
({"max_scan_rows": -1}, "整表统计行数上限"),
({"max_total_s": 5}, "总时长上限"),
({"enabled": "true"}, "开启数据剖析"),
({"max_queries": True}, "查询次数上限"),
])
def test_settings_problem_names_the_field_in_chinese(value, label):
problem = profile_settings_problem({PROFILE_OPTION: value})
assert problem and label in problem and "数据剖析" in problem
def test_settings_problem_rejects_unknown_keys_and_wrong_shape():
assert "无法识别" in profile_settings_problem({PROFILE_OPTION: {"enabled": True, "rows": 1}})
assert profile_settings_problem({PROFILE_OPTION: ["enabled"]})
assert profile_settings_problem({}) is None
assert profile_settings_problem({PROFILE_OPTION: None}) is None
assert profile_settings_problem({PROFILE_OPTION: {"enabled": True, "max_queries": 500, "query_timeout_s": 1.5,
"sample_size": 10, "max_scan_rows": 0,
"max_total_s": 600}}) is None
def test_profile_option_never_reaches_the_connection_url():
"""剖析设置是 AgentLab 自己的配置:拼进连接串,驱动会把它当未知参数拒掉。"""
source = SimpleNamespace(kind="mysql", host="db.example", port=3306, database="shop", username="ro",
password=None, options={"charset": "utf8mb4", PROFILE_OPTION: {"enabled": True}})
url = build_url(source)
assert "charset=utf8mb4" in url and PROFILE_OPTION not in url and "enabled" not in url
# ---------------------------------------------------------------- 字面量
@pytest.mark.parametrize("value, kind, expected", [
(42, "number", "42"),
(-7, None, "-7"),
(3.0, "number", "3"),
("12345678901234567890", "number", "12345678901234567890"), # DECIMAL 经查询层转成了字符串
("A-01", "text", "'A-01'"),
("O'Neil", "text", "'O''Neil'"),
("12:30", "text", "'12:30'"),
])
def test_sql_literal_quotes_safe_values(value, kind, expected):
assert catalog_profile.sql_literal(value, kind) == expected
@pytest.mark.parametrize("value, kind", [
(True, "boolean"), # 布尔不当键
(2.5, "number"), # 非整数的数不当键:浮点相等比较靠不住
("12.50", "number"),
("1e3", "number"),
("2026-07-01", "date"), # 日期的字面量各家写法不同,不猜
("a\\b", "text"), # 反斜杠:MySQL 不同设置下读法不同,守卫会拒
("a :b", "text"), # 「:名字」会被查询层当成绑定参数
(":b", "text"),
("a\nb", "text"), # 控制字符
("x" * 201, "text"),
(None, "text"),
([1], None),
])
def test_sql_literal_refuses_unsafe_or_unsupported_values(value, kind):
assert catalog_profile.sql_literal(value, kind) is None
# ---------------------------------------------------------------- 各方言的剖析 SQL
_META = {"visits": {"schema": None, "columns": [{"name": "memberTagId", "type": "INTEGER"}], "primary_key": ["id"]},
"orders": {"schema": "sales", "columns": [{"name": "status", "type": "INTEGER"}], "primary_key": ["id"]}}
@pytest.mark.parametrize("kind", ["mysql", "mariadb", "postgresql", "oracle", "sqlite"])
def test_profile_sql_passes_readonly_guard_on_every_dialect(kind):
"""剖析的每一种查询(含读统计信息的系统视图)都得过只读守卫——守卫不为剖析放宽。"""
d = catalog_profile.SqlDialect(kind)
sqls = []
for table in ("visits", "orders"):
stats = d.stats_sql(_META[table], table)
if kind == "sqlite":
assert stats is None # SQLite 没有可读的统计信息,改为数到上限
else:
assert stats
sqls.append(stats)
tbl = d.table(_META[table], table)
col = d.quote(_META[table]["columns"][0]["name"])
sqls += [
d.bounded_count_sql(tbl, 100_001),
d.distinct_sample_sql(tbl, col, 2000, scan_cap=None),
d.distinct_sample_sql(tbl, col, 2000, scan_cap=100_000),
d.match_count_sql(tbl, col, ["1", "'A'"]),
d.unique_check_sql(tbl, col),
d.value_counts_sql(tbl, col, 21, scan_cap=None),
d.value_counts_sql(tbl, col, 21, scan_cap=100_000),
d.min_max_sql(tbl, [col]),
]
for sql in sqls:
assert guard.check(sql, readonly=True, dialect=kind) == sql, sql
def test_dialect_quoting_and_row_limits():
my, ora, pg = (catalog_profile.SqlDialect(k) for k in ("mysql", "oracle", "postgresql"))
assert my.quote("memberTagId") == "`memberTagId`" and pg.quote("memberTagId") == '"memberTagId"'
assert my.table(_META["orders"], "orders") == "sales.orders"
assert pg.table({"schema": "Sales"}, "memberTags") == '"Sales"."memberTags"'
# Oracle 没有 LIMIT(11g 也没有 FETCH FIRST):一律用 ROWNUM
sample = ora.distinct_sample_sql("visits", '"memberTagId"', 50, scan_cap=None)
assert "LIMIT" not in sample and "ROWNUM <= 50" in sample
assert "LIMIT 50" in pg.distinct_sample_sql("visits", "x", 50, scan_cap=None)
# 统计信息按方言读各自的系统视图
assert "information_schema.TABLES" in my.stats_sql(_META["visits"], "visits")
assert "pg_class" in pg.stats_sql(_META["visits"], "visits")
stats = ora.stats_sql(_META["orders"], "orders")
assert "ALL_TABLES" in stats and "'ORDERS'" in stats and "'SALES'" in stats
# ---------------------------------------------------------------- 目录对剖析结果的容纳
def test_codes_with_pending_meanings_are_valid_and_render_as_pending():
"""剖析得到的码值只有取值、含义待人填写:目录要收得下,给模型看时写明含义待填写。"""
notes = {"columns": {"status": {"codes": make_item({"1": "", "0": ""}, "profile", "proposed")},
"channel": {"codes": make_item({"WEB": "官网", "BOX": ""}, "profile", "proposed")}}}
assert catalog.validate_notes(notes) == []
text = catalog.render_table_notes(notes)
assert "1、0(含义待填写)" in text
assert "WEB=官网,BOX 的含义待填写" in text
# 码值仍得是对照表:值不是文字的照样拒收
assert catalog.validate_notes({"columns": {"s": {"codes": make_item({"1": 2}, "profile")}}})
def _profile_rel(coverage, cardinality, status):
return {"id": catalog.relation_id("visits", ["gate_id"], "gates", ["id"]), "columns": ["gate_id"],
"to_table": "gates", "to_columns": ["id"], "cardinality": cardinality, "coverage": coverage,
"source": "profile", "status": status}
def test_reset_returns_profile_relation_to_what_profiling_concluded():
"""撤销审阅回到「来源的初始状态」。剖析的结论随数据而定:覆盖率够、父键唯一(基数有值)就是有确证。"""
verified = {"relations": [_profile_rel(1.0, "many_to_one", "verified")]}
rid = verified["relations"][0]["id"]
out = catalog.review_item(verified, f"relations.{rid}", "confirm", at=AT)
assert out["relations"][0]["status"] == "confirmed"
out = catalog.review_item(out, f"relations.{rid}", "reset", at=AT)
assert out["relations"][0]["status"] == "verified"
out = catalog.review_item(catalog.review_item(out, f"relations.{rid}", "reject", at=AT),
f"relations.{rid}", "reset", at=AT)
assert out["relations"][0]["status"] == "verified"
# 覆盖率低、父键没核实唯一的:撤销后仍是推断
for coverage, cardinality in ((0.62, "many_to_one"), (1.0, None)):
low = {"relations": [_profile_rel(coverage, cardinality, "proposed")]}
out = catalog.review_item(catalog.review_item(low, f"relations.{rid}", "confirm", at=AT),
f"relations.{rid}", "reset", at=AT)
assert out["relations"][0]["status"] == "proposed"
# 剖析提的码值候选、业务日期永远是推断
codes = {"columns": {"status": {"codes": make_item({"1": ""}, "profile", "proposed")}}}
out = catalog.review_item(catalog.review_item(codes, "columns.status.codes", "confirm", at=AT),
"columns.status.codes", "reset", at=AT)
assert out["columns"]["status"]["codes"]["status"] == "proposed"
def test_human_edit_may_restore_profile_conclusion_but_not_invent_one():
rel = _profile_rel(1.0, "many_to_one", "confirmed")
existing = {"relations": [rel]}
back = catalog.apply_human_edit(existing, {"relations": [{**rel, "status": "verified"}]}, table="visits", at=AT)
assert back["relations"][0]["status"] == "verified" and back["relations"][0]["source"] == "profile"
low = _profile_rel(0.4, "many_to_one", "proposed")
kept = catalog.apply_human_edit({"relations": [low]}, {"relations": [{**low, "status": "verified"}]},
table="visits", at=AT)
assert kept["relations"][0]["status"] == "proposed"
+2 -2
View File
@@ -24,6 +24,7 @@ from app.db.models import DataSource, Run, RunEvent, Workflow, WorkflowVersion
from app.engine.runner import run_manager
from app.main import app
from tests.fixtures.catalog import scenic_notes
from tests.fixtures.sources import drop_source
_CACHE: dict[str, dict] = {}
SQL = "SELECT COUNT(*) AS n FROM visits v JOIN parks p ON p.id = v.park_id WHERE v.status = 1"
@@ -72,9 +73,8 @@ async def source(scenic_db):
for wid in made:
if (wf := await session.get(Workflow, wid)) is not None:
await session.delete(wf)
if (row := await session.get(DataSource, sid)) is not None:
await session.delete(row)
await session.commit()
await drop_source(sid)
def node(nid, ntype, **config):
+2
View File
@@ -7,6 +7,7 @@ from app.data import catalog
from app.data.catalog import make_item
from app.db.base import SessionLocal
from app.db.models import Artifact, DataSource, Run
from tests.fixtures.sources import drop_source
MARK = "(推断,未确认)"
@@ -184,4 +185,5 @@ async def test_table_usage_counts_run_queries_per_table(monkeypatch):
await session.commit()
source = await session.get(DataSource, sid)
usage = await catalog.table_usage(session, source)
await drop_source(sid)
assert usage == {"visits": 4, "明细": 2, "parks": 1}
+2 -2
View File
@@ -14,12 +14,12 @@ from types import SimpleNamespace
from app.core.artifact_store import canonical_json, content_hash, load
from app.data import catalog
from app.data.catalog import make_item
from app.data.engine import engines
from app.data.introspect import describe_table, introspect
from app.db.base import SessionLocal
from app.db.models import Artifact, DataSource
from app.tools.datasource import _store_schema, build_datasource_tools
from app.tools.registry import ToolContext
from tests.fixtures.sources import drop_source
MARK = "(推断,未确认)"
@@ -182,4 +182,4 @@ async def test_tools_show_and_freeze_catalog_and_record_query_tables(scenic_db):
ref = await session.get(Artifact, (result["artifact"], run_id))
assert ref.meta == {"source_id": row.id, "source": row.name, "tables": ["visits", "parks"]}
finally:
await engines.invalidate(row.id)
await drop_source(row.id)
+2 -4
View File
@@ -20,6 +20,7 @@ from app.data.engine import engines
from app.data.introspect import introspect
from app.data.sqlcheck import SqlChecker
from tests.fixtures.catalog import scenic_notes
from tests.fixtures.sources import drop_source
FANOUT_SQL = ("SELECT SUM(o.total_amount) AS gmv FROM orders o JOIN order_items i ON i.order_id = o.id "
"WHERE o.status = 1")
@@ -177,10 +178,7 @@ async def scenic_source(scenic_db):
await catalog.write_entry(session, row.id, table, notes, if_version=0, actor="王敏")
source_id = row.id
yield name
async with SessionLocal() as session:
row = await session.get(DataSource, source_id)
row.enabled = False
await session.commit()
await drop_source(source_id)
async def _stream(monkeypatch, model, source):
+2 -4
View File
@@ -23,6 +23,7 @@ from app.engine.governance import lint_for_publish, publish_issues
from app.engine.schema import GraphSpec
from app.main import app
from tests.fixtures.catalog import scenic_notes
from tests.fixtures.sources import drop_source
FANOUT_SQL = ("SELECT SUM(o.total_amount) AS gmv FROM orders o JOIN order_items i ON i.order_id = o.id "
"WHERE o.status = 1")
@@ -133,10 +134,7 @@ async def scenic_source(scenic_db):
await catalog.write_entry(session, row.id, table, notes, if_version=0, actor="王敏")
source_id = row.id
yield name
async with SessionLocal() as session:
row = await session.get(DataSource, source_id)
row.enabled = False
await session.commit()
await drop_source(source_id)
@pytest.fixture
+13 -2
View File
@@ -32,6 +32,7 @@ from app.main import app
from app.tools.datasource import build_datasource_tools
from app.tools.registry import ToolContext
from tests.fixtures.catalog import scenic_notes
from tests.fixtures.sources import drop_source
FANOUT_SQL = ("SELECT SUM(o.total_amount) AS gmv FROM orders o JOIN order_items i ON i.order_id = o.id "
"WHERE o.status = 1")
@@ -54,6 +55,16 @@ async def _schema(path: str) -> dict:
return _CACHE[path]
_CREATED: list[str] = []
@pytest.fixture(autouse=True)
async def _drop_created_sources():
yield
while _CREATED:
await drop_source(_CREATED.pop())
async def _source(path: str, *, with_catalog: bool = True) -> tuple[str, str]:
name = f"scenic_{uuid.uuid4().hex[:8]}"
async with SessionLocal() as session:
@@ -61,6 +72,7 @@ async def _source(path: str, *, with_catalog: bool = True) -> tuple[str, str]:
row.schema_cache = await _schema(path)
session.add(row)
await session.commit()
_CREATED.append(row.id)
if with_catalog:
for table, notes in scenic_notes.notes().items():
await catalog.write_entry(session, row.id, table, notes, if_version=0, actor="王敏")
@@ -69,9 +81,8 @@ async def _source(path: str, *, with_catalog: bool = True) -> tuple[str, str]:
@pytest.fixture
async def scenic_source(scenic_db):
source_id, name = await _source(scenic_db)
_, name = await _source(scenic_db)
yield name
await engines.invalidate(source_id)
async def _query_tool(name: str):
+10
View File
@@ -254,6 +254,16 @@ async def test_join_unconfirmed_negative(cache, sql):
assert "join_unconfirmed" not in codes(cache, sql)
async def test_sibling_join_through_a_shared_parent_is_only_a_note(cache):
"""两张表各自按 member_id 指向会员(都是外键),彼此按 member_id 相连:目录里没有这两张表之间的关系,
但这是经共同父表推得出来的连法,不是「连错了列」。只提示一句:关联后行数可能成倍增加。"""
sql = "SELECT COUNT(*) FROM orders o JOIN member_tag_links t ON t.member_id = o.member_id WHERE o.status = 1"
[check] = [c for c in run(cache, sql) if c.code == "join_unconfirmed"]
assert check.level == "info"
assert "会员" in check.message or "members" in check.message
assert "成倍" in check.message
async def test_join_level_follows_status(cache):
sql = "SELECT SUM(i.amount) FROM orders o JOIN order_items i ON i.order_id = o.id WHERE o.status = 1"
assert "join_unconfirmed" not in codes(cache, sql, "confirmed")
+2 -1
View File
@@ -34,6 +34,7 @@ from app.engine.expressions import TruncatedUse, eval_expression
from app.engine.issuance import decide_tier, incomplete_gaps
from app.engine.runner import run_manager
from app.main import app
from tests.fixtures.sources import drop_source
# --------------------------------------------------------------------------
# 求值器:整组用到截断结果时记一笔,单格不记
@@ -345,7 +346,7 @@ async def park(ticket_db, engine_up):
# 连接池按数据源 id 缓存:换了库文件不作废,查的还是上一个测试的库
await engines.invalidate(source_id)
yield
await engines.invalidate(source_id)
await drop_source(source_id)
def node(nid, ntype, **config):
+9 -1
View File
@@ -1,6 +1,7 @@
import type {
ActivateSnapshotBody, ActivateSnapshotOut, AiPreview, CatalogDetail, CatalogDraftOut, CatalogImpact, CatalogList, CatalogNotes,
CatalogPatchPreview, CatalogPatchSubmit, CatalogReviewAction, CommitOut, CurrentRecipe, EditPreview, EditRequest, ImportRecord,
CatalogPatchPreview, CatalogPatchSubmit, CatalogProfileOut, CatalogReviewAction, CommitOut, CurrentRecipe, EditPreview,
EditRequest, ImportRecord,
ManifestOut, PurgeRawBody, PurgeRawOut, QuestionAnswer, Recipe, RedraftRulesOut, RemovePeriodBody, RemovePeriodOut,
RevokeAcceptanceBody, RevokeAcceptanceOut, SnapshotOut, Staging,
} from '../types'
@@ -687,6 +688,13 @@ export const api = {
draft: (sourceId: string, body: { tables?: string[]; use_model: boolean }, opts?: RequestOptions) =>
request<CatalogDraftOut>(`/datasources/${encodeURIComponent(sourceId)}/catalog/draft`,
{ method: 'POST', body: JSON.stringify(body), ...opts }),
/**
* 同步剖析:对业务库发少量只读查询,核对关系、取码值候选,结论写进目录。不给 tables 时按使用次数取前 10 张
* 有待核实关系的表。没开启、数据源停用、没有表结构、同一个数据源正在剖析、快照被改动时回 409(detail 是一句中文)
*/
profile: (sourceId: string, body: { tables?: string[] }, opts?: RequestOptions) =>
request<CatalogProfileOut>(`/datasources/${encodeURIComponent(sourceId)}/catalog/profile`,
{ method: 'POST', body: JSON.stringify(body), ...opts }),
},
runs: {
+19
View File
@@ -77,9 +77,28 @@ export function revealField(nodeId: string, field?: FieldRef | string | null, op
const to = target.querySelector<HTMLElement>(FILLABLE) ?? target.querySelector<HTMLElement>(PRESSABLE)
to?.focus({ preventScroll: true })
}
// 子键没有自己的一行、字段是一段 JSON(调用工具的「参数」,问题落在 args.sql 上):在文字里找到那个键,
// 框内滚到那一行,聚焦时光标放在它的值开头
if (!part && at.sub) revealJsonKey(target, at.sub, !!opts.focus)
}))
}
/** JSON 文字框里的某个键:滚到它那一行;focus 时把光标放在值的开头(不选中,免得一敲键就把整段 SQL 换掉) */
function revealJsonKey(field: HTMLElement, key: string, focus: boolean): void {
const box = field.querySelector<HTMLTextAreaElement>('textarea')
if (!box) return
const m = new RegExp(`"${key.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}"\\s*:\\s*"?`).exec(box.value)
if (!m) return
const line = box.value.slice(0, m.index).split('\n').length - 1
const height = parseFloat(getComputedStyle(box).lineHeight) || 16
box.scrollTop = Math.max(0, (line - 1) * height)
if (focus) {
const at = m.index + m[0].length
box.focus({ preventScroll: true })
box.setSelectionRange(at, at)
}
}
// 选中 → 面板开不开。放在模块级:选中可能来自别处(问题面板定位、发布弹窗、快捷键),
// 规则只写这一份
useStudio.subscribe((s, prev) => {
+6 -2
View File
@@ -13,6 +13,7 @@ import { useCatalog } from '../store/catalog'
import { Modal, Spinner, toast, useRadioGroup } from '../components/ui'
import { humanizeError } from '../lib/errors'
import { PUBLISH_FIX_TEXT as T, WORKFLOW_STATUS_HINT, WORKFLOW_STATUS_LABEL } from '../lib/terms'
import { isSqlCheckCode, sqlRuleLabel } from '../lib/sqlcheck'
import { setLocalActor, useLocalActor } from '../lib/actor'
import {
autoFixIds, choiceReady, contentSig, fixFieldLabel, fixFor, fixValueLines, fixValueText, isMissingEndpoint, mergeIssues,
@@ -42,7 +43,8 @@ export function PublishDialog({ workflow, onClose, onDone, onLocate }: {
workflow: Workflow
onClose: () => void
onDone: () => void
onLocate: (nodeId: string) => void
/** 定位一条问题:选中节点,能落到具体配置项(调用工具的 SQL)就落到那一项 */
onLocate: (issue: ValidationIssue) => void
}) {
const load = useStudio((s) => s.load)
const dirty = useStudio((s) => s.dirty)
@@ -208,7 +210,7 @@ export function PublishDialog({ workflow, onClose, onDone, onLocate }: {
</dl>
<div className="mt-3" data-preflight="dialog">
<Preflight pf={pf} gate={gate} onLocate={(issue) => { if (issue.node_id) { onLocate(issue.node_id); onClose() } }} />
<Preflight pf={pf} gate={gate} onLocate={(issue) => { if (issue.node_id) { onLocate(issue); onClose() } }} />
</div>
</Modal>
)
@@ -564,6 +566,8 @@ function PreflightRow({ issue, others, fix, where, pf, onLocate }: {
aria-label={err ? '错误' : '提示'} />
<span className="min-w-0 flex-1 [overflow-wrap:anywhere]">
{where && !String(issue.message).includes(`「${where}」`) && <b className="font-semibold">「{where}」</b>}
{/* 对照数据目录的 SQL 检查:先写中文规则名,不露规则编号 */}
{isSqlCheckCode(issue.code) && <span className="font-medium" data-preflight-rule="">{sqlRuleLabel(issue.code)}:</span>}
{issue.message}
</span>
{issue.node_id && <CornerDownRight size={10} className="mt-px shrink-0 text-faint" aria-hidden />}
+1
View File
@@ -310,6 +310,7 @@ function ToastCard({ item }: { item: ToastItem }) {
<div
className="fade-up pointer-events-auto flex items-start gap-2 rounded-lg border bg-panel px-3 py-2 shadow-elev-2"
style={{ borderColor: item.kind === 'error' ? 'var(--err)' : item.kind === 'warn' ? 'var(--warn)' : 'var(--border)' }}
data-toast={item.kind}
onMouseEnter={() => setPaused(true)}
onMouseLeave={() => setPaused(false)}
onFocus={() => setPaused(true)}
+65
View File
@@ -0,0 +1,65 @@
import type { SqlCheckCode, SqlCheckItem, SqlCheckLevel } from '../types'
import { SQL_CHECK_LEVEL_LABEL, SQL_CHECK_RULE_LABEL, SQL_CHECK_TEXT } from './terms'
// ===========================================================================
// 基于数据目录的 SQL 检查(后端 data/sqlcheck.py)在界面上的读法:规则编号 → 中文规则名、级别 → 语义色,
// 以及从证据接口、助手自查、发布前检查的问题里读出一条检查。证据面板、运行时间线、助手面板、发布前检查共用。
// ===========================================================================
/** 七条规则的编号(服务端 CHECK_CODES),按严重程度大致排序 */
export const SQL_CHECK_CODES: readonly SqlCheckCode[] = [
'fanout_sum', 'stock_summed', 'join_unconfirmed', 'ratio_aggregated', 'missing_valid_filter', 'unknown_code', 'wrong_date_column',
]
export const isSqlCheckCode = (code: unknown): code is SqlCheckCode =>
typeof code === 'string' && (SQL_CHECK_CODES as readonly string[]).includes(code)
/** 级别:认不出的按「提示」(最轻的一档),不把拿不准的说重 */
export const sqlCheckLevelOf = (v: unknown): SqlCheckLevel => (v === 'error' || v === 'warning' ? v : 'info')
/** 规则的中文名;服务端新加、这里还不认识的规则统称「SQL 检查」,不露编号 */
export const sqlRuleLabel = (code: unknown): string => (isSqlCheckCode(code) ? SQL_CHECK_RULE_LABEL[code] : SQL_CHECK_TEXT.unknownRule)
/** 各级别的颜色(语义令牌) */
export const SQL_CHECK_TONE: Record<SqlCheckLevel, string> = {
error: 'var(--st-failed)',
warning: 'var(--st-waiting)',
info: 'var(--accent)',
}
const LEVEL_RANK: Record<SqlCheckLevel, number> = { error: 0, warning: 1, info: 2 }
/** 错误在前,同级保持原来的顺序 */
export function sortSqlChecks<T extends { level?: unknown }>(list: T[]): T[] {
return list.map((c, i) => ({ c, i }))
.sort((a, b) => LEVEL_RANK[sqlCheckLevelOf(a.c.level)] - LEVEL_RANK[sqlCheckLevelOf(b.c.level)] || a.i - b.i)
.map((x) => x.c)
}
/** 涉及的表和列:「orders.amount」「orders」 */
export const sqlCheckWhere = (c: Pick<SqlCheckItem, 'table' | 'column'>): string =>
(c.table ? (c.column ? `${c.table}.${c.column}` : c.table) : (c.column ?? ''))
/** 从任意对象读一条检查(证据接口的 checks、助手自查和发布前检查的问题);不是 SQL 检查的返回 null */
export function sqlCheckOf(v: unknown): SqlCheckItem | null {
if (!v || typeof v !== 'object') return null
const o = v as Record<string, unknown>
if (!isSqlCheckCode(o.code)) return null
const str = (x: unknown) => (typeof x === 'string' && x ? x : undefined)
return {
code: o.code,
level: sqlCheckLevelOf(o.level),
message: String(o.message ?? ''),
table: str(o.table) ?? '',
...(str(o.column) ? { column: str(o.column) } : {}),
...(str(o.relation_id) ? { relation_id: str(o.relation_id) } : {}),
...(str(o.sql_excerpt) ? { sql_excerpt: str(o.sql_excerpt) } : {}),
}
}
/** 一条检查写成一行(时间线的展开区、交给助手再修的清单):「错误 · 一对多关联后重复计算:……(涉及 orders.amount)」 */
export function sqlCheckLine(c: SqlCheckItem): string {
const where = sqlCheckWhere(c)
return `${SQL_CHECK_LEVEL_LABEL[sqlCheckLevelOf(c.level)]} · ${sqlRuleLabel(c.code)}:${c.message}`
+ (where ? `(${SQL_CHECK_TEXT.where} ${where})` : '')
}
+287 -2
View File
@@ -43,7 +43,8 @@
*/
import type {
CatalogCardinality, CatalogMeasure, CatalogSource, CatalogStatus, CatalogTableKind, NodeType, SnapshotReasonCode, ToolTrust,
CatalogCardinality, CatalogMeasure, CatalogProfileNumberKey, CatalogSource, CatalogStatus, CatalogTableKind, NodeType, SnapshotReasonCode,
SqlCheckCode, SqlCheckLevel, ToolTrust,
} from '../types'
import { formatNumber } from './format'
@@ -247,6 +248,9 @@ export const EVIDENCE_TEXT = {
/** 指标拿截断的查询结果整组算出来(计数、求和……):值只算到了取回的那部分,出具按缺口降档 */
incomplete: '结果不完整',
incompleteFallback: '基于被截断的查询结果计算,结果不完整',
/** 指标所依据的查询对照数据目录查出了错误级的问题:值照算,出具按缺口降档 */
sqlCheckFailed: 'SQL 检查未通过',
sqlCheckFallback: '所依据的查询未通过 SQL 检查,结果不可靠',
/** 输入是 Agent 交来、被截断切开的数组字段 */
inputTruncated: '查询结果已截断,该字段只含取回的部分行',
unshowable: (value: string) => `有值(${value}),但无法按口径卡的格式显示`,
@@ -2127,7 +2131,7 @@ export const CATALOG_TEXT = {
keysPlaceholder: '列名,用顿号或逗号分隔',
keysUnknown: (cols: string) => `表结构中没有这些列:${cols}`,
codesPlaceholder: '1=已支付',
codesHint: '每行一个,写成「码值=含义」',
codesHint: '每行一个,写成「码值=含义」,含义可暂不填写',
codesInvalid: (line: number) => `第 ${formatNumber(line)} 行应写成「码值=含义」`,
codesDuplicate: (code: string) => `码值 ${code} 重复`,
dateColumnRequired: '填写规则或时区时,日期列必填',
@@ -2241,3 +2245,284 @@ export const CATALOG_DRIFT_TEXT = {
unchanged: '发布之后这些表的目录没有变化',
empty: '这一版的 SQL 没有用到数据库表',
}
// ===========================================================================
// 数据剖析(数据源设置里的开关和预算;数据目录页的「数据剖析」)
//
// 术语:
// - 「数据剖析」:对业务库发少量只读查询,核实推断的关联关系(覆盖率、基数),取状态类列的码值候选,提议业务日期。
// 动作写「剖析」,不写扫描、探测。默认关闭,按数据源开启。
// - 预算五项:查询次数上限、单条查询时限、抽样键值数、整表统计行数上限、总时长上限。标签和服务端校验报错里的叫法
// 一致(catalog_profile._NUMBER_FIELDS),保存被拒时按它认出是哪一项。
// ===========================================================================
export const PROFILE_FIELD_LABEL: Record<'enabled' | CatalogProfileNumberKey, string> = {
enabled: '开启数据剖析',
max_queries: '查询次数上限',
query_timeout_s: '单条查询时限',
sample_size: '抽样键值数',
max_scan_rows: '整表统计行数上限',
max_total_s: '总时长上限',
}
export const PROFILE_FIELD_UNIT: Record<CatalogProfileNumberKey, string> = {
max_queries: '条',
query_timeout_s: '秒',
sample_size: '个',
max_scan_rows: '行',
max_total_s: '秒',
}
export const PROFILE_FIELD_HINT: Record<CatalogProfileNumberKey, string> = {
max_queries: '一次剖析最多发出的查询条数,用完即停止',
query_timeout_s: '超过即中断这条查询;不超过数据源自身的查询时限',
sample_size: '核对一条关联关系时,从本表抽取的不同键值个数',
max_scan_rows: '不超过此行数的表才做去重计数、最小值和最大值等整表统计',
max_total_s: '一次剖析的总时长,到时即停止',
}
export const PROFILE_TEXT = {
// ---- 设置
section: '数据剖析',
enableHint: '默认关闭。开启后可在数据目录中核实推断的关联关系、取状态类列的码值候选',
risk: (queries: number, timeout: number, total: number) =>
`开启后,每次剖析会对这个数据源发出只读查询:最多 ${formatNumber(queries)} 条,单条不超过 ${formatNumber(timeout)} 秒,`
+ `总时长不超过 ${formatNumber(total)} 秒。不读取明细行,遮罩的列不取样。`,
range: (min: number, max: number, unit: string, def: number) =>
`范围 ${formatNumber(min)}–${formatNumber(max)} ${unit},默认 ${formatNumber(def)}`,
scanZero: '填 0 表示一律不做整表统计',
placeholder: (def: number) => `默认 ${formatNumber(def)}`,
rangeError: (min: number, max: number, integer: boolean) =>
`请填写 ${formatNumber(min)} 到 ${formatNumber(max)} 之间的${integer ? '整数' : '数'}`,
invalid: (n: number) => `数据剖析有 ${formatNumber(n)} 项设置不正确`,
// ---- 数据源卡片
chip: '数据剖析已开启',
chipHint: (queries: number, timeout: number) =>
`数据剖析已开启:每次最多 ${formatNumber(queries)} 条只读查询,单条不超过 ${formatNumber(timeout)} 秒`,
open: '设置数据剖析',
dialogTitle: (name: string) => `「${name}」的数据剖析设置`,
dialogBody: '设置保存在数据源上,对数据目录中的每次剖析生效。',
savedOn: (name: string) => `「${name}」已开启数据剖析`,
savedOff: (name: string) => `「${name}」已关闭数据剖析`,
saved: (name: string) => `已保存「${name}」的数据剖析设置`,
// ---- 数据目录页:范围与确认
action: '数据剖析',
actionHint: '对业务库发少量只读查询:核实推断的关联关系、取码值候选、提议业务日期',
profileSelected: (n: number) => `剖析所选(${formatNumber(n)})`,
title: '数据剖析',
intro: '对业务库发少量只读查询,核实推断的关联关系(覆盖率、基数),取状态类列的码值候选,提议业务日期。'
+ '结论写入数据目录;已确认、已驳回的项不会被改动。',
scope: '剖析范围',
scopeSelected: (n: number) => `已选的 ${formatNumber(n)} 张表`,
scopeFiltered: (n: number) => `当前筛选结果中的 ${formatNumber(n)} 张表`,
scopeDefault: '使用次数最多、有待核实关联关系的表',
scopeDefaultHint: (n: number) => `由服务端挑选,至多 ${formatNumber(n)} 张`,
scopeTooMany: (max: number) => `超过 ${formatNumber(max)} 张,请缩小范围`,
budget: '本次预算',
budgetQueries: (queries: number, timeout: number) =>
`最多 ${formatNumber(queries)} 条只读查询,单条不超过 ${formatNumber(timeout)} 秒`,
budgetTotal: (total: number) => `总时长不超过 ${formatNumber(total)} 秒,到时即停止`,
budgetSample: (n: number) => `每条关联关系抽样 ${formatNumber(n)} 个键值`,
budgetScan: (n: number) => (n > 0 ? `行数超过 ${formatNumber(n)} 的表不做整表统计` : '不做整表统计'),
budgetFrom: (name: string) => `按「${name}」的数据剖析设置`,
editSettings: '修改设置',
noCancel: '剖析在服务端一次完成,开始后无法中途停止。',
start: '开始剖析',
again: '重新剖析',
disabledTitle: '这个数据源未开启数据剖析',
disabledBody: '剖析会对业务库发出只读查询,需要先在数据源设置中开启。',
enable: '开启数据剖析',
// ---- 进行中
running: '正在剖析…',
elapsed: (used: string, total: string) => `已用时 ${used}(总时长上限 ${total})`,
runningHint: '剖析在服务端一次完成,无法中途停止。关闭此窗口不影响剖析,完成后会提示结果。',
background: '在后台继续',
headerRunning: (used: string) => `正在剖析 ${used}`,
headerRunningHint: '数据剖析进行中,点击查看',
leavePage: '离开本页不会中止剖析,结论仍会写入数据目录,但不再显示本次的报告。',
// ---- 报告
done: '剖析完成',
stoppedTitle: (reason: string) => `剖析已停止:${reason}`,
summary: (used: number, max: number, tables: number) =>
`用了 ${formatNumber(used)} / ${formatNumber(max)} 条查询,剖析 ${formatNumber(tables)} 张表`,
changes: (added: number, updated: number, removed: number) =>
`新增 ${formatNumber(added)} 项、更新 ${formatNumber(updated)} 项${removed ? `、删除 ${formatNumber(removed)} 项` : ''}`,
noChange: '目录没有变化',
toastDone: (summary: string) => `数据剖析完成:${summary}`,
toastStopped: (reason: string) => `数据剖析已停止:${reason}`,
toastFailed: '数据剖析未能完成',
viewReport: '查看报告',
close: '完成',
tableQueries: (n: number) => `${formatNumber(n)} 条查询`,
rowsStats: (n: number) => `约 ${formatNumber(n)} 行`,
rowsCount: (n: number) => `${formatNumber(n)} 行`,
rowsAtLeast: (n: number) => `超过 ${formatNumber(n - 1)} 行`,
rowsHint: { stats: '数据库统计信息中的估算值', count: '计数到整表统计行数上限为止' } as Record<string, string>,
openTable: '打开',
openTableLabel: (name: string) => `打开「${name}」`,
noFindings: '没有新的发现',
findingRelations: '关联关系',
findingCodes: '码值候选',
findingDate: '业务日期',
relationVerified: '升为已验证',
relationKept: '保持推断',
relationConfirmed: '已人工确认,只补充覆盖率和基数',
relationLowCoverage: '覆盖率不足,可能不是这条关系',
relationNotUnique: '被指向列未核实唯一',
coverage: (pct: string) => `覆盖率 ${pct}`,
sampled: (sample: number, matched: number) => `抽样 ${formatNumber(sample)} 个键值,对上 ${formatNumber(matched)} 个`,
cardinalityUnknown: '基数未核实',
codeRows: (n: number) => `${formatNumber(n)} 行`,
codesSummary: (values: number, rows: number) => `${formatNumber(values)} 个取值,共 ${formatNumber(rows)} 行`,
codesPendingN: (n: number) => `${formatNumber(n)} 个含义待填写`,
fillMeanings: '填写含义',
fillMeaningsLabel: (col: string) => `填写含义:列 ${col} 的码值`,
dateProposal: (col: string) => `提议按 ${col} 作为业务日期`,
dateSpan: (min: string, max: string) => `取值从 ${min} 到 ${max}`,
dateRanges: '日期列的取值范围',
skippedTitle: (n: number) => `跳过 ${formatNumber(n)} 项`,
moreSkipped: (n: number) => `…另有 ${formatNumber(n)} 项`,
tableError: '这张表未能剖析',
// ---- 无法开始(409)
blocked: {
disabled: '未开启数据剖析',
inactive: '数据源已停用',
noSchema: '还没有表结构',
busy: '这个数据源正在剖析',
tampered: '数据文件核对未通过',
other: '无法开始剖析',
} as Record<string, string>,
blockedNext: {
disabled: '开启后即可在这里剖析。',
inactive: '启用后回到这里重试。',
noSchema: '探查完成后回到这里重试。',
busy: '同一个数据源同一时间只能进行一次剖析。',
tampered: '请重新上传表格,或联系管理员核对数据文件。',
other: '请稍后重试。',
} as Record<string, string>,
goSource: '前往数据源',
retry: '重试',
failedTitle: '剖析未能完成',
backToSetup: '返回',
}
/** 整次剖析中途停下的原因 */
export const PROFILE_STOP_LABEL: Record<string, string> = {
budget: '查询次数已用完',
deadline: '总时长已用完',
failed: '连续多条查询失败',
}
/** 停下之后怎么办 */
export const PROFILE_STOP_NEXT: Record<string, string> = {
budget: '其余检查未执行。可在数据剖析设置中调高查询次数上限,或缩小剖析范围后重新剖析。',
deadline: '其余检查未执行。可在数据剖析设置中调高总时长上限,或缩小剖析范围后重新剖析。',
failed: '其余检查未执行。请先在数据源卡片上测试连接,确认可用后重新剖析。',
}
/** 跳过一项的原因(skipped.reason)。整句说明在 skipped.detail 里 */
export const PROFILE_SKIP_REASON_LABEL: Record<string, string> = {
budget: '查询次数用完',
deadline: '总时长用完',
failed: '连续失败',
timeout: '查询超时',
error: '查询失败',
rejected: '未通过安全守卫',
masked: '列已遮罩',
too_large: '表太大',
view: '视图',
unsupported: '暂不支持',
no_data: '没有数据',
missing: '表结构中没有',
high_cardinality: '取值太多',
}
/** 跳过的是哪一类检查(skipped.kind) */
export const PROFILE_SKIP_KIND_LABEL: Record<string, string> = {
relation: '关联关系',
codes: '码值',
date: '日期列',
row_estimate: '行数估算',
table: '整张表',
}
/** 码值候选:剖析只知道出现过哪些取值,含义等人填 */
export const CODES_TEXT = {
pending: '含义待填写',
pendingN: (n: number) => `${formatNumber(n)} 个含义待填写`,
fill: '填写含义',
fillLabel: (col: string) => `填写含义:列 ${col} 的码值`,
title: (col: string) => `列 ${col} 的码值含义`,
hint: '保存后这一项记为人工填写、已确认。暂不清楚的含义可以留空,之后再填。',
head: { code: '码值', meaning: '含义' },
note: '来源说明',
placeholder: '含义待填写',
saved: (n: number) => (n ? `已填写 ${formatNumber(n)} 个码值的含义` : '已保存'),
save: '保存',
cancel: '取消',
}
// ===========================================================================
// 基于数据目录的 SQL 检查(后端 data/sqlcheck.py)
//
// 术语:
// - 「SQL 检查」:对照数据目录找出能执行、数却可能不对的写法。七条规则,界面上写中文规则名,不露规则编号。
// - 级别:错误(依据已确认,结果必然有误)、提醒(依据已确认,结果很可能有误)、提示(依据只是推断,仅供参考)。
// 颜色走语义令牌:错误 --st-failed,提醒 --st-waiting,提示 --accent。
// ===========================================================================
export const SQL_CHECK_RULE_LABEL: Record<SqlCheckCode, string> = {
fanout_sum: '一对多关联后重复计算',
stock_summed: '存量跨期加总',
join_unconfirmed: '关联条件未经确认',
ratio_aggregated: '比率直接加总或平均',
missing_valid_filter: '未筛选有效记录',
unknown_code: '码值不在码值表中',
wrong_date_column: '未按业务日期统计',
}
export const SQL_CHECK_RULE_HINT: Record<SqlCheckCode, string> = {
fanout_sum: '沿一对多方向关联后,对「一」那一侧的度量求和或计数,会按明细行数重复计算',
stock_summed: '对存量列求和却没有按业务日期分组,不同日期的存量被加在一起',
join_unconfirmed: '关联条件对不上数据目录中有确证的关系,可能连错了列',
ratio_aggregated: '对比率列直接求和或求平均,得不到正确的比率',
missing_valid_filter: '表定义了有效记录条件,查询没有按它筛选',
unknown_code: '按码值筛选时用了码值表中没有的值',
wrong_date_column: '表定义了业务日期,查询却按另一个时间列分组或筛选',
}
export const SQL_CHECK_LEVEL_LABEL: Record<SqlCheckLevel, string> = {
error: '错误',
warning: '提醒',
info: '提示',
}
export const SQL_CHECK_LEVEL_HINT: Record<SqlCheckLevel, string> = {
error: '依据已确认,结果必然有误',
warning: '依据已确认,结果很可能有误',
info: '依据尚未确认(推断),仅供参考',
}
export const SQL_CHECK_TEXT = {
title: 'SQL 检查',
/** 规则编号不认识(服务端新加了规则)时的叫法 */
unknownRule: 'SQL 检查',
listLabel: (n: number) => `SQL 检查发现 ${formatNumber(n)} 处问题`,
where: '涉及',
excerpt: 'SQL 片段',
catalogHint: '检查依据来自数据目录。依据有误时,请在数据目录中修正对应项。',
// ---- 运行时间线:指标所依据的查询没通过 SQL 检查(日志 metric_sql_check)
metricTitle: (name: string, n = 1) =>
(n > 1 ? `指标「${name}」所依据的查询有 ${formatNumber(n)} 处未通过 SQL 检查` : `指标「${name}」所依据的查询未通过 SQL 检查`),
metricTitleAnon: '指标所依据的查询未通过 SQL 检查',
metricNext: '指标照常计算,但结果不可靠,出具按缺口降档。请在证据面板的查询步骤中查看问题,修改 SQL 后重新运行;'
+ '检查依据有误时,请在数据目录中修正对应项',
// ---- 助手:搭图时查出的问题
counts: (errors: number, warnings: number, infos: number) =>
[errors ? `${formatNumber(errors)} 处错误` : '', warnings ? `${formatNumber(warnings)} 处提醒` : '',
infos ? `${formatNumber(infos)} 处提示` : ''].filter(Boolean).join('、'),
assistantTitle: (counts: string) => `SQL 检查:${counts}`,
assistantNext: '请打开对应节点,在「参数」中修改 SQL;检查依据有误时,请在数据目录中修正对应项',
assistantNextChat: '可在画布中打开此工作流,修改对应节点「参数」中的 SQL;检查依据有误时,请在数据目录中修正对应项',
assistantInfoNext: '这些检查的依据尚未确认,仅供参考。在数据目录中确认相关项后,检查结论更可靠',
}
+65 -13
View File
@@ -2,7 +2,7 @@ import { useEffect, useId, useMemo, useRef, useState } from 'react'
import { useNavigate } from 'react-router-dom'
import {
AlertTriangle, BookMarked, ChevronRight, Database, EyeOff, FileSpreadsheet, History, Info, KeyRound, Layers, Lock, Plug, Plus,
RefreshCw, Search, Table2, Upload, X,
RefreshCw, ScanSearch, Search, Table2, Upload, X,
} from 'lucide-react'
import clsx from 'clsx'
import { ApiError, api, uploadDecision } from '../api/client'
@@ -21,9 +21,16 @@ import { checkHealth, forgetHealth, healthFromServer, setHealth, useHealth } fro
import type { HealthRecord } from '../lib/health'
import { workflowList, workflowsMentioning } from '../lib/mentions'
import {
CATALOG_TEXT, RAW_STATE_LABEL, RECIPE_ORIGIN_LABEL, RECIPE_TEXT, UPLOAD_CONVERSION_LABEL, UPLOAD_SHAPE_LABEL, UPLOAD_TEXT, VERSIONS_TEXT,
CATALOG_TEXT, PROFILE_TEXT, RAW_STATE_LABEL, RECIPE_ORIGIN_LABEL, RECIPE_TEXT, UPLOAD_CONVERSION_LABEL, UPLOAD_SHAPE_LABEL, UPLOAD_TEXT,
VERSIONS_TEXT,
} from '../lib/terms'
import { useRunClock } from '../run/useRunClock'
import { focusProfileField, ProfileSettingsDialog, ProfileSettingsSection } from './catalog/ProfileSettings'
import type { ProfileErrorKey } from './catalog/ProfileSettings'
import {
hasProfileOption, PROFILE_OPTION, profileFieldOfRejection, profileFormOf, profileOptionOf, profileProblems, profileSettingsOf,
} from './catalog/profile'
import type { ProfileForm } from './catalog/profile'
import { ImportWizard } from './import/ImportWizard'
import type { WizardEntry } from './import/ImportWizard'
import { VersionsDialog } from './import/VersionsDialog'
@@ -285,9 +292,11 @@ function SourceCard({ row, meta, onChange, onRemoved, onEdit, onReupload, onReci
// 传上来的表通常只有一两张,直接摊开;库动辄几十上百个对象,默认收着
const [open, setOpen] = useState(() => uploaded && row.table_count > 0 && row.table_count <= 3)
const [masking, setMasking] = useState(false)
const [profiling, setProfiling] = useState(false)
/** 版本页(当前版本的各期、历史版本、导入记录) */
const [versions, setVersions] = useState(false)
const masks = maskList(row.options?.[MASK_KEY])
const profile = profileSettingsOf(row.options)
const clock = useRunClock(!!busy)
const busySince = useRef(0)
useTicker(60_000)
@@ -525,12 +534,28 @@ function SourceCard({ row, meta, onChange, onRemoved, onEdit, onReupload, onReci
{masks.length ? '修改遮罩列' : '设置遮罩列'}
</button>
)}
{profile.enabled && (
<span className="chip" data-profile-on="" title={PROFILE_TEXT.chipHint(profile.max_queries, profile.query_timeout_s)}>
<ScanSearch size={10} aria-hidden /> {PROFILE_TEXT.chip}
</span>
)}
{uploaded && (
// 数据剖析的设置同理:手工登记的库在编辑框里改
<button type="button" className="text-2xs underline decoration-dotted underline-offset-2 hover:text-dim"
onClick={() => setProfiling(true)} data-edit-profile="">
{PROFILE_TEXT.open}
</button>
)}
{!!row.tools?.length && <span className="mono">{row.tools.join(' · ')}</span>}
</div>
{masking && (
<MaskColumnsDialog row={row} onClose={() => setMasking(false)}
onSaved={(next) => { setMasking(false); onChange(next); syncCatalog() }} />
)}
{profiling && (
<ProfileSettingsDialog row={row} onClose={() => setProfiling(false)}
onSaved={(next) => { setProfiling(false); onChange(next); syncCatalog() }} />
)}
{versions && (
// onChange 是父级的「更新这一行并刷新全局目录」:回滚之后表结构可能变了,检查器、问数据读的目录要跟上
<VersionsDialog source={row} onClose={() => setVersions(false)} onChange={onChange} onReload={onReload} />
@@ -940,11 +965,16 @@ interface SourceForm {
sid: string
/** options.mask_columns:证据面板展示查询原始行时遮掉的列 */
maskColumns: string[]
/** options 里除 schema / service_name / sid / mask_columns 之外的键,全部摊开可编辑 */
/** options.catalog_profile:数据剖析的开关和预算 */
profile: ProfileForm
/** 打开时 options 里已经有剖析设置:没开、也没填时也要照写(关掉之后写回 enabled: false) */
profileStored: boolean
/** options 里除 schema / service_name / sid / mask_columns / catalog_profile 之外的键,全部摊开可编辑 */
advanced: [string, string][]
}
const OWN_OPTION_KEYS = new Set(['schema', 'service_name', 'sid', MASK_KEY])
// 剖析设置是一个对象,不能摊进「高级连接参数」:那里按文字回显,保存时会写成「[object Object]」被服务端拒收
const OWN_OPTION_KEYS = new Set(['schema', 'service_name', 'sid', MASK_KEY, PROFILE_OPTION])
function toForm(source: any): SourceForm {
const o = source.options ?? {}
@@ -966,6 +996,8 @@ function toForm(source: any): SourceForm {
serviceName: o.service_name ?? (oracle ? source.database ?? '' : ''),
sid: o.sid ?? '',
maskColumns: maskList(o[MASK_KEY]),
profile: profileFormOf(o),
profileStored: hasProfileOption(o),
advanced: Object.entries(o)
.filter(([k]) => !OWN_OPTION_KEYS.has(k))
.map(([k, v]) => [k, String(v ?? '')]),
@@ -973,10 +1005,12 @@ function toForm(source: any): SourceForm {
}
function toBody(form: SourceForm, isNew: boolean, wasOracle = true): any {
const options: Record<string, string | string[]> = {}
const options: Record<string, unknown> = {}
for (const [k, v] of form.advanced) if (k.trim() && v.trim()) options[k.trim()] = v.trim()
if (form.schema.trim()) options.schema = form.schema.trim()
if (form.maskColumns.length) options[MASK_KEY] = form.maskColumns
const profile = profileOptionOf(form.profile, form.profileStored)
if (profile) options[PROFILE_OPTION] = profile
const oracle = form.kind === 'oracle'
if (oracle) {
// 只写选中的那一个,另一个不带:两处都有值时引擎听 service_name,
@@ -1005,10 +1039,12 @@ function toBody(form: SourceForm, isNew: boolean, wasOracle = true): any {
}
/**
* 422 的中文 detail 说的是哪一项。眼下只有查询时限按字段拒(后端 query_timeout_problem),
* 认不出的交回 toast
* 422 的中文 detail 说的是哪一项:查询时限(后端 query_timeout_problem);数据剖析的某一项(profile_settings_problem,
* 记为 profile.<键>,它的「单条查询时限」也带「查询时限」四个字,要先认)。认不出的交回 toast
*/
function fieldOfRejection(message: string): string | null {
const profile = profileFieldOfRejection(message)
if (profile) return `profile.${profile}`
if (/查询时限/.test(message)) return 'query_timeout_s'
return null
}
@@ -1019,10 +1055,11 @@ const FIELD_LABEL: Record<string, string> = {
/**
* 决定连到哪个库的那几项(提交体里的)。schema 不在内:它只影响探查哪一片,不影响
* 连不连得上,后端判断测连接结果还作不作数时同样不看它;遮罩列只管面板上怎么显示,也不在内
* 连不连得上,后端判断测连接结果还作不作数时同样不看它;遮罩列只管面板上怎么显示,数据剖析的设置只管剖析,
* 都不在内(服务端拼连接串时同样摘掉这几项)
*/
function connectionOf(body: any): string {
const { schema: _schema, [MASK_KEY]: _mask, ...options } = body.options ?? {}
const { schema: _schema, [MASK_KEY]: _mask, [PROFILE_OPTION]: _profile, ...options } = body.options ?? {}
return JSON.stringify([
body.kind, body.host ?? null, body.port ?? null, body.database ?? null, body.username ?? null,
options, !!body.password,
@@ -1076,6 +1113,12 @@ function SourceEditor({ source, kinds, onClose, onSaved }: {
}
const required = (key: string) => needs.includes(key)
const blocked = missing.length > 0 || !!nameError
// 剖析设置填错只拦保存,不拦测试连接(和连接无关)
const profileBad = Object.keys(profileProblems(form.profile)).length
const saveBlocked = blocked || profileBad > 0
const saveHint = blocked ? `缺少必填项:${missing.join('、') || '标识格式'}` : profileBad ? PROFILE_TEXT.invalid(profileBad) : undefined
const profileError = fieldError?.key.startsWith('profile.')
? { key: fieldError.key.slice('profile.'.length) as ProfileErrorKey, message: fieldError.message } : null
const body = toBody(form, isNew, source.kind === 'oracle')
const sig = JSON.stringify({ ...body, password: form.password })
const dirty = JSON.stringify(form) !== JSON.stringify(initial)
@@ -1097,7 +1140,7 @@ function SourceEditor({ source, kinds, onClose, onSaved }: {
}
const save = async () => {
if (blocked) return
if (saveBlocked) return
setSaving(true)
setFieldError(null)
try {
@@ -1109,7 +1152,10 @@ function SourceEditor({ source, kinds, onClose, onSaved }: {
} catch (e) {
const rejected = e instanceof ApiError && e.status === 422 ? e.message : ''
const key = rejected ? fieldOfRejection(rejected) : null
if (key) {
if (key?.startsWith('profile.')) {
setFieldError({ key, message: rejected })
focusProfileField('ds-profile', key.slice('profile.'.length) as ProfileErrorKey)
} else if (key) {
setFieldError({ key, message: rejected })
setAdvOpen(true)
requestAnimationFrame(() => document.getElementById(`ds-adv-${key}`)?.focus())
@@ -1200,8 +1246,8 @@ function SourceEditor({ source, kinds, onClose, onSaved }: {
title={blocked ? `缺少必填项:${missing.join('、') || '标识格式'}` : '使用当前配置测试连接,不保存'}>
{test.since ? <Spinner size={11} /> : <Plug size={12} aria-hidden />} 测试连接
</button>
<button className="btn btn-primary" disabled={saving || blocked} onClick={() => void save()}
title={blocked ? `缺少必填项:${missing.join('、') || '标识格式'}` : undefined}>
<button className="btn btn-primary" disabled={saving || saveBlocked} onClick={() => void save()}
title={saveHint} data-source-save="">
{saving ? <Spinner size={11} /> : null} 保存
</button>
</>
@@ -1318,6 +1364,12 @@ function SourceEditor({ source, kinds, onClose, onSaved }: {
<MaskColumnsField sourceId={source.id} synced={!!source.table_count} value={form.maskColumns}
onChange={(maskColumns) => set({ maskColumns })} />
<ProfileSettingsSection idPrefix="ds-profile" form={form.profile} serverError={profileError}
onChange={(profile) => {
if (profileError) setFieldError(null)
set({ profile })
}} />
<label className="flex cursor-pointer items-start gap-2 rounded-lg border p-2.5"
style={form.readonly ? undefined : { borderColor: 'var(--warn)', background: 'color-mix(in srgb, var(--warn) 7%, transparent)' }}>
<input type="checkbox" className="mt-0.5" checked={form.readonly}
+7 -2
View File
@@ -14,7 +14,7 @@ import { LineageLayer } from '../canvas/LineageLayer'
import { VersionsSheet } from '../canvas/VersionsSheet'
import { PublishDialog } from '../canvas/PublishDialog'
import { WorkflowPicker, confirmDiscard, createWorkflow, takeDiscarded } from '../canvas/WorkflowPicker'
import { problemsOf, type Problem } from '../canvas/issues'
import { fieldOfIssue, problemsOf, type Problem } from '../canvas/issues'
import { STUDIO_SHORTCUTS, hintOf, studioShortcut, type StudioShortcutId } from '../canvas/shortcuts'
import { AssistantPanel } from '../run/AssistantPanel'
import { copilotProgress } from '../run/Composer'
@@ -620,7 +620,12 @@ export function StudioPage() {
{publishing && workflow && (
<PublishDialog workflow={workflow} onClose={() => setPublishing(false)}
onDone={() => { setPublishing(false); void refresh() }}
onLocate={(id) => { select(id); focusNode(id) }} />
onLocate={(issue) => {
// 和问题面板同一个落点:选中节点、取景,问题落在某一项配置上(调用工具的 SQL)就翻到那一项
if (!issue.node_id) return
const node = useStudio.getState().nodes.find((n) => n.id === issue.node_id)
revealField(issue.node_id, fieldOfIssue(issue, node))
}} />
)}
</div>
)
+135 -5
View File
@@ -1,15 +1,19 @@
import { useCallback, useEffect, useMemo, useRef, useState } from 'react'
import type { ReactNode } from 'react'
import { useNavigate, useParams } from 'react-router-dom'
import { ArrowLeft, BookMarked, Database, ListFilter as ListFilterIcon, Sparkles } from 'lucide-react'
import { ArrowLeft, BookMarked, Database, ListFilter as ListFilterIcon, ScanSearch, Sparkles } from 'lucide-react'
import clsx from 'clsx'
import { ApiError, api } from '../../api/client'
import type { CatalogDetail, CatalogList, CatalogTableRow } from '../../types'
import { EmptyState, ErrorState, PageHeader, Skeleton, toast } from '../../components/ui'
import { useDatasources, useOnReconnect } from '../../store/catalog'
import { CATALOG_TEXT as CT } from '../../lib/terms'
import { EmptyState, ErrorState, PageHeader, Skeleton, Spinner, toast, useTicker } from '../../components/ui'
import { useCatalog, useDatasources, useOnReconnect } from '../../store/catalog'
import { CATALOG_TEXT as CT, PROFILE_STOP_LABEL, PROFILE_TEXT as PT } from '../../lib/terms'
import { StatusLegend, SystemNotesNotice } from './parts'
import { DraftDialog } from './DraftDialog'
import { ProfileDialog } from './ProfileDialog'
import type { ProfileJob } from './ProfileDialog'
import { ProfileSettingsDialog } from './ProfileSettings'
import { profileBlockOf, profileSettingsOf, secondsText } from './profile'
import { TableDetail } from './TableDetail'
import { TableIndex } from './TableIndex'
import { filterCounts, progressOf, rowFromDetail, usageSummary, visibleRows } from './model'
@@ -65,6 +69,24 @@ export function CatalogPage() {
const [drafting, setDrafting] = useState(false)
/** 起草写进了这些表:详情据此重新载入(编辑中的不动,保存时服务端会用版本号拦下) */
const [drafted, setDrafted] = useState<{ seq: number; tables: Set<string> }>({ seq: 0, tables: new Set() })
const [profiling, setProfiling] = useState(false)
const [profileSettingsOpen, setProfileSettingsOpen] = useState(false)
/** 一次剖析的状态。关掉弹窗剖析照常进行:状态留在页面上,页头有入口,完成后提示并可再打开报告 */
const [job, setJob] = useState<(ProfileJob & { sourceId: string }) | null>(null)
/** 剖析设置刚存过:数据源列表重取回来之前按存下的这份算(预算、开没开) */
const [savedOptions, setSavedOptions] = useState<{ sourceId: string; options: Record<string, unknown> } | null>(null)
/** 从剖析报告点「填写含义」:打开那张表并弹出那一列的码值 */
const [fillCodes, setFillCodes] = useState<{ table: string; column: string; seq: number } | null>(null)
const profilingRef = useRef(profiling)
profilingRef.current = profiling
const alive = useRef(true)
// 开发模式的严格模式会先卸载再挂载一次:挂载时要重新置为真
useEffect(() => {
alive.current = true
return () => { alive.current = false }
}, [])
const sourceRef = useRef(sourceId)
sourceRef.current = sourceId
const rows = useMemo(() => data?.tables ?? [], [data])
const visible = useMemo(() => visibleRows(rows, query, filter, sort), [rows, query, filter, sort])
@@ -87,6 +109,77 @@ export function CatalogPage() {
void load()
}, [load])
const options = savedOptions?.sourceId === sourceId ? savedOptions.options : source?.options
const profileSettings = useMemo(() => profileSettingsOf(options), [options])
const myJob = job?.sourceId === sourceId ? job : null
const jobRunning = myJob?.phase === 'running'
useTicker(1000, jobRunning)
/**
* 剖析:同步请求,做完才返回。期间关掉弹窗不影响;回来时页面已经换了数据源或卸载了,只提示、不再改这一页。
* 写进去的表重新载入清单和正在看的详情(编辑中的不动,同起草)
*/
const startProfile = useCallback(async (tables: string[] | null) => {
const src = sourceId
const startedAt = Date.now()
setJob({ sourceId: src, phase: 'running', startedAt, tables, settings: profileSettings })
let next: ProfileJob
try {
const report = await api.dataCatalog.profile(src, tables ? { tables } : {})
next = { phase: 'done', report, ms: Date.now() - startedAt }
} catch (e) {
next = e instanceof ApiError && e.status === 409
? { phase: 'blocked', kind: profileBlockOf(e.message), message: e.message, tables }
: { phase: 'failed', error: e, tables }
}
const here = alive.current && sourceRef.current === src
if (here) setJob({ ...next, sourceId: src })
if (next.phase === 'done') {
const r = next.report
const touched = r.tables.filter((t) => !t.error && t.added + t.updated + t.removed > 0).map((t) => t.table_name)
if (here) {
if (touched.length) setDrafted((cur) => ({ seq: cur.seq + 1, tables: new Set(touched) }))
void load()
}
// 弹窗开着就看报告;关了(在后台继续)才提示,并给「查看报告」
if (!here || !profilingRef.current) {
const summary = PT.summary(r.queries_used, r.settings.max_queries, r.tables.length)
const text = r.stopped ? PT.toastStopped(PROFILE_STOP_LABEL[r.stopped] ?? r.stopped) : PT.toastDone(summary)
const show = r.stopped ? toast.warn : toast.ok
show(text, here ? { key: `profile:${src}`, duration: 10000, action: { label: PT.viewReport, onClick: () => setProfiling(true) } }
: { key: `profile:${src}` })
}
} else if (!here || !profilingRef.current) {
toast.error(next.phase === 'blocked' ? next.message : next.error, here
? { key: `profile:${src}`, action: { label: PT.viewReport, onClick: () => setProfiling(true) } } : { key: `profile:${src}` })
}
}, [sourceId, profileSettings, load])
const openProfile = useCallback(() => {
// 上一次的结果(报告、被拒)看过了,再点就是重新开始;正在剖析时打开的是进度
setJob((cur) => (cur && cur.phase === 'running' && cur.sourceId === sourceId ? cur : null))
setProfiling(true)
}, [sourceId])
const onProfileSettingsSaved = useCallback((row: any) => {
setProfileSettingsOpen(false)
setSavedOptions({ sourceId, options: row?.options ?? {} })
void useCatalog.getState().reload('datasources')
// 因为没开启被拒的:开了就回到选范围,可以直接开始
setJob((cur) => (cur && cur.phase === 'blocked' && cur.kind === 'disabled' ? null : cur))
}, [sourceId])
const openFromReport = useCallback((t: string) => {
setProfiling(false)
openTable(t)
}, [openTable])
const fillFromReport = useCallback((t: string, column: string) => {
setProfiling(false)
setFillCodes((cur) => ({ table: t, column, seq: (cur?.seq ?? 0) + 1 }))
openTable(t)
}, [openTable])
// 上一张 / 下一张按清单当前的筛选和排序走;正在看的表不在筛选结果里(刚确认完、从关联关系跳过来)时按全部表走
const [prev, next] = useMemo(() => {
if (!table) return [null, null]
@@ -116,6 +209,19 @@ export function CatalogPage() {
<Sparkles size={11} aria-hidden /> {CT.draft}
</button>
)
// 页头的「数据剖析」:剖析进行中换成计时,点开看进度(关掉弹窗剖析照常进行)
const headerProfile = jobRunning && myJob?.phase === 'running'
? (
<button type="button" className="btn btn-sm" onClick={() => setProfiling(true)} title={PT.headerRunningHint}
aria-label={PT.headerRunning(secondsText(Date.now() - myJob.startedAt))} data-catalog-profile="running">
<Spinner size={11} /> <span className="tnum hidden sm:inline">{PT.headerRunning(secondsText(Date.now() - myJob.startedAt))}</span>
</button>
)
: (
<button type="button" className="btn btn-sm" onClick={openProfile} aria-label={PT.action} title={PT.actionHint} data-catalog-profile="">
<ScanSearch size={11} aria-hidden /> <span className="hidden sm:inline">{PT.action}</span>
</button>
)
// 页头的「起草」窄屏只留图标,标题才放得下
const headerDraft = (
<button type="button" className="btn btn-sm" onClick={() => setDrafting(true)} aria-label={CT.draft} title={CT.draft} data-catalog-draft="">
@@ -171,12 +277,13 @@ export function CatalogPage() {
active={table}
onOpen={openTable}
onDraft={() => setDrafting(true)}
onProfile={openProfile}
/>
</aside>
<section className={clsx('min-h-0 min-w-0 flex-1 flex-col lg:flex', table ? 'flex' : 'hidden')} aria-label={table ?? CT.overviewTitle}>
{table
? (
<TableDetail sourceId={sourceId} table={table} rows={rows} drafted={drafted} prev={prev} next={next}
<TableDetail sourceId={sourceId} table={table} rows={rows} drafted={drafted} fillCodes={fillCodes} prev={prev} next={next}
onOpen={openTable} onBack={() => navigate(base)} onDetail={onDetail} />
)
: <Overview rows={rows} empty={empty} systemNotes={data.system_notes} draft={draftButton(true)} onOpen={openTable} />}
@@ -193,6 +300,7 @@ export function CatalogPage() {
subtitle={CT.subtitle}
actions={(
<div className="flex shrink-0 items-center gap-1.5">
{hasTables && headerProfile}
{hasTables && headerDraft}
<button type="button" className="btn btn-sm btn-ghost" onClick={() => navigate(back)} aria-label={CT.back} data-catalog-back="">
<ArrowLeft size={12} aria-hidden /> <span className="hidden sm:inline">{CT.back}</span>
@@ -213,6 +321,28 @@ export function CatalogPage() {
onDrafted={onDrafted}
/>
)}
{profiling && data && (
<ProfileDialog
sourceName={source?.name ?? sourceId}
settings={profileSettings}
rows={rows}
visible={visible}
selected={[...selected]}
filtered={filtered}
job={myJob}
onStart={(tables) => void startProfile(tables)}
onClose={() => setProfiling(false)}
onSettings={() => setProfileSettingsOpen(true)}
onGoSource={() => navigate(back)}
onOpenTable={openFromReport}
onFillCodes={fillFromReport}
onReset={() => setJob(null)}
/>
)}
{profileSettingsOpen && source && (
<ProfileSettingsDialog row={{ id: source.id, name: source.name, options }} onClose={() => setProfileSettingsOpen(false)}
onSaved={onProfileSettingsSaved} />
)}
</div>
)
}
@@ -0,0 +1,79 @@
import { useState } from 'react'
import type { CatalogItem } from '../../types'
import { isComposing, Modal, Spinner } from '../../components/ui'
import { CODES_TEXT as KT } from '../../lib/terms'
import { StatusChip } from './parts'
// ===========================================================================
// 填写码值的含义。数据剖析只知道列里出现过哪些取值(码值候选的含义是空串),含义要人填:逐个码值一个输入框,
// 比在编辑模式里改「码值=含义」的多行文字省事。保存走整份提交,这一项记为人工填写、已确认;
// 暂不清楚的含义可以留空,表格里照旧显示「含义待填写」。
// ===========================================================================
export function CodesDialog({ column, item, saving, onSave, onClose }: {
column: string
item: CatalogItem<Record<string, string>>
saving: boolean
/** 新的码值对照(码值不变,只改含义) */
onSave: (value: Record<string, string>) => void
onClose: () => void
}) {
const [initial] = useState(() => ({ ...item.value }))
const [value, setValue] = useState<Record<string, string>>(initial)
const codes = Object.keys(initial)
const dirty = codes.some((k) => (value[k] ?? '').trim() !== (initial[k] ?? '').trim())
const save = () => {
if (!dirty || saving) return
onSave(Object.fromEntries(codes.map((k) => [k, (value[k] ?? '').trim()])))
}
return (
<Modal open onClose={onClose} dirty={dirty} width={520} title={KT.title(column)}
footer={(
<>
<button type="button" className="btn" onClick={onClose}>{KT.cancel}</button>
<button type="button" className="btn btn-primary" disabled={!dirty || saving} onClick={save} data-codes-save="">
{saving ? <Spinner size={11} /> : null} {KT.save}
</button>
</>
)}>
<div className="space-y-3" data-codes-dialog={column}>
<div className="flex flex-wrap items-center gap-1.5 text-2xs text-faint">
<StatusChip status={item.status} source={item.source} />
</div>
{item.note && (
<div className="rounded border bg-bg px-2 py-1.5 text-2xs leading-relaxed text-dim">
<span className="text-faint">{KT.note}:</span>{item.note}
</div>
)}
<div className="overflow-hidden rounded-lg border">
<table className="w-full border-collapse text-xs">
<thead>
<tr className="border-b bg-elev text-left text-2xs text-faint">
<th scope="col" className="w-[32%] px-3 py-1.5 font-medium">{KT.head.code}</th>
<th scope="col" className="px-2 py-1.5 font-medium">{KT.head.meaning}</th>
</tr>
</thead>
<tbody>
{codes.map((k, i) => (
<tr key={k} className="border-b border-hairline last:border-b-0">
<th scope="row" className="mono px-3 py-1 text-left font-normal [overflow-wrap:anywhere]">
<label htmlFor={`codes-${column}-${i}`}>{k}</label>
</th>
<td className="px-2 py-1">
<input id={`codes-${column}-${i}`} className="field !py-1" value={value[k] ?? ''} placeholder={KT.placeholder}
maxLength={500} autoFocus={i === codes.findIndex((c) => !(initial[c] ?? '').trim())}
onChange={(e) => setValue((v) => ({ ...v, [k]: e.target.value }))}
// 回车保存;输入法组字时的回车是选词,不算
onKeyDown={(e) => { if (e.key === 'Enter' && !isComposing(e)) { e.preventDefault(); save() } }}
data-code={k} />
</td>
</tr>
))}
</tbody>
</table>
</div>
<p className="text-2xs leading-relaxed text-faint">{KT.hint}</p>
</div>
</Modal>
)
}
+39 -16
View File
@@ -1,13 +1,13 @@
import { memo } from 'react'
import type { ReactNode } from 'react'
import { AlertTriangle, CheckCheck, Lock } from 'lucide-react'
import { AlertTriangle, CheckCheck, Lock, PencilLine } from 'lucide-react'
import clsx from 'clsx'
import type {
CatalogColumnNotes, CatalogItem, CatalogMeasure, CatalogReviewAction, CatalogStructureColumn,
} from '../../types'
import { IconButton } from '../../components/ui'
import {
CATALOG_COLUMN_FIELD_LABEL, CATALOG_MEASURE_HINT, CATALOG_MEASURE_LABEL, CATALOG_TEXT as CT,
CATALOG_COLUMN_FIELD_LABEL, CATALOG_MEASURE_HINT, CATALOG_MEASURE_LABEL, CATALOG_TEXT as CT, CODES_TEXT as KT,
} from '../../lib/terms'
import { ItemMark } from './parts'
import type { ReviewTarget } from './parts'
@@ -32,7 +32,7 @@ export interface ColumnModel {
type ColumnTexts = Partial<Record<ColumnField, string>>
export function ColumnTable({ columns, form, initial, problems, busy, systemNotes, onReview, onConfirmColumn, onChange }: {
export function ColumnTable({ columns, form, initial, problems, busy, systemNotes, onReview, onConfirmColumn, onFillCodes, onChange }: {
columns: ColumnModel[]
/** 编辑中:列名 → 字段 → 文字(form 是当前的,initial 是进入编辑时的);不在编辑时为 null */
form: Record<string, ColumnTexts> | null
@@ -44,6 +44,8 @@ export function ColumnTable({ columns, form, initial, problems, busy, systemNote
systemNotes: boolean
onReview: (target: ReviewTarget, action: CatalogReviewAction) => void
onConfirmColumn: (col: string) => void
/** 码值里有含义空着的:打开逐个填写含义的弹窗 */
onFillCodes: (col: string) => void
onChange: (col: string, field: ColumnField, value: string) => void
}) {
return (
@@ -66,7 +68,7 @@ export function ColumnTable({ columns, form, initial, problems, busy, systemNote
<tbody>
{columns.map((c) => (
<ColumnRow key={c.name} col={c} editing={!!form} draft={form?.[c.name]} initial={initial?.[c.name]} problems={problems} busy={busy}
systemNotes={systemNotes} onReview={onReview} onConfirmColumn={onConfirmColumn} onChange={onChange} />
systemNotes={systemNotes} onReview={onReview} onConfirmColumn={onConfirmColumn} onFillCodes={onFillCodes} onChange={onChange} />
))}
</tbody>
</table>
@@ -74,7 +76,7 @@ export function ColumnTable({ columns, form, initial, problems, busy, systemNote
)
}
const ColumnRow = memo(function ColumnRow({ col, editing, draft, initial, problems, busy, systemNotes, onReview, onConfirmColumn, onChange }: {
const ColumnRow = memo(function ColumnRow({ col, editing, draft, initial, problems, busy, systemNotes, onReview, onConfirmColumn, onFillCodes, onChange }: {
col: ColumnModel
editing: boolean
/** 这一列的表单。只有改了这一列时引用才会变,别的行不重画 */
@@ -85,6 +87,7 @@ const ColumnRow = memo(function ColumnRow({ col, editing, draft, initial, proble
systemNotes: boolean
onReview: (target: ReviewTarget, action: CatalogReviewAction) => void
onConfirmColumn: (col: string) => void
onFillCodes: (col: string) => void
onChange: (col: string, field: ColumnField, value: string) => void
}) {
const { name, structure, items } = col
@@ -166,7 +169,10 @@ const ColumnRow = memo(function ColumnRow({ col, editing, draft, initial, proble
)}
{cell(
'codes',
<Value item={codes} render={(v) => <Codes value={v as Record<string, string>} />} />,
<Value item={codes} render={(v) => (
<Codes value={v as Record<string, string>} label={KT.fillLabel(name)}
onFill={codes?.status !== 'rejected' && !busy ? () => onFillCodes(name) : undefined} />
)} />,
<textarea className="field mono !min-h-0 !px-1.5 !py-1 !text-2xs" rows={Math.min(4, Math.max(1, (draft?.codes ?? '').split('\n').length))}
value={draft?.codes ?? ''}
placeholder={codes?.status === 'rejected' ? CT.rejectedPlaceholder(codesInline(codes.value)) : CT.codesPlaceholder}
@@ -183,7 +189,8 @@ const ColumnRow = memo(function ColumnRow({ col, editing, draft, initial, proble
</tr>
)
}, (a, b) => a.col === b.col && a.editing === b.editing && a.draft === b.draft && a.initial === b.initial && a.busy === b.busy
&& a.systemNotes === b.systemNotes && a.onReview === b.onReview && a.onConfirmColumn === b.onConfirmColumn && a.onChange === b.onChange
&& a.systemNotes === b.systemNotes && a.onReview === b.onReview && a.onConfirmColumn === b.onConfirmColumn && a.onFillCodes === b.onFillCodes
&& a.onChange === b.onChange
// 格式问题每次都是新对象:只比这一列的几格
&& COLUMN_FIELDS.every((f) => a.problems[`c:${a.col.name}:${f}`] === b.problems[`c:${b.col.name}:${f}`]))
@@ -195,17 +202,33 @@ export function Value<T>({ item, render }: { item: CatalogItem<T> | undefined; r
return <span className={clsx(item.status === 'rejected' && 'text-faint line-through')}>{render(item.value)}</span>
}
function Codes({ value }: { value: Record<string, string> }) {
/** 含义还空着的码值个数(数据剖析给的码值候选只有取值,含义等人填) */
export const pendingCodes = (v: Record<string, string> | undefined): number =>
Object.values(v ?? {}).filter((x) => !String(x ?? '').trim()).length
function Codes({ value, label, onFill }: { value: Record<string, string>; label: string; onFill?: () => void }) {
const entries = Object.entries(value)
const pending = pendingCodes(value)
return (
<span className="flex flex-wrap gap-1" title={entries.map(([k, v]) => `${k}=${v}`).join('\n')}>
{entries.slice(0, CODES_SHOWN).map(([k, v]) => (
<span key={k} className="inline-flex max-w-full items-center gap-1 rounded border bg-bg px-1 text-2xs">
<span className="mono text-dim">{k}</span>
<span className="truncate">{v}</span>
</span>
))}
{entries.length > CODES_SHOWN && <span className="text-2xs text-faint">+{entries.length - CODES_SHOWN}</span>}
<span className="block">
<span className="flex flex-wrap gap-1" title={entries.map(([k, v]) => `${k}=${v.trim() || KT.pending}`).join('\n')}>
{entries.slice(0, CODES_SHOWN).map(([k, v]) => (
<span key={k} className="inline-flex max-w-full items-center gap-1 rounded border bg-bg px-1 text-2xs" data-code={k}>
<span className="mono text-dim">{k}</span>
{v.trim()
? <span className="truncate">{v}</span>
: <span className="truncate" style={{ color: 'var(--st-waiting)' }} data-code-pending="">{KT.pending}</span>}
</span>
))}
{entries.length > CODES_SHOWN && <span className="text-2xs text-faint">+{entries.length - CODES_SHOWN}</span>}
</span>
{pending > 0 && onFill && (
// 各码值的「含义待填写」已经写在上面,这里只给入口;几个待填写写在悬停里
<button type="button" className="mt-1 inline-flex items-center gap-1 whitespace-nowrap rounded px-1 text-2xs text-dim underline decoration-dotted underline-offset-2 hover:text-fg"
onClick={onFill} aria-label={label} title={KT.pendingN(pending)} data-codes-fill={pending}>
<PencilLine size={10} aria-hidden /> {KT.fill}
</button>
)}
</span>
)
}
@@ -0,0 +1,450 @@
import { useMemo, useState } from 'react'
import type { ReactNode } from 'react'
import { ArrowUpRight, CircleCheck, OctagonX, ScanSearch, ShieldAlert, TriangleAlert } from 'lucide-react'
import clsx from 'clsx'
import type {
CatalogProfileCodesFinding, CatalogProfileDateFinding, CatalogProfileOut, CatalogProfileRelationFinding, CatalogProfileSettings,
CatalogProfileSize, CatalogProfileSkip, CatalogProfileTable, CatalogTableRow,
} from '../../types'
import { ErrorState, Modal, Notice, Spinner, useTicker } from '../../components/ui'
import {
CATALOG_CARDINALITY_LABEL, CATALOG_TEXT as CT, PROFILE_SKIP_KIND_LABEL, PROFILE_SKIP_REASON_LABEL, PROFILE_STOP_LABEL, PROFILE_STOP_NEXT, PROFILE_TEXT as PT,
} from '../../lib/terms'
import { STATUS_TONE, StatusChip } from './parts'
import { PROFILE_DEFAULT_TABLES, PROFILE_MAX_TABLES, coverageText, secondsText } from './profile'
import type { ProfileBlock } from './profile'
// ===========================================================================
// 数据目录页的「数据剖析」。接口是同步的:一次请求做完整个剖析才返回,服务端不支持中途停止,所以这里不给「停止」,
// 进行中只写已用时长和总时长上限,并明说无法中途停止。关掉窗口剖析照常进行(状态在页面上,页头有进度入口),
// 完成后弹提示、可以再打开报告。
//
// 四段:选范围、看预算、确认开始 → 进行中 → 报告(用了几条查询、为什么停、每张表的发现和跳过的项);
// 开始不了(409)时写明是哪一种情况和下一步,未开启的就地打开剖析设置。
// ===========================================================================
/** 一次剖析的状态。放在页面上而不是弹窗里:关掉弹窗剖析照常进行,完成后还能再打开报告 */
export type ProfileJob =
| { phase: 'running'; startedAt: number; tables: string[] | null; settings: CatalogProfileSettings }
| { phase: 'done'; report: CatalogProfileOut; ms: number }
| { phase: 'blocked'; kind: ProfileBlock; message: string; tables: string[] | null }
| { phase: 'failed'; error: unknown; tables: string[] | null }
type Scope = 'selected' | 'filtered' | 'default'
/** 报告里每类最多逐条列几项,其余合成一句 */
const SKIP_SHOWN = 6
const CODES_SHOWN = 12
export function ProfileDialog({
sourceName, settings, rows, visible, selected, filtered, job, onStart, onClose, onSettings, onGoSource, onOpenTable, onFillCodes, onReset,
}: {
sourceName: string
/** 这个数据源此刻的剖析设置(数据源列表里读的) */
settings: CatalogProfileSettings
rows: CatalogTableRow[]
visible: CatalogTableRow[]
selected: string[]
filtered: boolean
/** null:还没开始(选范围) */
job: ProfileJob | null
/** tables 为 null 时由服务端挑表 */
onStart: (tables: string[] | null) => void
onClose: () => void
/** 打开这个数据源的剖析设置 */
onSettings: () => void
/** 去数据源卡片(停用、没有表结构、快照被改动时) */
onGoSource: () => void
onOpenTable: (table: string) => void
onFillCodes: (table: string, column: string) => void
/** 回到选范围 */
onReset: () => void
}) {
const live = useMemo(() => rows.filter((r) => r.in_schema).map((r) => r.table_name), [rows])
const scopes = useMemo(() => {
const liveSet = new Set(live)
const out: { key: Scope; label: string; hint?: string; tables: string[] | null }[] = []
const sel = selected.filter((t) => liveSet.has(t))
if (sel.length) out.push({ key: 'selected', label: PT.scopeSelected(sel.length), tables: sel })
const vis = visible.filter((r) => r.in_schema).map((r) => r.table_name)
if (filtered && vis.length && vis.length !== live.length) out.push({ key: 'filtered', label: PT.scopeFiltered(vis.length), tables: vis })
out.push({ key: 'default', label: PT.scopeDefault, hint: PT.scopeDefaultHint(PROFILE_DEFAULT_TABLES), tables: null })
return out
}, [live, selected, visible, filtered])
const [scope, setScope] = useState<Scope>(() => scopes[0].key)
const picked = scopes.find((s) => s.key === scope) ?? scopes[0]
const tooMany = (picked.tables?.length ?? 0) > PROFILE_MAX_TABLES
const running = job?.phase === 'running'
useTicker(1000, running)
let body: ReactNode
let footer: ReactNode
if (!job) {
body = (
<>
<p className="text-xs leading-relaxed text-dim">{PT.intro}</p>
{!settings.enabled && (
<Notice tone="warn" attr={{ 'data-profile-disabled': '' }}>
<p className="font-medium">{PT.disabledTitle}</p>
<p className="mt-0.5 text-dim">{PT.disabledBody}</p>
<button type="button" className="btn btn-sm mt-2" onClick={onSettings} data-profile-enable="">{PT.enable}</button>
</Notice>
)}
<fieldset>
<legend className="label">{PT.scope}</legend>
<div className="space-y-1.5" role="radiogroup" aria-label={PT.scope}>
{scopes.map((s) => {
const over = (s.tables?.length ?? 0) > PROFILE_MAX_TABLES
return (
<label key={s.key} className={clsx('flex cursor-pointer items-start gap-2 rounded-md border px-3 py-2 text-xs',
scope === s.key ? 'border-[var(--accent)] bg-accent-soft' : 'hover:bg-hover')} data-profile-scope={s.key}>
<input type="radio" className="mt-0.5" name="catalog-profile-scope" checked={scope === s.key} onChange={() => setScope(s.key)} />
<span className="min-w-0">
<span>{s.label}</span>
{(s.hint || over) && (
<span className={clsx('mt-0.5 block text-2xs', over ? 'text-[var(--err)]' : 'text-faint')}>
{over ? PT.scopeTooMany(PROFILE_MAX_TABLES) : s.hint}
</span>
)}
</span>
</label>
)
})}
</div>
</fieldset>
{settings.enabled && <Budget settings={settings} sourceName={sourceName} onSettings={onSettings} />}
<p className="flex items-start gap-1.5 text-2xs leading-relaxed text-faint">
<ShieldAlert size={11} className="mt-px shrink-0" aria-hidden /> {PT.noCancel}
</p>
</>
)
footer = (
<>
<button type="button" className="btn" onClick={onClose}>{CT.cancel}</button>
<button type="button" className="btn btn-primary" disabled={!settings.enabled || tooMany} onClick={() => onStart(picked.tables)}
data-profile-start="">
<ScanSearch size={12} aria-hidden /> {PT.start}
</button>
</>
)
} else if (job.phase === 'running') {
const used = Date.now() - job.startedAt
const max = job.settings.max_total_s * 1000
body = (
<div className="space-y-3" aria-live="polite">
<div className="flex items-center gap-2 text-xs">
<Spinner size={13} />
<span className="font-medium" data-profile-title="">{PT.running}</span>
<span className="flex-1" />
<span className="tnum text-2xs text-faint" data-profile-elapsed={Math.floor(used / 1000)}>
{PT.elapsed(secondsText(used), secondsText(max))}
</span>
</div>
{/* 不是进度:服务端不报进度,这条只量已用时长占总时长上限的多少,剖析最晚在上限处停下 */}
<div className="h-1 overflow-hidden rounded-full bg-hover" role="meter" aria-label={PT.elapsed(secondsText(used), secondsText(max))}
aria-valuemin={0} aria-valuemax={job.settings.max_total_s} aria-valuenow={Math.min(job.settings.max_total_s, Math.floor(used / 1000))}>
<div className="h-full rounded-full bg-[var(--border-strong)] transition-[width] duration-1000 ease-linear"
style={{ width: `${Math.min(100, (used / max) * 100)}%` }} />
</div>
<Notice tone="info" attr={{ 'data-profile-no-cancel': '' }}>{PT.runningHint}</Notice>
<p className="text-2xs leading-relaxed text-faint">{PT.leavePage}</p>
</div>
)
footer = <button type="button" className="btn" onClick={onClose} data-autofocus data-profile-background="">{PT.background}</button>
} else if (job.phase === 'done') {
body = <Report report={job.report} ms={job.ms} onSettings={onSettings} onOpenTable={onOpenTable} onFillCodes={onFillCodes} />
footer = (
<>
<button type="button" className="btn btn-ghost mr-auto" onClick={onReset} data-profile-again="">{PT.again}</button>
<button type="button" className="btn btn-primary" onClick={onClose} data-autofocus data-profile-close="">{PT.close}</button>
</>
)
} else if (job.phase === 'blocked') {
const action = job.kind === 'disabled'
? <button type="button" className="btn btn-sm btn-primary" onClick={onSettings} data-profile-enable="">{PT.enable}</button>
: job.kind === 'busy' || job.kind === 'other'
? <button type="button" className="btn btn-sm" onClick={() => onStart(job.tables)} data-profile-retry="">{PT.retry}</button>
: <button type="button" className="btn btn-sm" onClick={onGoSource} data-profile-go-source="">{PT.goSource}</button>
body = (
<Notice tone={job.kind === 'busy' ? 'info' : 'warn'} attr={{ 'data-profile-blocked': job.kind }}>
<p className="font-medium">{PT.blocked[job.kind]}</p>
<p className="mt-0.5 text-dim">{job.message}</p>
{job.kind !== 'other' && <p className="mt-0.5 text-dim">{PT.blockedNext[job.kind]}</p>}
<div className="mt-2">{action}</div>
</Notice>
)
footer = (
<>
<button type="button" className="btn btn-ghost mr-auto" onClick={onReset} data-profile-back="">{PT.backToSetup}</button>
<button type="button" className="btn" onClick={onClose}>{PT.close}</button>
</>
)
} else {
body = (
<div data-profile-failed="">
<p className="mb-2 text-xs font-medium">{PT.failedTitle}</p>
<ErrorState error={job.error} onRetry={() => onStart(job.tables)} />
</div>
)
footer = (
<>
<button type="button" className="btn btn-ghost mr-auto" onClick={onReset}>{PT.backToSetup}</button>
<button type="button" className="btn" onClick={onClose}>{PT.close}</button>
</>
)
}
return (
<Modal open onClose={onClose} title={PT.title} width={640} footer={footer}>
<div className="space-y-4" data-profile-dialog={job?.phase ?? 'setup'}>{body}</div>
</Modal>
)
}
/** 确认前写明预算:按这个数据源的剖析设置,最多发几条查询、单条和总时长、抽样规模、整表统计的门槛 */
function Budget({ settings: s, sourceName, onSettings }: { settings: CatalogProfileSettings; sourceName: string; onSettings: () => void }) {
const lines = [PT.budgetQueries(s.max_queries, s.query_timeout_s), PT.budgetTotal(s.max_total_s), PT.budgetSample(s.sample_size),
PT.budgetScan(s.max_scan_rows)]
return (
<div className="rounded-lg border px-3 py-2.5" data-profile-budget={`${s.max_queries},${s.query_timeout_s},${s.max_total_s}`}>
<div className="mb-1.5 flex items-center gap-2 text-2xs">
<span className="font-medium text-dim">{PT.budget}</span>
<span className="text-faint">{PT.budgetFrom(sourceName)}</span>
<span className="flex-1" />
<button type="button" className="rounded px-1 text-2xs text-dim underline decoration-dotted underline-offset-2 hover:text-fg"
onClick={onSettings} data-profile-settings-open="">{PT.editSettings}</button>
</div>
<ul className="space-y-0.5 text-xs">
{lines.map((l) => (
<li key={l} className="flex items-start gap-1.5">
<span className="mt-[7px] h-1 w-1 shrink-0 rounded-full bg-[var(--text-faint)]" aria-hidden />
<span className="tnum">{l}</span>
</li>
))}
</ul>
</div>
)
}
// ---------------------------------------------------------------------------
// 报告
// ---------------------------------------------------------------------------
function Report({ report: r, ms, onSettings, onOpenTable, onFillCodes }: {
report: CatalogProfileOut
ms: number
onSettings: () => void
onOpenTable: (table: string) => void
onFillCodes: (table: string, column: string) => void
}) {
const changed = r.total.added + r.total.updated + r.total.removed
const stop = r.stopped
// 一项发现、一项跳过都没有的表合成一行,几十张表时报告不被空卡片撑长
const quiet = r.tables.filter((t) => !t.error && !t.findings.length && !t.skipped.length && !t.date_ranges.length)
const shown = r.tables.filter((t) => !quiet.includes(t))
return (
<div className="space-y-3" data-profile-report={stop ?? 'done'}>
<div className="flex items-start gap-2">
{stop
? <TriangleAlert size={15} className="mt-px shrink-0" style={{ color: 'var(--st-waiting)' }} aria-hidden />
: <CircleCheck size={15} className="mt-px shrink-0" style={{ color: 'var(--st-done)' }} aria-hidden />}
<div className="min-w-0 flex-1">
<p className="text-sm font-medium" data-profile-title="">{stop ? PT.stoppedTitle(PROFILE_STOP_LABEL[stop] ?? stop) : PT.done}</p>
<p className="tnum mt-0.5 text-xs text-dim" data-profile-summary={`${r.queries_used}/${r.settings.max_queries}`}>
{PT.summary(r.queries_used, r.settings.max_queries, r.tables.length)}
<span className="text-faint"> · {secondsText(ms)}</span>
</p>
<p className="tnum mt-0.5 text-xs" data-profile-changes={`${r.total.added},${r.total.updated},${r.total.removed}`}>
{changed ? PT.changes(r.total.added, r.total.updated, r.total.removed) : <span className="text-faint">{PT.noChange}</span>}
</p>
</div>
</div>
{stop && (
<Notice tone="warn" attr={{ 'data-profile-stopped': stop }}>
<p>{PROFILE_STOP_NEXT[stop] ?? ''}</p>
{stop !== 'failed' && (
<button type="button" className="btn btn-sm mt-2" onClick={onSettings} data-profile-settings-open="">{PT.editSettings}</button>
)}
</Notice>
)}
{r.note && <Notice tone="info" attr={{ 'data-profile-note': '' }}>{r.note}</Notice>}
{shown.map((t) => <TableReport key={t.table_name} t={t} onOpenTable={onOpenTable} onFillCodes={onFillCodes} />)}
{quiet.length > 0 && (
<p className="text-2xs leading-relaxed text-faint" data-profile-quiet={quiet.length}>
{PT.noFindings}:<span className="mono">{quiet.map((t) => t.table_name).join('、')}</span>
</p>
)}
</div>
)
}
function sizeText(size: CatalogProfileSize | null): { text: string; hint?: string } | null {
if (!size) return null
if (size.rows != null) return { text: size.method === 'stats' ? PT.rowsStats(size.rows) : PT.rowsCount(size.rows), hint: PT.rowsHint[size.method] }
if (size.at_least != null) return { text: PT.rowsAtLeast(size.at_least), hint: PT.rowsHint.count }
return null
}
function TableReport({ t, onOpenTable, onFillCodes }: {
t: CatalogProfileTable
onOpenTable: (table: string) => void
onFillCodes: (table: string, column: string) => void
}) {
const relations = t.findings.filter((f): f is CatalogProfileRelationFinding => f.kind === 'relation')
const codes = t.findings.filter((f): f is CatalogProfileCodesFinding => f.kind === 'codes')
const date = t.findings.find((f): f is CatalogProfileDateFinding => f.kind === 'business_date')
const size = sizeText(t.row_estimate)
const changed = t.added + t.updated + t.removed
return (
<section className="rounded-lg border" data-profile-table={t.table_name} aria-label={t.table_name}>
<header className="flex flex-wrap items-center gap-x-2 gap-y-0.5 border-b bg-elev px-3 py-1.5">
<h3 className="mono min-w-0 text-xs font-medium [overflow-wrap:anywhere]">{t.table_name}</h3>
<span className="tnum text-2xs text-faint">
{PT.tableQueries(t.queries)}
{size && <span title={size.hint}> · {size.text}</span>}
{changed > 0 && <span> · {PT.changes(t.added, t.updated, t.removed)}</span>}
</span>
<span className="flex-1" />
{!t.error && (
<button type="button" className="inline-flex items-center gap-0.5 rounded px-1 text-2xs text-dim hover:bg-hover hover:text-fg"
onClick={() => onOpenTable(t.table_name)} aria-label={PT.openTableLabel(t.table_name)} data-profile-open={t.table_name}>
{PT.openTable} <ArrowUpRight size={10} aria-hidden />
</button>
)}
</header>
<div className="space-y-2.5 px-3 py-2">
{t.error && (
<p className="flex items-start gap-1.5 text-xs text-[var(--err)]" data-profile-table-error="">
<OctagonX size={12} className="mt-0.5 shrink-0" aria-hidden /> <span>{t.error}</span>
</p>
)}
{relations.length > 0 && (
<Group title={PT.findingRelations}>
{relations.map((f) => <RelationLine key={f.path} f={f} />)}
</Group>
)}
{codes.length > 0 && (
<Group title={PT.findingCodes}>
{codes.map((f) => <CodesLine key={f.path} f={f} onFill={() => onFillCodes(t.table_name, f.column)} />)}
</Group>
)}
{date && (
<Group title={PT.findingDate}>
<li className="text-xs" data-profile-finding="business_date">
<StatusChip status={date.status} className="mr-1.5 align-middle" />
{PT.dateProposal(date.column)}
<span className="tnum text-dim">,{PT.dateSpan(date.min.slice(0, 19), date.max.slice(0, 19))}</span>
</li>
</Group>
)}
{!date && t.date_ranges.length > 0 && (
<Group title={PT.dateRanges}>
{t.date_ranges.map((d) => (
<li key={d.column} className="tnum text-xs text-dim" data-profile-date-range={d.column}>
<span className="mono text-fg">{d.column}</span>:{PT.dateSpan(d.min.slice(0, 19), d.max.slice(0, 19))}
</li>
))}
</Group>
)}
{!t.error && !t.findings.length && !t.date_ranges.length && (
<p className="text-2xs text-faint">{PT.noFindings}</p>
)}
{t.skipped.length > 0 && <Skipped list={t.skipped} />}
</div>
</section>
)
}
function Group({ title, children }: { title: string; children: ReactNode }) {
return (
<div>
<div className="mb-1 text-2xs font-medium text-faint">{title}</div>
<ul className="space-y-1.5">{children}</ul>
</div>
)
}
function RelationLine({ f }: { f: CatalogProfileRelationFinding }) {
const outcome = f.confirmed
? { text: PT.relationConfirmed, tone: 'var(--text-dim)' }
: f.status === 'verified'
? { text: PT.relationVerified, tone: STATUS_TONE.verified }
: { text: `${PT.relationKept}:${f.coverage < 0.95 ? PT.relationLowCoverage : PT.relationNotUnique}`, tone: STATUS_TONE.proposed }
return (
<li className="text-xs" data-profile-finding="relation" data-status={f.status} data-path={f.path}>
<div className="flex flex-wrap items-center gap-x-1.5 gap-y-0.5">
<StatusChip status={f.status} />
<span className="mono min-w-0 [overflow-wrap:anywhere]">{f.target}</span>
</div>
<div className="tnum mt-0.5 text-2xs text-dim">
<span data-profile-coverage="">{PT.coverage(coverageText(f.coverage))}</span>
{' · '}
<span data-profile-cardinality="">{f.cardinality ? CATALOG_CARDINALITY_LABEL[f.cardinality] : PT.cardinalityUnknown}</span>
{' · '}
{PT.sampled(f.sample, f.matched)}
</div>
<div className="mt-0.5 text-2xs" style={{ color: outcome.tone }} data-profile-outcome="">{outcome.text}</div>
</li>
)
}
function CodesLine({ f, onFill }: { f: CatalogProfileCodesFinding; onFill: () => void }) {
// 服务端在 summary 里写了几个含义待填写(别的来源写过的含义会沿用);认不出时按都待填写
const pending = Number(f.summary.match(/(\d+)\s*个含义待填写/)?.[1] ?? (f.summary.includes('待填写') ? f.values.length : 0))
return (
<li className="text-xs" data-profile-finding="codes" data-column={f.column}>
<div className="flex flex-wrap items-center gap-x-1.5 gap-y-0.5">
<StatusChip status={f.status} />
<span className="mono font-medium">{f.column}</span>
<span className="tnum text-2xs text-faint">{PT.codesSummary(f.values.length, f.rows)}</span>
</div>
<div className="mt-1 flex flex-wrap gap-1">
{f.values.slice(0, CODES_SHOWN).map((v) => (
<span key={v.value} className="inline-flex items-center gap-1 rounded border bg-bg px-1 text-2xs">
<span className="mono">{v.value}</span>
<span className="tnum text-faint">{PT.codeRows(v.rows)}</span>
</span>
))}
{f.values.length > CODES_SHOWN && <span className="text-2xs text-faint">+{f.values.length - CODES_SHOWN}</span>}
</div>
<div className="mt-1 flex flex-wrap items-center gap-2">
{pending > 0 && <span className="text-2xs" style={{ color: 'var(--st-waiting)' }} data-codes-pending={pending}>{PT.codesPendingN(pending)}</span>}
<button type="button" className="btn btn-xs" onClick={onFill} aria-label={PT.fillMeaningsLabel(f.column)} data-profile-fill={f.column}>
{PT.fillMeanings}
</button>
</div>
</li>
)
}
/** 跳过的项:原因(短)+ 哪一项 + 服务端给的整句。多了折起来,每类停下的原因只列前几项 */
function Skipped({ list }: { list: CatalogProfileSkip[] }) {
const [all, setAll] = useState(false)
const shown = all ? list : list.slice(0, SKIP_SHOWN)
return (
<div data-profile-skipped={list.length}>
<div className="mb-1 text-2xs font-medium text-faint">{PT.skippedTitle(list.length)}</div>
<ul className="space-y-1">
{shown.map((s, i) => (
<li key={`${s.kind}:${s.target}:${i}`} className="flex items-start gap-1.5 text-2xs" data-profile-skip={s.reason}>
<span className="chip mt-px shrink-0 !px-1.5 !py-0" style={{ color: 'var(--text-dim)' }}>
{PROFILE_SKIP_REASON_LABEL[s.reason] ?? s.reason}
</span>
<span className="min-w-0 leading-relaxed">
<span className="text-faint">{PROFILE_SKIP_KIND_LABEL[s.kind] ?? s.kind} </span>
<span className="mono text-dim [overflow-wrap:anywhere]">{s.target}</span>
<span className="block text-dim">{s.detail}</span>
</span>
</li>
))}
</ul>
{list.length > SKIP_SHOWN && !all && (
<button type="button" className="mt-1 rounded px-1 text-2xs text-dim underline decoration-dotted underline-offset-2 hover:text-fg"
onClick={() => setAll(true)}>
{PT.moreSkipped(list.length - SKIP_SHOWN)}
</button>
)}
</div>
)
}
@@ -0,0 +1,162 @@
import { useState } from 'react'
import { ShieldAlert } from 'lucide-react'
import { ApiError, api } from '../../api/client'
import type { CatalogProfileNumberKey } from '../../types'
import { Field, Modal, Spinner, toast } from '../../components/ui'
import { PROFILE_FIELD_HINT, PROFILE_FIELD_LABEL, PROFILE_FIELD_UNIT, PROFILE_TEXT as PT } from '../../lib/terms'
import {
PROFILE_DEFAULTS, PROFILE_LIMITS, PROFILE_NUMBER_KEYS, hasProfileOption, profileFieldOfRejection, profileFormOf, profileOptionOf,
profileProblems, profileSettingsOfForm, sameProfileForm, withProfileOption,
} from './profile'
import type { ProfileForm } from './profile'
// ===========================================================================
// 数据剖析的设置:开关、五项预算、风险说明。数据源编辑框里是其中一段;导入表格的源没有编辑框,数据目录页在
// 剖析被拒(未开启)时也要就地打开它,所以另有一个只改这一项的弹窗(options 的其余键原样带上)。
// 剖析会对业务库发查询,开关默认关着;开启后五项预算才展开,风险说明写的是此刻填的数。
// ===========================================================================
/** 保存被拒时的落点:某一项,或者整段(认不出是哪一项) */
export type ProfileErrorKey = 'enabled' | CatalogProfileNumberKey | 'section'
export function ProfileSettingsSection({ idPrefix, form, onChange, serverError }: {
/** 输入框 id 的前缀:报错时按 `${idPrefix}-${键}` 聚焦 */
idPrefix: string
form: ProfileForm
onChange: (next: ProfileForm) => void
/** 服务端拒收(422)的那一句,落在对应的项上 */
serverError?: { key: ProfileErrorKey; message: string } | null
}) {
const problems = profileProblems(form)
const errorOf = (k: CatalogProfileNumberKey) => (serverError?.key === k ? serverError.message : problems[k])
// 开关关着时预算收起;但填错的项(或服务端点了名的项)要看得见,不然保存被拦下却找不到原因
const open = form.enabled || Object.keys(problems).length > 0
|| (!!serverError && serverError.key !== 'enabled' && serverError.key !== 'section')
const s = profileSettingsOfForm(form)
const setValue = (k: CatalogProfileNumberKey, v: string) => onChange({ ...form, values: { ...form.values, [k]: v } })
const switchError = serverError && (serverError.key === 'enabled' || serverError.key === 'section') ? serverError.message : null
return (
<fieldset className="rounded-lg border" data-field="catalog_profile" data-profile-settings={form.enabled ? 'on' : 'off'}>
<legend className="sr-only">{PT.section}</legend>
<label className="flex cursor-pointer items-start gap-2 p-2.5">
<input id={`${idPrefix}-enabled`} type="checkbox" className="mt-0.5" checked={form.enabled}
aria-describedby={`${idPrefix}-enabled-hint`} aria-invalid={switchError ? true : undefined}
onChange={(e) => onChange({ ...form, enabled: e.target.checked })} data-profile-enabled="" />
<span className="min-w-0 text-xs">
{PT.section}
<span id={`${idPrefix}-enabled-hint`} className="ml-1.5 text-2xs leading-relaxed text-faint">{PT.enableHint}</span>
</span>
</label>
{switchError && (
<p className="px-2.5 pb-2 text-2xs leading-relaxed text-[var(--err)]" role="alert" data-profile-error="section">{switchError}</p>
)}
{open && (
<div className="space-y-3 border-t px-2.5 py-2.5">
{form.enabled && (
<p className="flex items-start gap-1.5 rounded-md px-2 py-1.5 text-2xs leading-relaxed text-dim" data-profile-risk=""
style={{ background: 'var(--st-waiting-soft)' }}>
<ShieldAlert size={12} className="mt-px shrink-0" style={{ color: 'var(--st-waiting)' }} aria-hidden />
<span>{PT.risk(s.max_queries, s.query_timeout_s, s.max_total_s)}</span>
</p>
)}
<div className="grid gap-3 sm:grid-cols-2">
{PROFILE_NUMBER_KEYS.map((k) => {
const { min, max } = PROFILE_LIMITS[k]
const unit = PROFILE_FIELD_UNIT[k]
const err = errorOf(k)
const id = `${idPrefix}-${k}`
const hint = `${PT.range(min, max, unit, PROFILE_DEFAULTS[k])}${k === 'max_scan_rows' ? `,${PT.scanZero}` : ''}。${PROFILE_FIELD_HINT[k]}`
return (
<Field key={k} htmlFor={id} label={PROFILE_FIELD_LABEL[k]} hint={hint} error={err}>
{(p) => (
<div className="relative" data-profile-field={k}>
<input {...p} className="field tnum pr-8" inputMode={PROFILE_LIMITS[k].integer ? 'numeric' : 'decimal'}
autoComplete="off" spellCheck={false} value={form.values[k]}
placeholder={PT.placeholder(PROFILE_DEFAULTS[k])}
style={err ? { borderColor: 'var(--err)' } : undefined}
onChange={(e) => setValue(k, e.target.value)} />
<span className="pointer-events-none absolute right-2.5 top-1/2 -translate-y-1/2 text-2xs text-faint" aria-hidden>
{unit}
</span>
</div>
)}
</Field>
)
})}
</div>
</div>
)}
</fieldset>
)
}
/** 聚焦保存被拒的那一项(预算收着时先等它展开) */
export function focusProfileField(idPrefix: string, key: ProfileErrorKey) {
const id = key === 'section' ? `${idPrefix}-enabled` : `${idPrefix}-${key}`
requestAnimationFrame(() => requestAnimationFrame(() => {
const el = document.getElementById(id)
el?.scrollIntoView({ block: 'nearest' })
el?.focus({ preventScroll: true })
}))
}
/**
* 只改数据剖析设置的弹窗:导入表格的源(没有编辑框)、数据目录页里剖析被拒时用。
* 保存时 options 的其余键(遮罩列、schema……)原样带上,只换剖析设置这一项
*/
export function ProfileSettingsDialog({ row, onClose, onSaved }: {
row: { id: string; name: string; options?: Record<string, unknown> | null }
onClose: () => void
onSaved: (row: any) => void
}) {
const [initial] = useState(() => profileFormOf(row.options))
const [form, setForm] = useState<ProfileForm>(initial)
const [saving, setSaving] = useState(false)
const [serverError, setServerError] = useState<{ key: ProfileErrorKey; message: string } | null>(null)
const dirty = !sameProfileForm(form, initial)
const problems = Object.keys(profileProblems(form)).length
const idPrefix = `profile-${row.id}`
const save = async () => {
if (!dirty || problems) return
setSaving(true)
setServerError(null)
try {
const options = withProfileOption(row.options, profileOptionOf(form, hasProfileOption(row.options)))
const next = await api.datasources.update(row.id, { options })
onSaved(next)
toast.ok(form.enabled === initial.enabled ? PT.saved(row.name) : form.enabled ? PT.savedOn(row.name) : PT.savedOff(row.name))
} catch (e) {
const key = e instanceof ApiError && e.status === 422 ? profileFieldOfRejection(e.message) : null
if (key) {
setServerError({ key, message: e instanceof ApiError ? e.message : '' })
focusProfileField(idPrefix, key)
} else {
toast.error(e)
}
} finally {
setSaving(false)
}
}
return (
<Modal open onClose={onClose} dirty={dirty} width={560} title={PT.dialogTitle(row.name)}
footer={(
<>
{problems > 0 && <span className="mr-auto text-2xs text-[var(--err)]" data-profile-invalid={problems}>{PT.invalid(problems)}</span>}
<button type="button" className="btn" onClick={onClose}>取消</button>
<button type="button" className="btn btn-primary" disabled={saving || !dirty || problems > 0} onClick={() => void save()}
data-profile-save="">
{saving ? <Spinner size={11} /> : null} 保存
</button>
</>
)}>
<div className="space-y-3" data-profile-settings-dialog={row.id}>
<p className="text-2xs leading-relaxed text-faint">{PT.dialogBody}</p>
<ProfileSettingsSection idPrefix={idPrefix} form={form} serverError={serverError}
onChange={(next) => { setForm(next); setServerError(null) }} />
</div>
</Modal>
)
}
+63 -5
View File
@@ -12,13 +12,15 @@ import { useLeaveGuard } from '../../lib/leave'
import { formatDateTime, formatNumber } from '../../lib/format'
import {
CATALOG_CARDINALITY_LABEL, CATALOG_IMPACT_TEXT, CATALOG_KIND_HINT, CATALOG_KIND_LABEL, CATALOG_TABLE_FIELD_HINT,
CATALOG_TABLE_FIELD_LABEL, CATALOG_TEXT as CT,
CATALOG_TABLE_FIELD_LABEL, CATALOG_TEXT as CT, CODES_TEXT as KT,
} from '../../lib/terms'
import { CatalogImpactList } from '../../components/CatalogImpact'
import { CountBadges, ItemMark, StatusChip, StatusLegend, SystemNotesNotice } from './parts'
import type { ReviewTarget } from './parts'
import { ColumnTable, Value } from './ColumnTable'
import { ColumnTable, Value, pendingCodes } from './ColumnTable'
import type { ColumnModel } from './ColumnTable'
import { CodesDialog } from './CodesDialog'
import { coverageText } from './profile'
import {
COLUMN_FIELDS, DATE_KEYS, TABLE_FIELDS, buildNotes, changeCount, columnPath, confirmProposed, countsOf, formFromNotes,
relationDraft, relationPath, tablePath, unknownKeys, validateForm,
@@ -49,13 +51,15 @@ const PROBLEM_TEXT = {
type Editing = { initial: EditForm; form: EditForm }
type Conflict = 'edit' | 'review'
export function TableDetail({ sourceId, table, rows, drafted, prev, next, onOpen, onBack, onDetail }: {
export function TableDetail({ sourceId, table, rows, drafted, fillCodes, prev, next, onOpen, onBack, onDetail }: {
sourceId: string
table: string
/** 全部表:关联关系的目标表能点就跳过去,编辑时给目标表做候选 */
rows: CatalogTableRow[]
/** 起草刚写进了这些表:正在看的表在里面、又没在编辑,就重新载入 */
drafted: { seq: number; tables: Set<string> }
/** 从剖析报告点「填写含义」过来:载入后打开这一列的码值弹窗(seq 让同一列连点两次也会再开) */
fillCodes?: { table: string; column: string; seq: number } | null
prev: string | null
next: string | null
onOpen: (table: string) => void
@@ -73,6 +77,8 @@ export function TableDetail({ sourceId, table, rows, drafted, prev, next, onOpen
const [live, setLive] = useState('')
const [colQuery, setColQuery] = useState('')
const [onlyPending, setOnlyPending] = useState(false)
/** 正在填写哪一列的码值含义 */
const [codesFor, setCodesFor] = useState<string | null>(null)
const loadSeq = useRef(0)
const onDetailRef = useRef(onDetail)
onDetailRef.current = onDetail
@@ -101,6 +107,7 @@ export function TableDetail({ sourceId, table, rows, drafted, prev, next, onOpen
setColQuery('')
setOnlyPending(false)
setLive('')
setCodesFor(null)
void load()
}, [load])
@@ -115,6 +122,15 @@ export function TableDetail({ sourceId, table, rows, drafted, prev, next, onOpen
if (drafted.tables.has(table) && !editingRef.current) void load()
}, [drafted, table, load])
// 剖析报告里点了「填写含义」:这张表载入后打开那一列的码值弹窗。编辑中不打开,免得同一项两处同时改
const fillSeen = useRef(0)
useEffect(() => {
if (!fillCodes || fillCodes.table !== table || fillCodes.seq === fillSeen.current) return
if (!detail || detail.table_name !== table || editingRef.current) return
fillSeen.current = fillCodes.seq
if (detail.notes.columns?.[fillCodes.column]?.codes) setCodesFor(fillCodes.column)
}, [fillCodes, table, detail])
const changes = edit ? changeCount(edit.initial, edit.form) : 0
const problems = useMemo(() => (edit ? validateForm(edit.form, PROBLEM_TEXT) : {}), [edit])
const problemCount = Object.keys(problems).length
@@ -163,6 +179,41 @@ export function TableDetail({ sourceId, table, rows, drafted, prev, next, onOpen
if (done) setLive(CT.confirmedN(n))
}, [sourceId, table, write])
const fillCodesOf = useCallback((col: string) => setCodesFor(col), [])
/**
* 填好的码值含义整份提交:只换这一列码值的值,服务端记为人工填写、已确认。别人刚改过(409)时收起弹窗、挂冲突横幅;
* 其余失败弹窗留着,填的字不丢
*/
const saveCodes = useCallback(async (col: string, value: Record<string, string>) => {
const d = detailRef.current
const item = d?.notes.columns?.[col]?.codes
if (!d || !item) return
const notes = structuredClone(d.notes)
notes.columns![col] = { ...notes.columns![col], codes: { ...item, value } }
setBusy(`codes:${col}`)
setWriteError(null)
try {
const next = await api.dataCatalog.put(sourceId, table, notes, d.version)
setDetail(next)
setConflict(null)
onDetailRef.current(next)
setCodesFor(null)
const filled = Math.max(0, pendingCodes(item.value) - pendingCodes(value))
toast.ok(KT.saved(filled))
setLive(KT.saved(filled))
} catch (e) {
if (e instanceof ApiError && e.status === 409) {
setCodesFor(null)
setConflict('review')
} else {
toast.error(e)
}
} finally {
setBusy('')
}
}, [sourceId, table])
const setColumn = useCallback((col: string, f: ColumnField, v: string) => {
setEdit((e) => e && ({ ...e, form: { ...e.form, columns: { ...e.form.columns, [col]: { ...e.form.columns[col], [f]: v } } } }))
}, [])
@@ -368,7 +419,7 @@ export function TableDetail({ sourceId, table, rows, drafted, prev, next, onOpen
? (
<ColumnTable columns={shownColumns} form={edit?.form.columns ?? null} initial={edit?.initial.columns ?? null}
problems={problems} busy={!!busy} systemNotes={detail.system_notes}
onReview={review} onConfirmColumn={confirmColumn} onChange={setColumn} />
onReview={review} onConfirmColumn={confirmColumn} onFillCodes={fillCodesOf} onChange={setColumn} />
)
: <p className="rounded-lg border px-3 py-6 text-center text-xs text-faint">{CT.noColumns}</p>}
</section>
@@ -394,6 +445,11 @@ export function TableDetail({ sourceId, table, rows, drafted, prev, next, onOpen
</div>
</div>
{codesFor && notes.columns?.[codesFor]?.codes && (
<CodesDialog column={codesFor} item={notes.columns[codesFor]!.codes!} saving={busy === `codes:${codesFor}`}
onSave={(v) => void saveCodes(codesFor, v)} onClose={() => setCodesFor(null)} />
)}
{edit && (
<footer className="flex flex-wrap items-center gap-2 border-t bg-panel px-4 py-2.5" data-catalog-edit-bar="">
<span className="tnum text-xs text-dim" data-catalog-changes={changes}>{changes ? CT.changes(changes) : CT.noChanges}</span>
@@ -597,7 +653,9 @@ function RelationList({ relations, busy, targetOf, onOpen, onReview }: {
<td className={clsx('mono px-2 py-1.5 align-top', rejected && 'text-faint line-through')}>{r.to_columns.join(', ')}</td>
<td className="px-2 py-1.5 align-top">{r.cardinality ? CATALOG_CARDINALITY_LABEL[r.cardinality] : <span className="text-faint">{CT.cardinalityNone}</span>}</td>
<td className="tnum px-2 py-1.5 align-top">
{typeof r.coverage === 'number' ? CT.coverageValue(Math.round(r.coverage * 100)) : <span className="text-faint">—</span>}
{typeof r.coverage === 'number'
? <span data-relation-coverage={r.coverage}>{coverageText(r.coverage)}</span>
: <span className="text-faint">—</span>}
</td>
<td className="px-2 py-1.5 align-top">
<ItemMark target={{ path: relationPath(r.id), where, source: r.source, status: r.status, note: r.note, updated_at: r.updated_at }}
+10 -3
View File
@@ -1,11 +1,11 @@
import { memo, useCallback, useEffect, useRef } from 'react'
import type { KeyboardEvent as ReactKeyboardEvent, MouseEvent as ReactMouseEvent } from 'react'
import { AlertTriangle, Search, Sparkles, X } from 'lucide-react'
import { AlertTriangle, ScanSearch, Search, Sparkles, X } from 'lucide-react'
import clsx from 'clsx'
import type { CatalogTableRow } from '../../types'
import { useRadioGroup } from '../../components/ui'
import { formatNumber } from '../../lib/format'
import { CATALOG_KIND_LABEL, CATALOG_TEXT as CT } from '../../lib/terms'
import { CATALOG_KIND_LABEL, CATALOG_TEXT as CT, PROFILE_TEXT as PT } from '../../lib/terms'
import { CountBadges } from './parts'
import { LIST_FILTERS, LIST_SORTS, progressOf } from './model'
import type { ListFilter, ListSort } from './model'
@@ -17,7 +17,7 @@ import type { ListFilter, ListSort } from './model'
// ===========================================================================
export function TableIndex({
rows, total, counts, query, onQuery, filter, onFilter, sort, onSort, selected, onSelected, active, onOpen, onDraft,
rows, total, counts, query, onQuery, filter, onFilter, sort, onSort, selected, onSelected, active, onOpen, onDraft, onProfile,
}: {
/** 筛选、排序后的行 */
rows: CatalogTableRow[]
@@ -35,6 +35,8 @@ export function TableIndex({
active: string | undefined
onOpen: (table: string) => void
onDraft: () => void
/** 剖析所选(数据剖析弹窗,范围默认是已选的表) */
onProfile?: () => void
}) {
const radio = useRadioGroup(LIST_FILTERS, filter, onFilter)
const listRef = useRef<HTMLUListElement>(null)
@@ -162,6 +164,11 @@ export function TableIndex({
<button type="button" className="btn btn-primary btn-sm" onClick={onDraft} data-catalog-draft-selected="">
<Sparkles size={11} aria-hidden /> {CT.draftSelected(selected.size)}
</button>
{onProfile && (
<button type="button" className="btn btn-sm" onClick={onProfile} data-catalog-profile-selected="">
<ScanSearch size={11} aria-hidden /> {PT.profileSelected(selected.size)}
</button>
)}
</div>
)}
</div>
+6 -3
View File
@@ -205,15 +205,18 @@ function codesText(v: Record<string, string>): string {
return Object.entries(v).map(([k, x]) => `${k}=${x}`).join('\n')
}
/** 码值的文字:每行一个「码值=含义」(全角等号也认)。返回解析结果和第一处错误 */
/**
* 码值的文字:每行一个「码值=含义」(全角等号也认)。含义可以空着(「2=」):数据剖析只知道列里出现过哪些取值,
* 含义等人填,服务端也收空含义。返回解析结果和第一处错误
*/
export function parseCodes(text: string): { value: Record<string, string>; error: { line: number } | { dup: string } | null } {
const value: Record<string, string> = {}
const lines = text.split('\n')
for (let i = 0; i < lines.length; i++) {
const line = lines[i].trim()
if (!line) continue
const m = line.match(/^(.+?)\s*[==]\s*(.+)$/)
if (!m) return { value, error: { line: i + 1 } }
const m = line.match(/^(.+?)\s*[==]\s*(.*)$/)
if (!m || !m[1].trim()) return { value, error: { line: i + 1 } }
const code = m[1].trim()
if (code in value) return { value, error: { dup: code } }
value[code] = m[2].trim()
+175
View File
@@ -0,0 +1,175 @@
import type { CatalogProfileNumberKey, CatalogProfileSettings } from '../../types'
import { PROFILE_FIELD_LABEL, PROFILE_TEXT } from '../../lib/terms'
// ===========================================================================
// 数据剖析的设置:数据源 options.catalog_profile 和表单之间的来回转换、校验、保存被拒时认出是哪一项。
// 规则跟着服务端(backend/app/data/catalog_profile.py 的 ProfileSettings、profile_settings、
// profile_settings_problem)走:缺省即关闭;读的时候宽松(某一项不对就用缺省),写的时候严格(超出范围的拒收)。
// ===========================================================================
/** 数据源 options 里剖析设置的键 */
export const PROFILE_OPTION = 'catalog_profile'
export const PROFILE_NUMBER_KEYS: readonly CatalogProfileNumberKey[] = [
'max_queries', 'query_timeout_s', 'sample_size', 'max_scan_rows', 'max_total_s',
]
/** 缺省值(服务端 ProfileSettings 的默认) */
export const PROFILE_DEFAULTS: CatalogProfileSettings = {
enabled: false, max_queries: 60, query_timeout_s: 10, sample_size: 2000, max_scan_rows: 100_000, max_total_s: 120,
}
/** 各数值项的范围(服务端 _NUMBER_FIELDS):下限、上限、是否必须是整数 */
export const PROFILE_LIMITS: Record<CatalogProfileNumberKey, { min: number; max: number; integer: boolean }> = {
max_queries: { min: 1, max: 500, integer: true },
query_timeout_s: { min: 1, max: 60, integer: false },
sample_size: { min: 10, max: 10_000, integer: true },
max_scan_rows: { min: 0, max: 10_000_000, integer: true },
max_total_s: { min: 10, max: 600, integer: false },
}
/** 一个数:数字和数字写成的文字都认(同服务端 _number)。不合规返回 null */
function numberIn(value: unknown, key: CatalogProfileNumberKey): number | null {
if (value == null || typeof value === 'boolean') return null
const text = String(value).trim().replace(/,/g, '')
if (!text) return null
const n = Number(text)
const { min, max, integer } = PROFILE_LIMITS[key]
if (!Number.isFinite(n) || n < min || n > max) return null
if (integer && !Number.isInteger(n)) return null
return n
}
const rawOf = (options: Record<string, unknown> | null | undefined): Record<string, unknown> | null => {
const raw = options?.[PROFILE_OPTION]
return raw && typeof raw === 'object' && !Array.isArray(raw) ? raw as Record<string, unknown> : null
}
/** options → 生效的设置。宽松:某一项不对用缺省;开关只认 true */
export function profileSettingsOf(options: Record<string, unknown> | null | undefined): CatalogProfileSettings {
const raw = rawOf(options)
const out: CatalogProfileSettings = { ...PROFILE_DEFAULTS, enabled: raw?.enabled === true }
if (!raw) return out
for (const key of PROFILE_NUMBER_KEYS) {
const n = numberIn(raw[key], key)
if (n != null) out[key] = n
}
return out
}
// ---------------------------------------------------------------------------
// 表单:数值项是文字(输入时允许写一半),空着表示用缺省
// ---------------------------------------------------------------------------
export interface ProfileForm {
enabled: boolean
values: Record<CatalogProfileNumberKey, string>
}
export function profileFormOf(options: Record<string, unknown> | null | undefined): ProfileForm {
const raw = rawOf(options)
const values = Object.fromEntries(PROFILE_NUMBER_KEYS.map((k) => {
const v = raw?.[k]
return [k, v == null || typeof v === 'object' ? '' : String(v)]
})) as Record<CatalogProfileNumberKey, string>
return { enabled: raw?.enabled === true, values }
}
/** options 里已经有剖析设置这一项 */
export const hasProfileOption = (options: Record<string, unknown> | null | undefined): boolean => !!rawOf(options)
/**
* 表单 → 要写进 options 的设置。原来没有(stored 为假)、现在也没开、一项都没填的不写(不给 options 平白多一个键);
* 其余写开关和填了的项。填的不是数时原样交上去,由服务端拒收并说明是哪一项(表单上已经先拦过一次)
*/
export function profileOptionOf(form: ProfileForm, stored: boolean): Record<string, unknown> | undefined {
const filled = PROFILE_NUMBER_KEYS.filter((k) => form.values[k].trim())
if (!stored && !form.enabled && !filled.length) return undefined
const out: Record<string, unknown> = { enabled: form.enabled }
for (const k of filled) {
const text = form.values[k].trim().replace(/,/g, '')
const n = Number(text)
out[k] = Number.isFinite(n) ? n : form.values[k].trim()
}
return out
}
/** 表单的格式问题:键 → 说明。空对象表示可以保存 */
export function profileProblems(form: ProfileForm): Partial<Record<CatalogProfileNumberKey, string>> {
const out: Partial<Record<CatalogProfileNumberKey, string>> = {}
for (const k of PROFILE_NUMBER_KEYS) {
if (!form.values[k].trim()) continue
if (numberIn(form.values[k], k) == null) {
const { min, max, integer } = PROFILE_LIMITS[k]
out[k] = PROFILE_TEXT.rangeError(min, max, integer)
}
}
return out
}
/** 表单此刻对应的设置:没填、填错的项按缺省(风险说明据此写出「最多 N 条」) */
export function profileSettingsOfForm(form: ProfileForm): CatalogProfileSettings {
const out: CatalogProfileSettings = { ...PROFILE_DEFAULTS, enabled: form.enabled }
for (const k of PROFILE_NUMBER_KEYS) {
const n = numberIn(form.values[k], k)
if (n != null) out[k] = n
}
return out
}
export const sameProfileForm = (a: ProfileForm, b: ProfileForm): boolean =>
a.enabled === b.enabled && PROFILE_NUMBER_KEYS.every((k) => a.values[k] === b.values[k])
/**
* 服务端保存时拒收的 422(一句中文)说的是剖析设置的哪一项:
* 「数据剖析的「查询次数上限」需要填写 1 到 500 之间的整数;当前为「0」」→ max_queries。
* 说的是剖析设置、但认不出哪一项(格式不对、有认不出的项)返回 'section';说的不是剖析设置返回 null
*/
export function profileFieldOfRejection(message: string): 'enabled' | CatalogProfileNumberKey | 'section' | null {
if (!message.includes('数据剖析')) return null
const named = message.match(/「([^」]+)」/)?.[1] ?? ''
for (const k of ['enabled', ...PROFILE_NUMBER_KEYS] as const) {
if (named && named.startsWith(PROFILE_FIELD_LABEL[k])) return k
}
return 'section'
}
// ---------------------------------------------------------------------------
// 剖析:范围、报告里的读数、409 的分类
// ---------------------------------------------------------------------------
/** 不指定表时服务端剖析几张(PROFILE_DEFAULT_TABLES) */
export const PROFILE_DEFAULT_TABLES = 10
/** 一次最多指定几张表(接口 tables 的 max_length) */
export const PROFILE_MAX_TABLES = 200
/** 覆盖率写成百分数:最多一位小数,不到 100% 的不进成 100%(同服务端 _percent) */
export function coverageText(ratio: number): string {
const text = (ratio * 100).toFixed(1).replace(/\.0$/, '')
return text === '100' && ratio < 1 ? '99.9%' : `${text}%`
}
/** 已用时长的读数:「不到 1 秒」「8 秒」「1 分 05 秒」 */
export function secondsText(ms: number): string {
const s = Math.max(0, Math.floor(ms / 1000))
if (s < 1) return '不到 1 秒'
return s < 60 ? `${s} 秒` : `${Math.floor(s / 60)} 分 ${String(s % 60).padStart(2, '0')} 秒`
}
/** 剖析接口回 409 的几种情况(按服务端的原话认:catalog_profile.ensure_enabled、api/catalog.py、engine.SnapshotTampered) */
export type ProfileBlock = 'disabled' | 'inactive' | 'noSchema' | 'busy' | 'tampered' | 'other'
export function profileBlockOf(message: string): ProfileBlock {
if (message.includes('未开启数据剖析')) return 'disabled'
if (message.includes('已停用')) return 'inactive'
if (message.includes('正在进行数据剖析')) return 'busy'
if (message.includes('探查结构')) return 'noSchema'
if (/已拒绝(?:这次)?查询/.test(message)) return 'tampered'
return 'other'
}
/** 设置里有没有剖析设置这一项(options 的其余键原样带上) */
export function withProfileOption(options: Record<string, unknown> | null | undefined, value: Record<string, unknown> | undefined): Record<string, unknown> {
const { [PROFILE_OPTION]: _old, ...rest } = options ?? {}
return value ? { ...rest, [PROFILE_OPTION]: value } : rest
}
+45 -24
View File
@@ -14,13 +14,16 @@ import { splitShrunk } from '../canvas/copilotMerge'
import { confirmDialog, IconButton, StatusBadge, toast } from '../components/ui'
import { formatCost, formatLapse, formatOffset, formatSpan, formatTokens, shortId, NONE } from '../lib/format'
import { statusLabel } from '../lib/status'
import { runClassLabel } from '../lib/terms'
import { runClassLabel, SQL_CHECK_TEXT } from '../lib/terms'
import { sortSqlChecks } from '../lib/sqlcheck'
import type { RunUsage, ToolChange } from '../types'
import { AssistantStream, type StreamTurn } from './AssistantStream'
import { Composer } from './Composer'
import { SqlCheckBody } from './SqlChecks'
import { EvidenceHostContext, type EvidenceHost } from './evidenceHost'
import {
COPILOT_PHASE_TEXT, copilotOutcome, decodeCopilot, decodeRun, exitLabels, issueLine, SELF_CHECK_ROUNDS, type CopilotIssue, type CopilotOutcome,
COPILOT_PHASE_TEXT, copilotOutcome, decodeCopilot, decodeRun, exitLabels, issueLine, issueSqlCheck, SELF_CHECK_ROUNDS, type CopilotIssue,
type CopilotOutcome,
} from './decode'
import { catalogPatchesOf } from './catalogPatch'
import { ApprovalCard } from './RunPanel'
@@ -289,11 +292,13 @@ function ChatView({ hasRun, onOpenRun }: { hasRun: boolean; onOpenRun: () => voi
)
}
const issues = out.check?.status === 'failed' ? out.check.issues : []
// 对照数据目录的 SQL 检查:不挡运行的提醒、提示(错误交回模型改过,没改好的在上面那组)
const sqlNotes = sortSqlChecks(out.sqlChecks)
const { unasked, asked } = toolLossOf(out)
// 画布一处没变(unchanged)就没有可撤的:给了按钮,撤掉的是这一轮之前的那一步
const undo = source.outcome === 'applied'
|| (out.kind === 'built' && source.outcome !== 'reverted' && source.outcome !== 'unchanged')
if (!issues.length && !out.skipped.length && !unasked.length && !asked.length) {
if (!issues.length && !out.skipped.length && !unasked.length && !asked.length && !sqlNotes.length) {
return undo ? (
<button type="button" className="btn btn-xs btn-ghost" onClick={() => undoCopilotTurn(source.id)}>
<Undo2 size={11} aria-hidden /> 撤销这次生成
@@ -306,6 +311,10 @@ function ChatView({ hasRun, onOpenRun }: { hasRun: boolean; onOpenRun: () => voi
<div className="w-full space-y-1.5">
{/* 没修好的问题直接摊开,不折叠:这是此刻唯一要做的事 */}
{issues.length > 0 && <IssueList issues={issues} onLocate={locateIssue} />}
{sqlNotes.length > 0 && (
<IssueList issues={sqlNotes} onLocate={locateIssue} title={SQL_CHECK_TEXT.title}
plain={sqlNotes.every((x) => x.level === 'info')} />
)}
{(unasked.length > 0 || asked.length > 0) && <ToolLossList unasked={unasked} asked={asked} onLocate={locateTools} />}
<div className="flex flex-wrap items-center gap-1.5">
{repairable && (
@@ -380,28 +389,40 @@ function ChatView({ hasRun, onOpenRun }: { hasRun: boolean; onOpenRun: () => voi
)
}
/** 自查没修好的问题,逐条带「定位」。问题落在哪个节点上认得出来才给;写节点名,认不出才写 id */
function IssueList({ issues, onLocate }: { issues: CopilotIssue[]; onLocate: (x: CopilotIssue) => void }) {
/**
* 自查没修好的问题,逐条带「定位」。问题落在哪个节点上认得出来才给;写节点名,认不出才写 id。
* 对照数据目录的 SQL 检查另画级别、中文规则名、涉及的表和列、SQL 片段(和证据面板同一套),「定位」落到 SQL 那一项。
* title 有值时是不挡运行的那一组(SQL 检查的提醒、提示):全是提示时不用提醒色
*/
function IssueList({ issues, onLocate, title, plain = false }: {
issues: CopilotIssue[]; onLocate: (x: CopilotIssue) => void; title?: string; plain?: boolean
}) {
return (
<ul data-issue-list="" className="space-y-0.5 rounded border px-2 py-1.5 text-2xs leading-relaxed"
style={{ borderColor: 'color-mix(in srgb, var(--st-waiting) 45%, var(--border))',
background: 'var(--st-waiting-soft)' }}>
{issues.map((x, i) => (
<li key={i} className="flex items-start gap-1.5" data-issue-node={x.nodeId} data-issue-field={x.field}>
<span className="min-w-0 flex-1 [overflow-wrap:anywhere]" title={issueLine(x)}>
{x.nodeId && <span className={clsx('text-dim', !x.label && 'mono')}>「{x.label ?? x.nodeId}」</span>}{x.message}
</span>
{x.nodeId && (
<button type="button"
className="inline-flex shrink-0 items-center gap-0.5 rounded px-1 text-dim transition-colors hover:bg-hover hover:text-fg"
title={x.field ? '打开该节点的设置,定位到出问题的配置项' : '在画布上定位这个节点'}
onClick={() => onLocate(x)}>
<Crosshair size={10} aria-hidden /> 定位
</button>
)}
</li>
))}
</ul>
<div data-issue-list={title ? 'sql' : ''} className="rounded border px-2 py-1.5 text-2xs leading-relaxed"
style={plain ? undefined : { borderColor: 'color-mix(in srgb, var(--st-waiting) 45%, var(--border))', background: 'var(--st-waiting-soft)' }}>
{title && <div className="mb-1 font-medium text-fg">{title}</div>}
<ul className="space-y-1">
{issues.map((x, i) => {
const sql = issueSqlCheck(x)
return (
<li key={i} className="flex items-start gap-1.5" data-issue-node={x.nodeId} data-issue-field={x.field} data-issue-code={x.code}>
<div className="min-w-0 flex-1 [overflow-wrap:anywhere]" title={issueLine(x)}>
{x.nodeId && <span className={clsx('text-dim', !x.label && 'mono')}>「{x.label ?? x.nodeId}」</span>}
{sql ? <div className="mt-0.5"><SqlCheckBody check={sql} /></div> : x.message}
</div>
{x.nodeId && (
<button type="button"
className="inline-flex shrink-0 items-center gap-0.5 rounded px-1 text-dim transition-colors hover:bg-hover hover:text-fg"
title={x.field ? '打开该节点的设置,定位到出问题的配置项' : '在画布上定位这个节点'}
onClick={() => onLocate(x)}>
<Crosshair size={10} aria-hidden /> 定位
</button>
)}
</li>
)
})}
</ul>
</div>
)
}
+4 -3
View File
@@ -8,7 +8,7 @@ import { CatalogImpactList } from '../components/CatalogImpact'
import { CATALOG_PATCH_TEXT as T, CATALOG_STATUS_LABEL } from '../lib/terms'
import type { CatalogPatchChange } from '../types'
import {
catalogTableHref, patchSubmit, patchTarget, patchValueText, patchWhere, type CatalogPatch,
catalogTableHref, codeMeaning, patchSubmit, patchTarget, patchValueText, patchWhere, type CatalogPatch,
} from './catalogPatch'
// ===========================================================================
@@ -211,8 +211,9 @@ function CodesAfter({ before, after }: { before: unknown; after: unknown }) {
{entries.map(([k, v], i) => (
<span key={k}>
{i > 0 && '、'}
<span className={clsx(prev[k] !== v && 'font-medium text-[var(--accent)]')} data-code-changed={prev[k] !== v ? k : undefined}>
{k}={v}
<span className={clsx(prev[k] !== v && 'font-medium text-[var(--accent)]', !String(v ?? '').trim() && 'text-faint')}
data-code-changed={prev[k] !== v ? k : undefined}>
{k}={codeMeaning(v)}
</span>
</span>
))}
+21 -1
View File
@@ -18,7 +18,7 @@ import {
} from '../lib/evidence'
import { humanizeError } from '../lib/errors'
import { formatDateTime, formatNumber, NONE, shortId } from '../lib/format'
import { EVIDENCE_TEXT, JUDGE_TEXT, MERGE_TEXT, NODE_TYPE_LABEL, nodeTypeLabel } from '../lib/terms'
import { EVIDENCE_TEXT, JUDGE_TEXT, MERGE_TEXT, NODE_TYPE_LABEL, SQL_CHECK_TEXT, nodeTypeLabel } from '../lib/terms'
import { useCatalog } from '../store/catalog'
import { askKey, segmentKey, useEvidence, useExplore, useVerdicts } from '../store/evidence'
import type {
@@ -29,6 +29,7 @@ import { ArtifactViewer, ResultTable } from './AssistantStream'
import { useEvidenceHost } from './evidenceHost'
import { EvidenceProvenance } from './EvidenceProvenance'
import { CopyChip } from './Markdown'
import { SqlCheckList } from './SqlChecks'
/**
* 证据面板:点开报告里的一个片段,看它从哪来。
@@ -1109,6 +1110,9 @@ function MetricPart({ step, entry, inputs, runId, seal, pending, failed, resolve
// 拿截断的查询结果整组算出来的:值是真的算出来的,但只算到了取回的那部分,出具已按缺口降档
const incomplete = step?.incomplete === true || entry?.incomplete === true
const incompleteReason = step?.incomplete_reason || entry?.incomplete_reason || EVIDENCE_TEXT.incompleteFallback
// 所依据的查询对照数据目录查出了错误级的问题:值照算,结果不可靠,出具同样按缺口降档。问题和改法在查询步骤里
const sqlFailed = step?.sql_check_failed === true || entry?.sql_check_failed === true
const sqlReason = step?.sql_check_reason || entry?.sql_check_reason || EVIDENCE_TEXT.sqlCheckFallback
return (
<Part title={EVIDENCE_TEXT.metric} data-ev-metric="">
@@ -1121,6 +1125,12 @@ function MetricPart({ step, entry, inputs, runId, seal, pending, failed, resolve
{EVIDENCE_TEXT.incomplete}
</span>
)}
{sqlFailed && (
<span className="chip" data-ev-sql-check-failed="" title={sqlReason}
style={{ color: 'var(--st-waiting)', borderColor: 'var(--st-waiting)' }}>
{EVIDENCE_TEXT.sqlCheckFailed}
</span>
)}
</div>
{(caliber || version || artifact || from) && (
<div className="mt-0.5 flex flex-wrap items-center gap-x-2 text-2xs text-dim">
@@ -1139,6 +1149,9 @@ function MetricPart({ step, entry, inputs, runId, seal, pending, failed, resolve
{incomplete && (
<p className="mt-1" style={{ color: 'var(--st-waiting)' }} data-ev-incomplete-reason="">{incompleteReason}</p>
)}
{sqlFailed && (
<p className="mt-1" style={{ color: 'var(--st-waiting)' }} data-ev-sql-check-reason="">{sqlReason}</p>
)}
{unshowable && (
<p className="mt-1" style={{ color: 'var(--st-waiting)' }} data-ev-unshowable="">
{EVIDENCE_TEXT.unshowable(valueText(value))}
@@ -1366,6 +1379,13 @@ function QueryPart({ id, step, seal, masked }: { id: string; step: EvidenceStep;
{step.sql}
</pre>
)}
{!!step.checks?.length && (
// 这次查询对照数据目录查出的问题:说明、涉及的表和列、SQL 片段。没查出问题时接口不给这个键
<div className="mb-1.5" data-ev-sql-checks="">
<div className="mb-0.5 text-2xs text-faint">{SQL_CHECK_TEXT.title}</div>
<SqlCheckList checks={step.checks} />
</div>
)}
{!!step.merge?.warnings.length && (
<div className="mb-1.5 text-2xs" data-ev-merge-warnings="">
<div className="mb-0.5 text-faint">{MERGE_TEXT.executionWarnings}</div>
+74
View File
@@ -0,0 +1,74 @@
import { Info, OctagonX, TriangleAlert } from 'lucide-react'
import type { LucideIcon } from 'lucide-react'
import clsx from 'clsx'
import type { SqlCheckItem, SqlCheckLevel } from '../types'
import { SQL_CHECK_LEVEL_HINT, SQL_CHECK_LEVEL_LABEL, SQL_CHECK_RULE_HINT, SQL_CHECK_TEXT } from '../lib/terms'
import { SQL_CHECK_TONE, isSqlCheckCode, sortSqlChecks, sqlCheckLevelOf, sqlCheckWhere, sqlRuleLabel } from '../lib/sqlcheck'
// ===========================================================================
// SQL 检查的结果:级别标识(错误、提醒、提示,四通道:图标、文字、颜色、边框)、中文规则名、给人看的说明、涉及的表和列、
// SQL 片段。证据面板的查询步骤和助手面板共用这一份。
// ===========================================================================
const LEVEL_ICON: Record<SqlCheckLevel, LucideIcon> = { error: OctagonX, warning: TriangleAlert, info: Info }
/** 级别标识:图标 + 「错误 / 提醒 / 提示」,悬停写明依据的确证程度 */
export function SqlLevelBadge({ level: raw, className }: { level: unknown; className?: string }) {
const level = sqlCheckLevelOf(raw)
const Icon = LEVEL_ICON[level]
const tone = SQL_CHECK_TONE[level]
return (
<span className={clsx('inline-flex shrink-0 items-center gap-0.5 whitespace-nowrap rounded-full border px-1.5 text-2xs leading-[16px]', className)}
style={{ color: tone, borderColor: `color-mix(in srgb, ${tone} 45%, var(--border))`, background: `color-mix(in srgb, ${tone} 8%, transparent)` }}
title={SQL_CHECK_LEVEL_HINT[level]} data-sql-level={level}>
<Icon size={10} aria-hidden />
{SQL_CHECK_LEVEL_LABEL[level]}
</span>
)
}
/** 一条检查的正文:规则名、说明、涉及的表和列、SQL 片段 */
export function SqlCheckBody({ check: c, excerpt = true }: { check: SqlCheckItem; excerpt?: boolean }) {
const where = sqlCheckWhere(c)
return (
<>
<div className="flex min-w-0 flex-wrap items-center gap-x-1.5 gap-y-0.5">
<SqlLevelBadge level={c.level} />
<span className="font-medium text-fg" title={isSqlCheckCode(c.code) ? SQL_CHECK_RULE_HINT[c.code] : undefined} data-sql-rule="">
{sqlRuleLabel(c.code)}
</span>
{where && (
<span className="text-faint" data-sql-where={where}>
{SQL_CHECK_TEXT.where} <span className="mono text-dim [overflow-wrap:anywhere]">{where}</span>
</span>
)}
</div>
{c.message && <p className="mt-0.5 leading-relaxed text-dim [overflow-wrap:anywhere]" data-sql-message="">{c.message}</p>}
{excerpt && c.sql_excerpt && (
<pre className="mono mt-1 max-h-20 overflow-auto whitespace-pre-wrap rounded bg-bg px-1.5 py-1 leading-relaxed text-dim [overflow-wrap:anywhere]"
aria-label={SQL_CHECK_TEXT.excerpt} title={SQL_CHECK_TEXT.excerpt} data-sql-excerpt="">
{c.sql_excerpt}
</pre>
)}
</>
)
}
/** 检查结果的清单:错误在前。每条一个淡色框,框的颜色跟级别走 */
export function SqlCheckList({ checks, className }: { checks: SqlCheckItem[]; className?: string }) {
const list = sortSqlChecks(checks)
return (
<ul className={clsx('space-y-1 text-2xs', className)} aria-label={SQL_CHECK_TEXT.listLabel(list.length)} data-sql-checks={list.length}>
{list.map((c, i) => {
const tone = SQL_CHECK_TONE[sqlCheckLevelOf(c.level)]
return (
<li key={`${c.code}:${c.table}:${c.column ?? ''}:${i}`} className="rounded border px-2 py-1.5"
style={{ borderColor: `color-mix(in srgb, ${tone} 30%, var(--border))`, background: `color-mix(in srgb, ${tone} 5%, transparent)` }}
data-sql-check={c.code} data-level={sqlCheckLevelOf(c.level)}>
<SqlCheckBody check={c} />
</li>
)
})}
</ul>
)
}
+6 -3
View File
@@ -1,7 +1,7 @@
import type { CatalogBusinessDate, CatalogCardinality, CatalogMeasure, CatalogPatchChange, CatalogTableKind } from '../types'
import {
CATALOG_CARDINALITY_LABEL, CATALOG_COLUMN_FIELD_LABEL, CATALOG_KIND_LABEL, CATALOG_MEASURE_LABEL, CATALOG_PATCH_TEXT,
CATALOG_TABLE_FIELD_LABEL, CATALOG_TEXT,
CATALOG_TABLE_FIELD_LABEL, CATALOG_TEXT, CODES_TEXT,
} from '../lib/terms'
// ===========================================================================
@@ -108,10 +108,13 @@ export function relationText(v: unknown): string {
return `${left} → ${right}${card ? `(${card})` : ''}`
}
/** 码值写成一行:1=有效、9=作废 */
/** 码值的含义:空串是数据剖析写进来的候选,含义还要人填,和目录页同一个说法 */
export const codeMeaning = (x: unknown) => (String(x ?? '').trim() ? String(x) : CODES_TEXT.pending)
/** 码值写成一行:1=有效、9=作废、8=含义待填写 */
export function codesText(v: unknown): string {
if (!v || typeof v !== 'object') return String(v)
return Object.entries(v as Record<string, string>).map(([k, x]) => `${k}=${x}`).join('、')
return Object.entries(v as Record<string, string>).map(([k, x]) => `${k}=${codeMeaning(x)}`).join('、')
}
/** 一项的值写成一行字(过程里的明细、卡片里的改前改后都用它)。没有值写「未填写」 */
+84 -9
View File
@@ -1,11 +1,13 @@
import type { CatalogDriftTable, RunEvent, TeamMember, TeamRound, TeamRun, ToolChange } from '../types'
import type { CatalogDriftTable, RunEvent, SqlCheckItem, TeamMember, TeamRound, TeamRun, ToolChange } from '../types'
import type { NodeState, RunPhase } from './trace'
import { formatDuration, formatNumber } from '../lib/format'
import {
CATALOG_DRIFT_TEXT, CATALOG_PATCH_TEXT, JUDGE_TEXT, MERGE_TEXT, TYPE_LABEL, claimTally, evidenceTally, issuanceLabel, nodeTypeLabel,
CATALOG_DRIFT_TEXT, CATALOG_PATCH_TEXT, JUDGE_TEXT, MERGE_TEXT, SQL_CHECK_TEXT, TYPE_LABEL, claimTally, evidenceTally,
issuanceLabel, nodeTypeLabel,
} from '../lib/terms'
import { catalogPatchOf, patchValueText, patchWhere } from './catalogPatch'
import { claimCountsOf, isJudged, statsTally } from '../lib/evidence'
import { isSqlCheckCode, sortSqlChecks, sqlCheckWhere, sqlRuleLabel } from '../lib/sqlcheck'
// 泳道数据画布也要用(supervisor 节点要展开成协作矩阵),所以类型放在
// types.ts 里;这里再导出一遍,老引用不用改
@@ -778,6 +780,20 @@ function explainLog(code: string | undefined, message: string): Pick<Step, 'titl
fix: 'canvas',
}
}
case 'metric_sql_check': {
// 「指标「客单价」所依据的查询未通过 SQL 检查(「订单」关联「订单明细」是一对多,对「订单」的「订单金额」求和会
// 重复计算),结果不可靠」;几处时是「所依据的查询有 2 处未通过 SQL 检查(…等),结果不可靠」(metrics.py
// _sql_check_reason)。括号里的问题本身可能带括号:取到「,结果不可靠」前面最后一个右括号
const name = message.match(/^指标「(.+?)」/)?.[1]
const n = message.match(/有\s*(\d+)\s*处未通过 SQL 检查/)?.[1]
const why = message.match(/未通过 SQL 检查(([\s\S]+))[,,]\s*结果不可靠\s*$/)?.[1]
return {
title: name ? SQL_CHECK_TEXT.metricTitle(name, n ? Number(n) : 1) : SQL_CHECK_TEXT.metricTitleAnon,
sub: why,
next: SQL_CHECK_TEXT.metricNext,
fix: 'canvas',
}
}
case 'judge_limit': {
// 「结论句裁判已到上限(这份报告的裁判金额上限 $0.05):3 句没判,记为未裁判;已判的保留」(judge.py run_request)
const n = message.match(/[::]\s*(\d+)\s*句(?:没判|未裁判)/)?.[1]
@@ -2452,8 +2468,14 @@ export interface CopilotIssue {
label?: string
/** 落在节点的哪一项配置上(condition、tools、agents[1].tools):「定位」据此聚焦到检查器里那个输入框 */
field?: string
/** 机读代号:datasource_out_of_scope、tools_dropped…… */
/** 机读代号:datasource_out_of_scope、tools_dropped……;SQL 检查是规则编号(fanout_sum……) */
code?: string
/** error / warning / info。老会话、老后端不给 */
level?: 'error' | 'warning' | 'info'
/** SQL 检查的问题另带:涉及的表和列、SQL 片段 */
table?: string
column?: string
sqlExcerpt?: string
}
/**
@@ -2468,11 +2490,16 @@ function parseIssue(v: unknown, labels?: (id: string) => string | undefined): Co
}
if (v && typeof v === 'object') {
const o = v as Record<string, unknown>
const str = (x: unknown) => (typeof x === 'string' && x ? x : undefined)
return named({
message: String(o.message ?? ''),
...(o.node_id ? { nodeId: String(o.node_id) } : {}),
...(typeof o.field === 'string' && o.field ? { field: o.field } : {}),
...(typeof o.code === 'string' && o.code ? { code: o.code } : {}),
...(str(o.field) ? { field: str(o.field) } : {}),
...(str(o.code) ? { code: str(o.code) } : {}),
...(o.level === 'error' || o.level === 'warning' || o.level === 'info' ? { level: o.level } : {}),
...(str(o.table) ? { table: str(o.table) } : {}),
...(str(o.column) ? { column: str(o.column) } : {}),
...(str(o.sql_excerpt) ? { sqlExcerpt: str(o.sql_excerpt) } : {}),
})
}
const text = String(v ?? '')
@@ -2480,9 +2507,37 @@ function parseIssue(v: unknown, labels?: (id: string) => string | undefined): Co
return named(m ? { nodeId: m[1], message: m[2] || text } : { message: text })
}
/** 一条自查问题写成一行:带节点名(认得出时),认不出才退到 id */
export const issueLine = (x: CopilotIssue): string =>
x.nodeId ? `「${x.label ?? x.nodeId}」${x.message}` : x.message
/**
* 一条自查问题写成一行:带节点名(认得出时),认不出才退到 id。SQL 检查的问题前面写中文规则名、后面写涉及的表和列,
* 不露规则编号
*/
export const issueLine = (x: CopilotIssue): string => {
const where = isSqlCheckCode(x.code) ? sqlCheckWhere({ table: x.table ?? '', column: x.column }) : ''
const body = isSqlCheckCode(x.code)
? `${sqlRuleLabel(x.code)}:${x.message}${where ? `(${SQL_CHECK_TEXT.where} ${where})` : ''}`
: x.message
return x.nodeId ? `「${x.label ?? x.nodeId}」${body}` : body
}
/** 一条自查问题按 SQL 检查读(界面上画级别、规则名、表和列、SQL 片段);不是 SQL 检查的返回 null */
export function issueSqlCheck(x: CopilotIssue): SqlCheckItem | null {
if (!isSqlCheckCode(x.code)) return null
return {
code: x.code, level: x.level ?? 'error', message: x.message, table: x.table ?? '',
...(x.column ? { column: x.column } : {}), ...(x.sqlExcerpt ? { sql_excerpt: x.sqlExcerpt } : {}),
}
}
/** final.issues 里对照数据目录的 SQL 检查结果(不挡运行的提醒、提示也在里面);自查没修好、已经列过的错误不再算 */
function sqlIssuesOf(ops: CopilotOp[], labels: (id: string) => string | undefined): CopilotIssue[] {
const final = [...ops].reverse().find((o) => o.op === 'final')
const list: unknown[] = final && Array.isArray(final.issues) ? final.issues : []
const failed = [...ops].reverse().find((o) => o.op === 'check' && o.status === 'failed')
const listed = new Set((failed && Array.isArray(failed.issues) ? failed.issues : [])
.map((x: unknown) => parseIssue(x)).map((x: CopilotIssue) => `${x.nodeId ?? ''}|${x.code ?? ''}|${x.message}`))
return list.map((x) => parseIssue(x, labels))
.filter((x) => isSqlCheckCode(x.code) && !listed.has(`${x.nodeId ?? ''}|${x.code ?? ''}|${x.message}`))
}
/**
* 节点 id → 名字。先认这一轮操作流里加过、改过的(问数据页没有画布,只有这一份),
@@ -2817,6 +2872,23 @@ export function decodeCopilot(ops: CopilotOp[], opts?: {
add({ id: `cm-${i}-${t}`, seq: i, kind: 'note', status: 'done', level: 'warn',
title: clip(`已跳过一个步骤:节点类型「${t}」不存在`, 80) })
}
// 对照数据目录的 SQL 检查:错误交回模型改过(没改好的在自查那一行),不挡运行的提醒、提示照样要看得见
const sql = sqlIssuesOf(ops.slice(0, i), labels)
if (sql.length) {
const n = { error: 0, warning: 0, info: 0 }
for (const x of sql) n[x.level ?? 'error'] += 1
const nodes = [...new Set(sql.map((x) => x.nodeId).filter(Boolean))]
add({
id: `cs-${i}`, seq: i, kind: 'note', status: 'done', code: 'sql_check',
...(n.error || n.warning ? { level: 'warn' as const } : {}),
title: SQL_CHECK_TEXT.assistantTitle(SQL_CHECK_TEXT.counts(n.error, n.warning, n.info)),
detail: sortSqlChecks(sql).map(issueLine).join('\n'),
// 问数据页没有画布:下一步说去画布里打开这个工作流
next: n.error || n.warning ? (canvas ? SQL_CHECK_TEXT.assistantNext : SQL_CHECK_TEXT.assistantNextChat)
: SQL_CHECK_TEXT.assistantInfoNext,
...(nodes.length === 1 ? { nodeId: nodes[0], ...(canvas ? { fix: 'canvas' as const } : {}) } : {}),
})
}
// 工具绑定变化单独成一行:改图回执以前只说「修改 1」,模型改提示词时漏写 tools、
// 把查库的工具整个抹掉,要到运行结果不对才发现
const changes = toolChangesOf(op)
@@ -2913,6 +2985,8 @@ export interface CopilotOutcome {
toolChanges: ToolChange[]
/** 自查的「工具被去掉了」警告(tools_dropped):这一轮的要求里没提到去掉 */
dropped: CopilotIssue[]
/** 对照数据目录的 SQL 检查结果(final.issues),自查没修好、已经列在 check 里的不重复 */
sqlChecks: CopilotIssue[]
}
export function copilotOutcome(
@@ -2920,7 +2994,7 @@ export function copilotOutcome(
): CopilotOutcome {
const res: CopilotOutcome = {
kind: running ? 'running' : 'empty', skipped: [], added: 0, updated: 0, removed: 0,
toolChanges: [], dropped: [],
toolChanges: [], dropped: [], sqlChecks: [],
}
const labels = labelsOf(ops, labelOf)
for (const op of ops) {
@@ -2952,6 +3026,7 @@ export function copilotOutcome(
res.toolChanges = toolChangesOf(op)
// 同一批警告 final.issues 里也有一份:自查已经带回来了就不再算一遍
if (!res.dropped.length) res.dropped = droppedOf(op)
res.sqlChecks = sqlIssuesOf(ops, labels)
if (!running) res.kind = 'built'
break
case 'error':
+141
View File
@@ -2147,6 +2147,12 @@ export interface EvidenceStep extends Omit<EvidenceInput, 'status'> {
incomplete?: boolean
/** metric:不完整的原因(人话,「基于被截断的查询结果计算(只取回了前 1000 行),结果不完整」) */
incomplete_reason?: string
/** metric:所依据的查询没通过 SQL 检查(有错误级的问题),值照算,出具已按缺口降档 */
sql_check_failed?: boolean
/** metric:没通过的原因(人话,「所依据的查询未通过 SQL 检查(…),结果不可靠」) */
sql_check_reason?: string
/** query:这次查询对照数据目录查出的问题。没查出问题时没有这个键 */
checks?: SqlCheckItem[]
inputs?: EvidenceInput[]
/** 口径卡钉在哪个工作流的哪一版(metric;方案第 5 节的写法,接口实际给在 source 上) */
caliber_from?: EvidenceCaliberSource | null
@@ -2786,3 +2792,138 @@ export interface CatalogImpact {
/** 直接引用的在前 */
templates: CatalogImpactTemplate[]
}
// ===========================================================================
// 数据剖析(后端 app/data/catalog_profile.py):对业务库发少量只读查询,核实推断的关联关系、取码值候选、
// 提议业务日期。默认关闭,按数据源在 options.catalog_profile 里开启。
// ===========================================================================
/** 剖析设置里的数值项 */
export type CatalogProfileNumberKey = 'max_queries' | 'query_timeout_s' | 'sample_size' | 'max_scan_rows' | 'max_total_s'
/** 一个数据源的剖析设置(服务端 ProfileSettings.to_dict) */
export interface CatalogProfileSettings {
enabled: boolean
max_queries: number
query_timeout_s: number
sample_size: number
/** 0 表示一律不做整表统计 */
max_scan_rows: number
max_total_s: number
}
/** 整次剖析中途停下的原因:查询次数用完、总时长用完、连续多条查询失败 */
export type CatalogProfileStop = 'budget' | 'deadline' | 'failed'
/** 一张表的行数:stats 数据库的统计信息(估算);count 数到上限为止;unknown 没能得到 */
export interface CatalogProfileSize {
rows: number | null
method: 'stats' | 'count' | 'unknown' | (string & {})
/** 数到上限也没数完:至少这么多行 */
at_least: number | null
}
/** 关系的核对结论:覆盖率、基数,升为已验证(verified)或保持推断(proposed);人工确认过的只补覆盖率和基数 */
export interface CatalogProfileRelationFinding {
kind: 'relation'
path: string
target: string
columns: string[]
to_table: string
to_columns: string[]
status: CatalogStatus
confirmed: boolean
coverage: number
cardinality: CatalogCardinality | null
/** 抽了几个不同的键值、在被指向表里对上几个 */
sample: number
matched: number
summary: string
}
/** 码值候选:观察到的取值和行数,含义留给人填 */
export interface CatalogProfileCodesFinding {
kind: 'codes'
path: string
column: string
values: { value: string; rows: number }[]
rows: number
status: CatalogStatus
summary: string
}
/** 业务日期提议:表里只有一个日期类列 */
export interface CatalogProfileDateFinding {
kind: 'business_date'
path: string
column: string
min: string
max: string
status: CatalogStatus
summary: string
}
export type CatalogProfileFinding = CatalogProfileRelationFinding | CatalogProfileCodesFinding | CatalogProfileDateFinding
/** 没做的一项。detail 是可以直接显示的整句 */
export interface CatalogProfileSkip {
/** relation / codes / date / row_estimate / table */
kind: string
target: string
path: string | null
/** budget / deadline / failed / timeout / error / rejected / masked / too_large / view / unsupported / no_data / missing / high_cardinality */
reason: string
detail: string
}
export interface CatalogProfileTable {
table_name: string
queries: number
row_estimate: CatalogProfileSize | null
findings: CatalogProfileFinding[]
skipped: CatalogProfileSkip[]
/** 日期类列的取值范围 */
date_ranges: { column: string; min: string; max: string }[]
added: number
updated: number
removed: number
version: number
/** 这张表没剖析(表结构里没有)或结果没写进去(写入一直冲突) */
error: string | null
}
/** POST /datasources/{id}/catalog/profile */
export interface CatalogProfileOut {
profiled_at: string
actor: string | null
settings: CatalogProfileSettings
queries_used: number
stopped: CatalogProfileStop | null
tables: CatalogProfileTable[]
total: { added: number; updated: number; removed: number }
/** 一张表都没有剖析时的说明 */
note: string | null
}
// ===========================================================================
// 基于数据目录的 SQL 检查(后端 app/data/sqlcheck.py):七条规则,编号稳定
// ===========================================================================
export type SqlCheckCode = 'fanout_sum' | 'stock_summed' | 'join_unconfirmed' | 'ratio_aggregated' | 'missing_valid_filter'
| 'unknown_code' | 'wrong_date_column'
/** error:依据有确证、结果必然有误;warning:依据有确证、很可能有误;info:依据只是推断 */
export type SqlCheckLevel = 'error' | 'warning' | 'info'
/** 一条检查结果(证据接口查询步骤里的 checks、助手自查和发布前检查的问题)。交回模型改写用的那句不上界面,这里不收 */
export interface SqlCheckItem {
code: SqlCheckCode | (string & {})
level: SqlCheckLevel | (string & {})
/** 给人看的说明 */
message: string
/** schema_cache 里的表名 */
table: string
column?: string
relation_id?: string
sql_excerpt?: string
}
+395
View File
@@ -4,6 +4,8 @@
// 所以这里查的是「请求里带没带读到的版本」「批量确认只动推断项、来源不变」「编辑时没动的项原样交回(被驳回的照旧
// 驳回)」「别人刚改过时有没有停下来让人重新载入,而不是悄悄覆盖」「起草按批调用、能停、出错时说得清」——
// 正常路径的页面检查一个字都不会说。
// 数据剖析(阶段 4B)会对业务库发查询:剖析接口、保存数据源一律在浏览器层答掉,查的是「确认前写没写预算」「进行中是否
// 明说停不下来」「报告说没说清用了几条查询、为什么停、跳过了什么」「各种 409 有没有下一步」「设置被拒时报错落没落到那一项」。
//
// 写法同 check-versions:沙箱里的数据源多半没探查过结构,目录也是空的,所以数据源列表、目录清单、单表目录都用
// page.route 伪造(虚构的景区业务库和一个导入表格的源);写请求(整份提交、单项审阅、起草)也在浏览器层答掉,
@@ -400,6 +402,14 @@ function catalogHandlers(state, replies = {}) {
const name = parts(key)[4]
return json({ table: name, templates: state.impact?.[name] ?? [] })(route)
})],
// 剖析没配答复的一律 503:检查脚本不能让剖析真的发到业务库
[/^POST \/datasources\/check-cat-[a-z]+\/catalog\/profile$/, reply('profile', json({ detail: '检查脚本没有为这次剖析准备答复' }, 503))],
// 保存数据源(剖析设置):照提交改假状态,回整条数据源
[/^PATCH \/datasources\/check-cat-[a-z]+$/, reply('patch', (route, { key, body }) => {
const id = parts(key)[2]
state.sources = state.sources.map((s) => (s.id === id ? { ...s, ...body, password: undefined } : s))
return json(state.sources.find((s) => s.id === id))(route)
})],
[/^GET \/datasources\/check-cat-[a-z]+\/catalog\/[^/]+$/, reply('get', (route, { key }) => {
const { src, t } = findTable(state, key)
return t ? json(detailOf(src, t))(route) : json({ detail: `数据源中没有表 ${parts(key)[4]},可能已被删除或尚未探查结构` }, 404)(route)
@@ -1103,6 +1113,384 @@ await section('数据目录 · 空状态与导入表格', async () => {
await close()
})
// ---------------------------------------------------------------------------
// 数据剖析(阶段 4B):剖析会对业务库发查询,这里一条都不能真的发出去——剖析接口和保存数据源都在浏览器层答掉,
// 照服务端(data/catalog_profile.py)的结论改假状态
// ---------------------------------------------------------------------------
const GATE_REL = relId(['gate_id'], 'gates', ['id'])
const MEMBER_REL = relId(['member_id'], 'members', ['id'])
const PROFILE_DAY = '2026-10-03'
const PROFILE_ON = { enabled: true, max_queries: 60, query_timeout_s: 10 }
const withProfile = (state, src, value) => {
state.sources = state.sources.map((s) => (s.id === src ? { ...s, options: { ...s.options, catalog_profile: value } } : s))
}
const BUDGET_DETAIL = '本次剖析的查询次数已用完(上限 60 条),未检查'
/**
* 剖析的默认答复:入园记录的闸机关系核实成已验证(覆盖率 98.7%),会员关系覆盖率不够保持推断,状态列的码值候选多出一个
* 没有含义的取值 2;订单只剖析了一半就用完了查询次数,订单明细一条没查。ms 是服务端「执行」多久
*/
function profileReply(state, { ms = 0 } = {}) {
return async (route) => {
if (ms) await sleep(ms)
const visits = state.tables[S1].find((t) => t.name === 'visits')
const notes = clone(visits.notes)
const at = `${PROFILE_DAY}T03:00:00+00:00`
notes.relations = notes.relations.map((r) => (r.id === GATE_REL ? {
...r, source: 'profile', status: 'verified', coverage: 0.987, cardinality: 'many_to_one', updated_at: at,
note: `数据剖析(${PROFILE_DAY}):子表抽样 300 个不同键值,父表对上 296 个,覆盖率 98.7%;被指向列是主键;子表一侧有重复值,多对一。`,
} : r))
notes.relations.push(rel(['member_id'], 'members', ['id'], 'profile', { coverage: 0.62, updated_at: at,
note: `数据剖析(${PROFILE_DAY}):抽样覆盖率 62%,可能不是这条关系;子表抽样 200 个不同键值,父表对上 124 个;被指向列是主键。` }))
notes.columns.status.codes = item({ 1: '已入园', 9: '已作废', 2: '' }, 'profile', 'proposed')
notes.columns.status.codes.note = `数据剖析(${PROFILE_DAY}):统计全表,182000 行非空值共 3 个取值:1(94%)、9(5%)、2(1%)。`
writeEntry(visits, notes)
const tables = [
{
table_name: 'visits', queries: 38, row_estimate: { rows: 182000, method: 'stats', at_least: null },
findings: [
{ kind: 'relation', path: `relations.${GATE_REL}`, target: 'gate_id → gates.id', columns: ['gate_id'], to_table: 'gates', to_columns: ['id'],
status: 'verified', confirmed: false, coverage: 0.987, cardinality: 'many_to_one', sample: 300, matched: 296, summary: '覆盖率 98.7%,升为有确证' },
{ kind: 'relation', path: `relations.${MEMBER_REL}`, target: 'member_id → members.id', columns: ['member_id'], to_table: 'members',
to_columns: ['id'], status: 'proposed', confirmed: false, coverage: 0.62, cardinality: 'many_to_one', sample: 200, matched: 124,
summary: '抽样覆盖率 62%,保持推断' },
{ kind: 'codes', path: 'columns.status.codes', column: 'status', values: [{ value: '1', rows: 171080 }, { value: '9', rows: 9100 },
{ value: '2', rows: 1820 }], rows: 182000, status: 'proposed', summary: '3 个取值,1 个含义待填写' },
],
skipped: [
{ kind: 'codes', target: 'member_level', path: 'columns.member_level.codes', reason: 'masked', detail: 'visits.member_level 在数据源设置中被遮罩,不取值' },
{ kind: 'date', target: 'visit_time', path: 'business_date', reason: 'too_large',
detail: 'visits 的行数超过整表统计的行数上限(100000 行),不取日期列的最小值和最大值' },
],
date_ranges: [], added: 1, updated: 2, removed: 0, version: visits.version, error: null,
},
{
table_name: 'orders', queries: 22, row_estimate: { rows: null, method: 'count', at_least: 100001 }, findings: [],
skipped: [{ kind: 'relation', target: 'member_id → members.id', path: 'relations.x', reason: 'budget', detail: BUDGET_DETAIL }],
date_ranges: [], added: 0, updated: 0, removed: 0, version: 1, error: null,
},
{
table_name: 'order_items', queries: 0, row_estimate: null, findings: [],
skipped: [{ kind: 'table', target: 'order_items', path: null, reason: 'budget', detail: BUDGET_DETAIL }],
date_ranges: [], added: 0, updated: 0, removed: 0, version: 1, error: null,
},
]
return json({
profiled_at: at, actor: ACTOR, settings: { ...PROFILE_ON, sample_size: 2000, max_scan_rows: 100000, max_total_s: 120 },
queries_used: 60, stopped: 'budget', tables, total: { added: 1, updated: 2, removed: 0 }, note: null,
})(route)
}
}
const profileBox = (page) => page.locator('[data-profile-dialog]')
const profilePhase = (page) => profileBox(page).getAttribute('data-profile-dialog').catch(() => null)
/** 报告里不该露的机读码:停止原因、跳过原因、状态、基数的英文值 */
const PROFILE_CODE_RE = /\b(?:budget|deadline|failed|masked|too_large|high_cardinality|no_data|unsupported|verified|proposed|many_to_one|row_estimate)\b/
await section('数据目录 · 数据剖析设置', async () => {
const state = freshState()
const replies = { patch: [] }
const { page, sent, errors, natives, close } = await open('/data/databases', { handlers: catalogHandlers(state, replies) })
// 手工登记的库:编辑框里有「数据剖析」一段,默认关闭,预算收着
await page.locator('[data-source="zzcatscenic"]').getByRole('button', { name: '编辑', exact: true }).click()
const box = page.locator('[data-profile-settings]')
await box.waitFor()
check('编辑框里有「数据剖析」,默认关闭、预算收着', (await box.getAttribute('data-profile-settings')) === 'off'
&& await box.locator('[data-profile-field]').count() === 0 && (await box.innerText()).includes('默认关闭'))
await box.locator('[data-profile-enabled]').check()
const labels = await box.locator('label.label').allInnerTexts()
check('开启后五项预算展开,中文标签', labels.join('、') === '查询次数上限、单条查询时限、抽样键值数、整表统计行数上限、总时长上限', labels.join('、'))
const units = await box.locator('[data-profile-field] span').allInnerTexts()
check('……每项带单位', units.join(',') === '条,秒,个,行,秒', units.join(','))
const hints = await box.locator('[id$="-hint"]').allInnerTexts()
check('……写明范围和默认值,整表统计写明 0 的含义', hints.some((h) => h.startsWith('范围 1–500 条,默认 60'))
&& hints.some((h) => h.includes('范围 0–10,000,000 行') && h.includes('填 0 表示一律不做整表统计')), hints.join(' | '))
const risk = () => box.locator('[data-profile-risk]').innerText().catch(() => '')
check('……风险说明:会发出只读查询,每次最多 60 条、单条 10 秒(缺省)', (await risk()).includes('只读查询:最多 60 条,单条不超过 10 秒'), await risk())
await page.locator('#ds-profile-max_queries').fill('80')
check('……风险说明跟着填的数走', (await risk()).includes('最多 80 条'), await risk())
await page.locator('#ds-profile-max_queries').fill('0')
const err = await page.locator('#ds-profile-max_queries-error').innerText().catch(() => '')
check('填错(0 条):就地写明范围,保存按钮禁用', err === '请填写 1 到 500 之间的整数' && await page.locator('[data-source-save]').isDisabled(), err)
check('……输入框标出错(aria-invalid)', (await page.locator('#ds-profile-max_queries').getAttribute('aria-invalid')) === 'true')
await page.locator('#ds-profile-max_queries').fill('80')
await page.locator('#ds-profile-query_timeout_s').fill('20')
// 服务端按字段拒收(422):报错落在那一项下面,光标放进去,不只弹提示
const REJECT = '数据剖析的「单条查询时限(秒)」需要填写 1 到 60 之间的数;当前为「20」'
replies.patch.push(json({ detail: REJECT }, 422))
await page.locator('[data-source-save]').click()
check('保存被拒(422):服务端原话写在「单条查询时限」下面', await until(async () =>
(await page.locator('#ds-profile-query_timeout_s-error').innerText().catch(() => '')) === REJECT),
await page.locator('#ds-profile-query_timeout_s-error').innerText().catch(() => ''))
check('……光标落在那一项上,框标出错', await until(async () => page.evaluate(() => document.activeElement?.id === 'ds-profile-query_timeout_s'))
&& (await page.locator('#ds-profile-query_timeout_s').getAttribute('aria-invalid')) === 'true')
check('……没有落到「高级连接参数」的查询时限上', await page.locator('#ds-adv-query_timeout_s-error').count() === 0)
if (SHOTS) await sleep(350)
await shot(page, 'catalog-profile-settings-422')
await page.locator('#ds-profile-query_timeout_s').fill('15')
check('改了这一项,报错收起', await page.locator('#ds-profile-query_timeout_s-error').count() === 0)
await page.locator('[data-source-save]').click()
await until(async () => (await page.locator('[data-profile-settings]').count()) === 0)
const saved = writes(sent, /^PATCH /).at(-1)?.body
check('保存:options 里写剖析设置(开关和填了的项,数字)', JSON.stringify(saved?.options?.catalog_profile)
=== JSON.stringify({ enabled: true, max_queries: 80, query_timeout_s: 15 }), JSON.stringify(saved?.options))
check('……卡片上标「数据剖析已开启」,悬停写明预算', await until(async () => (await page.locator('[data-source="zzcatscenic"] [data-profile-on]').count()) === 1)
&& (await page.locator('[data-source="zzcatscenic"] [data-profile-on]').getAttribute('title')) === '数据剖析已开启:每次最多 80 条只读查询,单条不超过 15 秒')
// 再打开:剖析设置回显,没有摊进「高级连接参数」变成一串文字
await page.locator('[data-source="zzcatscenic"]').getByRole('button', { name: '编辑', exact: true }).click()
await page.locator('[data-profile-settings="on"]').waitFor()
check('再打开编辑框:开关和填的数回显', await page.locator('#ds-profile-max_queries').inputValue() === '80'
&& await page.locator('#ds-profile-query_timeout_s').inputValue() === '15')
const advanced = await page.locator('[data-profile-settings] ~ details, details').allInnerTexts()
check('……剖析设置没有摊进「高级连接参数」', !advanced.join(' ').includes('catalog_profile') && !advanced.join(' ').includes('[object Object]'))
await page.getByRole('button', { name: '取消', exact: true }).click()
// 导入表格的源没有编辑框:卡片上单独设置,options 的其余键原样带上
withProfile(state, S2, undefined)
state.sources = state.sources.map((s) => (s.id === S2 ? { ...s, options: { mask_columns: ['联系电话'] } } : s))
await goto(page, '/data/tables')
await page.locator('[data-source="zzcatupload"] [data-edit-profile]').click()
const dialog = page.locator('[data-profile-settings-dialog]')
await dialog.waitFor()
check('导入表格的源:卡片上「设置数据剖析」打开只改这一项的弹窗', (await page.locator('[role="dialog"]').last().innerText()).includes('「zzcatupload」的数据剖析设置'))
check('……没改时保存禁用', await page.locator('[data-profile-save]').isDisabled())
await dialog.locator('[data-profile-enabled]').check()
await page.locator('[data-profile-save]').click()
await until(async () => (await dialog.count()) === 0)
const body = writes(sent, /^PATCH \/datasources\/check-cat-upload$/).at(-1)?.body
check('……保存只换剖析设置,遮罩列原样带上', JSON.stringify(body) === JSON.stringify({ options: { mask_columns: ['联系电话'], catalog_profile: { enabled: true } } }),
JSON.stringify(body))
check('……卡片上标「数据剖析已开启」', await until(async () => (await page.locator('[data-source="zzcatupload"] [data-profile-on]').count()) === 1))
check('写请求只有保存数据源', writes(sent, /./).every((s) => /^PATCH \/datasources\/check-cat-/.test(s.key)),
JSON.stringify(writes(sent, /./).map((s) => s.key)))
check('没有原生对话框', natives.length === 0, natives.join(' | '))
check('没有运行时报错', errors.length === 0, errors[0] ?? '')
await close()
})
await section('数据目录 · 数据剖析', async () => {
const state = freshState()
withProfile(state, S1, PROFILE_ON)
const replies = { profile: [] }
const { page, sent, errors, natives, close } = await open(`/data/catalog/${S1}`, { handlers: catalogHandlers(state, replies) })
await page.locator('[data-catalog-row]').first().waitFor()
const head = page.locator('[data-catalog-profile]')
check('页头有「数据剖析」', (await head.getAttribute('aria-label')) === '数据剖析')
await head.click()
await profileBox(page).waitFor()
const setup = await profileBox(page).innerText()
check('没选表、没筛选:范围只有「使用次数最多、有待核实关联关系的表」,写明至多 10 张',
(await profileBox(page).locator('[data-profile-scope]').evaluateAll((els) => els.map((e) => e.getAttribute('data-profile-scope')))).join(',') === 'default'
&& setup.includes('使用次数最多、有待核实关联关系的表') && setup.includes('至多 10 张'))
const budget = await profileBox(page).locator('[data-profile-budget]').innerText()
check('确认前写明预算:查询条数、单条和总时长、抽样规模、整表统计门槛,按哪个源的设置',
['最多 60 条只读查询,单条不超过 10 秒', '总时长不超过 120 秒', '抽样 2,000 个键值', '行数超过 100,000 的表不做整表统计', '按「zzcatscenic」的数据剖析设置']
.every((s) => budget.includes(s)), budget.replace(/\s+/g, ' '))
check('……写明开始后无法中途停止', setup.includes('开始后无法中途停止'))
check('……还没开始就没有写请求', writes(sent, /./).length === 0)
if (SHOTS) await sleep(350)
await shot(page, 'catalog-profile-setup')
await page.keyboard.press('Escape')
await until(async () => (await profileBox(page).count()) === 0)
// 打开入园记录的详情,选两张表剖析:完成后清单和正在看的详情都要重新载入
await openDetail(page, S1, 'visits')
const v0 = await version(page)
check('剖析前:闸机关系是「推断 · 命名推断」,没有覆盖率', (await markStatus(page, `relations.${GATE_REL}`)) === 'proposed'
&& (await page.locator(`[data-relation="${GATE_REL}"]`).innerText()).includes('—'))
await page.locator('[data-catalog-select="visits"]').click()
await page.locator('[data-catalog-select="orders"]').click()
await page.locator('[data-catalog-profile-selected]').click()
await profileBox(page).waitFor()
check('剖析所选:范围默认是「已选的 2 张表」', await profileBox(page).locator('[data-profile-scope="selected"] input').isChecked()
&& (await profileBox(page).locator('[data-profile-scope="selected"]').innerText()).includes('已选的 2 张表'))
replies.profile.push(profileReply(state, { ms: 2600 }))
const lists0 = count(sent, /^GET \/datasources\/check-cat-scenic\/catalog$/)
await page.locator('[data-profile-start]').click()
check('进行中:写「正在剖析…」和已用时长、总时长上限', await until(async () => (await profilePhase(page)) === 'running')
&& (await profileBox(page).locator('[data-profile-title]').innerText()) === '正在剖析…'
&& (await profileBox(page).locator('[data-profile-elapsed]').innerText()).includes('(总时长上限 2 分 00 秒)'))
check('……时长在走(不是一直「不到 1 秒」)', await until(async () => Number(await profileBox(page).locator('[data-profile-elapsed]').getAttribute('data-profile-elapsed')) >= 1, 2500))
check('……不给「停止」,明说无法中途停止', await profileBox(page).getByRole('button', { name: /停止|取消/ }).count() === 0
&& (await profileBox(page).locator('[data-profile-no-cancel]').innerText()).includes('无法中途停止'))
const req = writes(sent, /catalog\/profile$/).at(-1)
check('……请求带所选的两张表和本机署名', JSON.stringify([...(req?.body?.tables ?? [])].sort()) === JSON.stringify(['orders', 'visits'])
&& decodeURIComponent(req?.actor ?? '') === ACTOR, JSON.stringify(req?.body))
if (SHOTS) await sleep(350)
await shot(page, 'catalog-profile-running')
// 关掉窗口剖析照常进行:页头换成计时,完成后提示并可以打开报告
await page.locator('[data-profile-background]').click()
check('「在后台继续」:窗口关掉,页头换成「正在剖析」', await until(async () => (await profileBox(page).count()) === 0)
&& (await page.locator('[data-catalog-profile="running"]').innerText()).includes('正在剖析'))
const toastBox = page.locator('[data-toast]').filter({ hasText: '数据剖析已停止' })
check('完成:提示「数据剖析已停止:查询次数已用完」,带「查看报告」', await until(async () => (await toastBox.count()) > 0, 6000),
await page.locator('[data-toast]').allInnerTexts().then((t) => t.join(' | ')).catch(() => ''))
check('……页头回到「数据剖析」', await page.locator('[data-catalog-profile="running"]').count() === 0)
check('……写进去的表重新载入:清单重取、正在看的详情版本更新', await until(async () => count(sent, /^GET \/datasources\/check-cat-scenic\/catalog$/) > lists0)
&& await until(async () => (await version(page)) === v0 + 1))
await toastBox.getByRole('button', { name: '查看报告' }).click()
await profileBox(page).waitFor()
const report = profileBox(page).locator('[data-profile-report]')
check('报告:停止原因说中文', (await report.locator('[data-profile-title]').innerText()) === '剖析已停止:查询次数已用完')
check('……用了多少条查询、剖析几张表', (await report.locator('[data-profile-summary]').innerText()).startsWith('用了 60 / 60 条查询,剖析 3 张表'))
check('……目录的变化', (await report.locator('[data-profile-changes]').innerText()) === '新增 1 项、更新 2 项')
check('……停下之后怎么办:调高查询次数上限或缩小范围,给「修改设置」', (await report.locator('[data-profile-stopped="budget"]').innerText()).includes('调高查询次数上限')
&& await report.locator('[data-profile-stopped] [data-profile-settings-open]').count() === 1)
const visitsCard = report.locator('[data-profile-table="visits"]')
const gate = visitsCard.locator(`[data-profile-finding="relation"][data-path="relations.${GATE_REL}"]`)
check('关系核实:覆盖率、基数、抽样规模,升为已验证', (await gate.locator('[data-profile-coverage]').innerText()) === '覆盖率 98.7%'
&& (await gate.locator('[data-profile-cardinality]').innerText()) === '多对一' && (await gate.innerText()).includes('抽样 300 个键值,对上 296 个')
&& (await gate.locator('[data-profile-outcome]').innerText()) === '升为已验证' && (await gate.getAttribute('data-status')) === 'verified',
(await gate.innerText()).replace(/\s+/g, ' '))
const member = visitsCard.locator(`[data-profile-finding="relation"][data-path="relations.${MEMBER_REL}"]`)
check('……覆盖率不够的保持推断,写明原因', (await member.locator('[data-profile-outcome]').innerText()) === '保持推断:覆盖率不足,可能不是这条关系'
&& (await member.locator('[data-profile-coverage]').innerText()) === '覆盖率 62%')
const codes = visitsCard.locator('[data-profile-finding="codes"]')
check('码值候选:列、各取值的行数、几个含义待填写,给「填写含义」', (await codes.getAttribute('data-column')) === 'status'
&& (await codes.innerText()).includes('3 个取值,共 182,000 行') && (await codes.innerText()).includes('171,080 行')
&& (await codes.locator('[data-codes-pending]').innerText()) === '1 个含义待填写' && await codes.locator('[data-profile-fill="status"]').count() === 1,
(await codes.innerText()).replace(/\s+/g, ' '))
const skipped = visitsCard.locator('[data-profile-skipped]')
check('跳过的项:原因说中文,附服务端的整句', (await skipped.innerText()).includes('列已遮罩') && (await skipped.innerText()).includes('visits.member_level 在数据源设置中被遮罩,不取值')
&& (await skipped.innerText()).includes('表太大'), (await skipped.innerText()).replace(/\s+/g, ' '))
check('……行数估算写成人话:统计信息约 182,000 行、数到上限的超过 100,000 行',
(await visitsCard.locator('header').innerText()).includes('约 182,000 行') && (await report.locator('[data-profile-table="orders"] header').innerText()).includes('超过 100,000 行'))
check('……一条没查的表写明「查询次数用完」', (await report.locator('[data-profile-table="order_items"] [data-profile-skip="budget"]').innerText()).includes('查询次数用完'))
const reportText = await report.innerText()
check('报告不露机读码', !PROFILE_CODE_RE.test(reportText), reportText.match(PROFILE_CODE_RE)?.[0] ?? '')
if (SHOTS) await sleep(350)
await shot(page, 'catalog-profile-report')
await page.locator('[data-profile-close]').click()
await until(async () => (await profileBox(page).count()) === 0)
// 表详情:关系的来源写「数据剖析」,带覆盖率和基数;码值的含义空着写「含义待填写」
const gateRow = page.locator(`[data-relation="${GATE_REL}"]`)
check('表详情:闸机关系「已验证 · 数据剖析」,覆盖率 98.7%、多对一', (await markStatus(page, `relations.${GATE_REL}`)) === 'verified'
&& (await gateRow.innerText()).includes('数据剖析') && (await gateRow.innerText()).includes('98.7%') && (await gateRow.innerText()).includes('多对一'),
(await gateRow.innerText().catch(() => '')).replace(/\s+/g, ' '))
await openMark(page, `relations.${GATE_REL}`)
check('……点开来源:写明抽样规模和覆盖率的剖析说明', (await panel(page).innerText()).includes('子表抽样 300 个不同键值,父表对上 296 个'))
await page.keyboard.press('Escape')
const statusCodes = page.locator('[data-column="status"] [data-cell="codes"]')
check('……码值的含义空着写「含义待填写」,给「填写含义」', (await statusCodes.locator('[data-code="2"] [data-code-pending]').innerText()) === '含义待填写'
&& (await statusCodes.locator('[data-codes-fill]').innerText()).includes('填写含义'))
// 从报告点「填写含义」:窗口关掉,打开这张表那一列的码值弹窗,光标在空着的那一个上
await page.locator('[data-catalog-profile]').click()
await profileBox(page).waitFor()
check('页头再点:上一次的报告看过了就回到选范围', (await profilePhase(page)) === 'setup')
await page.keyboard.press('Escape')
await statusCodes.locator('[data-codes-fill]').click()
const codesBox = page.locator('[data-codes-dialog="status"]')
await codesBox.waitFor()
check('码值弹窗:每个码值一个输入框,空着的写「含义待填写」,光标在它上面', await codesBox.locator('input[data-code]').count() === 3
&& (await codesBox.locator('input[data-code="2"]').getAttribute('placeholder')) === '含义待填写'
&& await until(async () => page.evaluate(() => document.activeElement?.getAttribute('data-code') === '2')))
check('……写明剖析说明(各取值的占比)', (await codesBox.innerText()).includes('1(94%)、9(5%)、2(1%)'))
check('……没填时保存禁用', await page.locator('[data-codes-save]').isDisabled())
await codesBox.locator('input[data-code="2"]').fill('已退票')
if (SHOTS) await sleep(350)
await shot(page, 'catalog-codes-fill')
const v1 = await version(page)
await page.keyboard.press('Enter')
check('回车保存:整份提交,只换这一列码值的含义,带读到的版本', await until(async () => (await codesBox.count()) === 0)
&& JSON.stringify(lastBody(sent, /^PUT .*catalog\/visits$/)?.notes?.columns?.status?.codes?.value) === JSON.stringify({ 1: '已入园', 9: '已作废', 2: '已退票' })
&& lastBody(sent, /^PUT .*catalog\/visits$/)?.if_version === v1, JSON.stringify(lastBody(sent, /^PUT .*catalog\/visits$/)?.notes?.columns?.status))
check('……保存后不再有「含义待填写」,这一项记为人工确认', await until(async () => (await statusCodes.locator('[data-code-pending]').count()) === 0)
&& (await statusCodes.innerText()).includes('已退票') && (await markStatus(page, 'columns.status.codes')) === 'confirmed')
// 报告里的「填写含义」直达
await page.locator('[data-catalog-profile]').click()
replies.profile.push(profileReply(state))
await page.locator('[data-profile-start]').click()
await until(async () => (await profilePhase(page)) === 'done')
check('窗口开着时完成:直接看报告,不另弹提示', (await toastBox.count()) <= 1)
await openDetail(page, S1, 'orders')
await page.locator('[data-catalog-profile]').click()
await profileBox(page).waitFor()
check('换了一张表、再点页头:回到选范围(报告已看过)', (await profilePhase(page)) === 'setup')
await page.keyboard.press('Escape')
check('写请求只有剖析和整份提交', writes(sent, /./).every((s) => /catalog\/profile$|^PUT .*catalog\/visits$/.test(s.key)),
JSON.stringify(writes(sent, /./).map((s) => s.key)))
check('没有原生对话框', natives.length === 0, natives.join(' | '))
check('没有运行时报错', errors.length === 0, errors[0] ?? '')
await close()
})
await section('数据目录 · 数据剖析开始不了(409)', async () => {
const state = freshState()
const replies = { profile: [], patch: [] }
const { page, sent, errors, natives, close } = await open(`/data/catalog/${S1}`, { handlers: catalogHandlers(state, replies) })
await page.locator('[data-catalog-row]').first().waitFor()
// 数据源列表里就没开:不让开始,就地开启
await page.locator('[data-catalog-profile]').click()
await profileBox(page).waitFor()
check('没开启:写明需要先开启,「开始剖析」禁用,没有预算', (await profileBox(page).locator('[data-profile-disabled]').innerText()).includes('未开启数据剖析')
&& await page.locator('[data-profile-start]').isDisabled() && await profileBox(page).locator('[data-profile-budget]').count() === 0)
await profileBox(page).locator('[data-profile-disabled] [data-profile-enable]').click()
const settings = page.locator('[data-profile-settings-dialog]')
await settings.waitFor()
await settings.locator('[data-profile-enabled]').check()
await page.locator(`#profile-${S1}-max_queries`).fill('40')
await page.locator('[data-profile-save]').click()
check('就地开启:保存后回到剖析,预算按刚存的设置', await until(async () => (await settings.count()) === 0)
&& await until(async () => (await profileBox(page).locator('[data-profile-budget]').innerText().catch(() => '')).includes('最多 40 条只读查询'))
&& !(await page.locator('[data-profile-start]').isDisabled()))
check('……保存的是这个源的剖析设置', JSON.stringify(writes(sent, /^PATCH /).at(-1)?.body) === JSON.stringify({ options: { catalog_profile: { enabled: true, max_queries: 40 } } }),
JSON.stringify(writes(sent, /^PATCH /).at(-1)?.body))
const cases = [
['disabled', '数据源「zzcatscenic」未开启数据剖析。剖析会对业务库发查询,请先在数据源设置中开启数据剖析', '未开启数据剖析', 'data-profile-enable'],
['inactive', '数据源「zzcatscenic」已停用,无法进行数据剖析。请先在数据源设置中启用', '数据源已停用', 'data-profile-go-source'],
['noSchema', '尚未探查结构,无法进行数据剖析。请先在数据源卡片上点「探查结构」', '还没有表结构', 'data-profile-go-source'],
['busy', '数据源「zzcatscenic」正在进行数据剖析,请等待完成后再试', '这个数据源正在剖析', 'data-profile-retry'],
['tampered', '数据文件与登记的版本不一致,可能被修改过,已拒绝查询', '数据文件核对未通过', 'data-profile-go-source'],
]
for (const [kind, detail, title, action] of cases) {
replies.profile.push(json({ detail }, 409))
await page.locator('[data-profile-start]').click()
const blocked = profileBox(page).locator(`[data-profile-blocked="${kind}"]`)
const ok = await until(async () => (await blocked.count()) === 1)
const text = ok ? await blocked.innerText() : ''
check(`409(${title}):写明是哪种情况、服务端原话和下一步,给对应的按钮`, ok && text.includes(title) && text.includes(detail)
&& await blocked.locator(`[${action}]`).count() === 1, text.replace(/\s+/g, ' ').slice(0, 140))
if (kind === 'busy') {
// 稍后重试:同一个范围再发一次,这次成功
replies.profile.push(profileReply(state))
await blocked.locator('[data-profile-retry]').click()
check('……「重试」再发一次,成功后看到报告', await until(async () => (await profilePhase(page)) === 'done'))
}
if (kind === 'disabled') {
// 服务端说没开(列表里的设置过时了):同样就地开启,开了回到选范围
await blocked.locator('[data-profile-enable]').click()
await settings.waitFor()
await page.locator(`#profile-${S1}-max_queries`).fill('45')
await page.locator('[data-profile-save]').click()
check('……就地开启后回到选范围', await until(async () => (await profilePhase(page)) === 'setup'))
continue
}
// 底栏的按钮不在弹窗正文里
await page.locator(kind === 'busy' ? '[data-profile-again]' : '[data-profile-back]').click()
await until(async () => (await profilePhase(page)) === 'setup')
}
if (SHOTS) await sleep(350)
await shot(page, 'catalog-profile-blocked')
// 不是 409 的失败:照常的报错,能重试
replies.profile.push(json({ detail: '服务端处理剖析时出错,请稍后重试' }, 500))
await page.locator('[data-profile-start]').click()
check('其余失败:写「剖析未能完成」和原因,能重试', await until(async () => (await profileBox(page).locator('[data-profile-failed]').count()) === 1)
&& (await profileBox(page).locator('[data-profile-failed]').innerText()).includes('剖析未能完成'))
// 去数据源:停用、没有表结构、快照被改动时
replies.profile.push(json({ detail: cases[1][1] }, 409))
await profileBox(page).locator('[data-profile-failed]').getByRole('button', { name: /重试/ }).click()
await profileBox(page).locator('[data-profile-go-source]').click()
await page.waitForURL('**/data/databases', { timeout: 5000 }).catch(() => {})
check('「前往数据源」回到数据源卡片', new URL(page.url()).pathname === '/data/databases', page.url())
check('写请求只有剖析和保存剖析设置', writes(sent, /./).every((s) => /catalog\/profile$|^PATCH \/datasources\/check-cat-scenic$/.test(s.key)),
JSON.stringify(writes(sent, /./).map((s) => s.key)))
check('没有原生对话框', natives.length === 0, natives.join(' | '))
check('没有运行时报错', errors.length === 0, errors[0] ?? '')
await close()
})
await section('数据目录 · 窄屏', async () => {
const state = freshState()
const { page, errors, close } = await open(`/data/catalog/${S1}`, { handlers: catalogHandlers(state), viewport: { width: 390, height: 844 } })
@@ -1130,6 +1518,13 @@ await section('数据目录 · 窄屏', async () => {
await page.locator('[data-catalog-detail-back]').click()
check('……「返回表清单」回到清单', await until(async () => page.locator('[data-catalog-list]').isVisible()))
await shot(page, 'catalog-narrow')
// 数据剖析:页头只留图标,弹窗在视口里、不横向滚动
await page.locator('[data-catalog-profile]').click()
await profileBox(page).waitFor()
const dlg = await page.locator('[role="dialog"]').last().boundingBox()
const inner = await profileBox(page).evaluate((el) => el.scrollWidth - el.clientWidth)
check('……数据剖析弹窗在视口里,正文不横向滚动', !!dlg && dlg.x >= 0 && dlg.x + dlg.width <= 390 && inner <= 1, `${JSON.stringify(dlg)} ${inner}`)
await page.keyboard.press('Escape')
check('没有运行时报错', errors.length === 0, errors[0] ?? '')
await close()
})
+85
View File
@@ -714,6 +714,7 @@ await section('Copilot:建议更新数据目录(catalog_patch 操作)', as
cardinality: 'many_to_one' }) === 'visit_id → visits.id(多对一)')
check('表类型、度量类型写界面上的叫法', cp.patchValueText('kind', 'fact') === '明细表' && cp.patchValueText('columns.a.measure', 'stock') === '存量')
check('没有值写「未填写」', cp.patchValueText('grain', null) === '未填写')
check('码值含义为空(剖析的候选)写「含义待填写」', cp.patchValueText('columns.status.codes', { 1: '有效', 9: '' }) === '1=有效、9=含义待填写')
check('列名里有句点照样认', JSON.stringify(cp.patchTarget('columns.a.b.unit')) === '{"kind":"column","column":"a.b","field":"unit"}')
check('形状不对不出卡片、不出行', cp.catalogPatchesOf([
{ op: 'catalog_patch' }, { op: 'catalog_patch', source: 's', source_id: 'i', table: 't', changes: [{ path: 'nope', after: 1 }] },
@@ -1732,5 +1733,89 @@ await section('按后端原文匹配的地方:后端现在的原文、整改
sub.join(' | '))
})
await section('SQL 检查(数据目录阶段 4B):指标所依据的查询未通过检查说成一行;助手自查和交付的问题写中文规则名', async () => {
const E = (seq, type, node_id, data = {}) => ({ seq, type, node_id, ts: 1790000000 + seq, data })
// 原话和 backend/app/engine/nodes/metrics.py 的 _sql_check_reason 同一个句式。先核对后端源码还是这么说的
const metricsPy = readFileSync(`${root}backend/app/engine/nodes/metrics.py`, 'utf8')
check('后端发指标未通过 SQL 检查警告的格式还是夹具里这几种', metricsPy.includes('code="metric_sql_check"')
&& metricsPy.includes('message=f"指标「{m[\'name\']}」{m[\'sql_check_reason\']}"')
&& metricsPy.includes('"所依据的查询未通过 SQL 检查,结果不可靠"')
&& metricsPy.includes('f"所依据的查询未通过 SQL 检查({problems[0]}),结果不可靠"')
&& metricsPy.includes('f"所依据的查询有 {len(problems)} 处未通过 SQL 检查({problems[0]}等),结果不可靠"'),
'改了 metrics.py 的说法就同步改 decode.ts 的 explainLog 和这里的夹具')
const CODES = /fanout_sum|stock_summed|join_unconfirmed|ratio_aggregated|missing_valid_filter|unknown_code|wrong_date_column/
const decodeOne = (message) => {
const all = flatten(mod.decodeRun([
E(1, 'node.started', 'card', { node_type: 'metrics' }),
E(2, 'log', 'card', { level: 'warn', code: 'metric_sql_check', metric: 'aov', message }),
E(3, 'node.finished', 'card', { duration_ms: 3 }),
]))
return { step: all.find((s) => s.code === 'metric_sql_check'), node: all.find((s) => s.kind === 'node') }
}
// 问题本身带括号(列的中文名里写了单位):副标题要取到「,结果不可靠」前面最后一个右括号
const ONE = '指标「客单价」所依据的查询未通过 SQL 检查(「订单」关联「订单明细」是一对多,对「订单」的「订单金额(元)」求和会重复计算),结果不可靠'
const one = decodeOne(ONE)
check('metric_sql_check 一行:标题点名哪个指标', one.step?.title === '指标「客单价」所依据的查询未通过 SQL 检查', one.step?.title)
check('……原因放副标题(问题里的括号不截断),原话留在展开区',
one.step?.sub === '「订单」关联「订单明细」是一对多,对「订单」的「订单金额(元)」求和会重复计算' && one.step?.detail === ONE,
`${one.step?.sub} / ${one.step?.detail}`)
check('……是提醒(warn),下一步指到 SQL 和数据目录,节点本身照样成功', one.step?.level === 'warn'
&& !!one.step?.next?.includes('SQL') && !!one.step?.next?.includes('数据目录') && one.step?.fix === 'canvas'
&& one.node?.status === 'done', `${one.step?.level} ${one.step?.next}`)
const two = decodeOne('指标「客单价」所依据的查询有 2 处未通过 SQL 检查(「订单」关联「订单明细」是一对多,对「订单」的「订单金额」求和会重复计算等),结果不可靠')
check('几处问题:标题写处数,副标题是第一处', two.step?.title === '指标「客单价」所依据的查询有 2 处未通过 SQL 检查'
&& two.step?.sub === '「订单」关联「订单明细」是一对多,对「订单」的「订单金额」求和会重复计算等', `${two.step?.title} / ${two.step?.sub}`)
const bare = decodeOne('指标「客单价」所依据的查询未通过 SQL 检查,结果不可靠')
check('没有括号里的问题:只有标题,不编副标题', bare.step?.title === '指标「客单价」所依据的查询未通过 SQL 检查' && !bare.step?.sub,
`${bare.step?.title} / ${bare.step?.sub}`)
// 助手自查(check)和交付(final.issues)里的 SQL 检查问题:code 是规则编号,field 是 args.sql,另带表、列和 SQL 片段
const sqlIssue = (level, code, message, extra = {}) => ({ level, node_id: 'q', edge_id: null, field: 'args.sql', code, message,
table: 'orders', ...extra })
const ERR = sqlIssue('error', 'fanout_sum', '「订单」关联「订单明细」是一对多,对「订单」的「订单金额」求和会重复计算。请先按订单汇总明细再关联',
{ column: 'total_amount', sql_excerpt: 'SUM(o.total_amount)' })
const WARN = sqlIssue('warning', 'missing_valid_filter', '「订单」定义了有效记录条件「status <> 9」,查询中没有按它筛选')
const INFO = sqlIssue('info', 'join_unconfirmed', '「订单」与「门店」按推断的关系关联(o.store_id = s.id),这条关系尚未确认', { table: 'stores' })
const ops = [
{ op: 'add_node', node: { id: 'q', type: 'tool', data: { label: '订单汇总' } } },
{ op: 'done', explanation: '' },
{ op: 'check', status: 'repairing', round: 1, issues: [ERR] },
{ op: 'check', status: 'failed', issues: [ERR] },
{ op: 'final', graph: { nodes: [{ id: 'q' }] }, issues: [ERR, INFO, WARN,
{ level: 'warning', node_id: 'q', edge_id: null, field: null, code: 'tools_dropped', message: '工具被移除' }] },
]
const steps = mod.decodeCopilot(ops, { context: 'canvas' })
const repairing = steps.find((s) => s.title.startsWith('自查发现'))
check('自查交回去改的那一行:节点名 + 中文规则名 + 涉及的表和列,不露规则编号',
repairing?.detail === '「订单汇总」一对多关联后重复计算:「订单」关联「订单明细」是一对多,对「订单」的「订单金额」求和会重复计算。'
+ '请先按订单汇总明细再关联(涉及 orders.total_amount)', repairing?.detail)
const failedRow = steps.find((s) => s.kind === 'error')
check('没修好的那一行同样写规则名', !!failedRow?.detail?.includes('一对多关联后重复计算') && !CODES.test(failedRow.detail), failedRow?.detail)
const sqlRow = steps.find((s) => s.code === 'sql_check')
check('交付时的 SQL 检查单独一行:数提醒、提示,没修好的错误已经列过不重复', sqlRow?.title === 'SQL 检查:1 处提醒、1 处提示', sqlRow?.title)
check('……提醒在前、提示在后,每条写规则名和表', sqlRow?.detail === [
'「订单汇总」未筛选有效记录:「订单」定义了有效记录条件「status <> 9」,查询中没有按它筛选(涉及 orders)',
'「订单汇总」关联条件未经确认:「订单」与「门店」按推断的关系关联(o.store_id = s.id),这条关系尚未确认(涉及 stores)',
].join('\n'), sqlRow?.detail)
check('……有提醒时是 warn,下一步指到「参数」里的 SQL,单个节点时给打开设置', sqlRow?.level === 'warn' && !!sqlRow.next?.includes('参数')
&& sqlRow.nodeId === 'q' && sqlRow.fix === 'canvas', `${sqlRow?.level} ${sqlRow?.next} ${sqlRow?.nodeId}`)
check('整个助手流里不露规则编号', !steps.some((s) => CODES.test(`${s.title}\n${s.detail ?? ''}\n${s.sub ?? ''}`)))
const chatRow = mod.decodeCopilot(ops, { context: 'chat' }).find((s) => s.code === 'sql_check')
check('问数据页没有画布:下一步说去画布里打开这个工作流,不给「打开设置」', !!chatRow?.next?.includes('在画布中打开') && !chatRow.fix,
`${chatRow?.next} ${chatRow?.fix}`)
const out = mod.copilotOutcome(ops)
check('结局里带着不挡运行的 SQL 检查(没修好的错误在 check 里,不重复)', out.sqlChecks.length === 2
&& out.sqlChecks.map((x) => x.code).join(',') === 'join_unconfirmed,missing_valid_filter'
&& out.check?.issues.length === 1 && out.check.issues[0].code === 'fanout_sum', JSON.stringify(out.sqlChecks.map((x) => x.code)))
const asCheck = mod.issueSqlCheck(out.check.issues[0])
check('问题按 SQL 检查读得出级别、表、列、SQL 片段', asCheck?.level === 'error' && asCheck.table === 'orders' && asCheck.column === 'total_amount'
&& asCheck.sql_excerpt === 'SUM(o.total_amount)' && out.check.issues[0].field === 'args.sql', JSON.stringify(asCheck))
check('不是 SQL 检查的问题不按它读', mod.issueSqlCheck({ message: 'x', code: 'tools_dropped' }) === null)
const infoOnly = mod.decodeCopilot([{ op: 'final', graph: { nodes: [{ id: 'q' }] }, issues: [INFO] }], { context: 'canvas' })
.find((s) => s.code === 'sql_check')
check('只有提示:不用提醒色,下一步说依据尚未确认', !!infoOnly && !infoOnly.level && !!infoOnly.next?.includes('尚未确认'),
`${infoOnly?.level} ${infoOnly?.next}`)
})
console.log(failed ? `\n✗ ${failed} 项未通过` : '\n✓ 解码器全部通过')
process.exit(failed ? 1 : 0)
+52
View File
@@ -1036,6 +1036,58 @@ await section('incomplete', '截断的查询结果整组算出来的指标:标
await cut.ctx.close()
})
await section('sqlcheck', 'SQL 检查(数据目录阶段 4B):查询步骤列出级别、规则名、说明、表和列、SQL 片段;指标未通过时标出来并写明原因', async () => {
await openQ(page, '1,288')
check('没查出问题的查询步骤不出 SQL 检查', await panel(page).locator('[data-ev-sql-checks]').count() === 0)
await page.keyboard.press('Escape')
await openQ(page, '45,678.5元', { wait: '[data-ev-expression]' })
check('通过的指标不出「SQL 检查未通过」', await panel(page).locator('[data-ev-sql-check-failed], [data-ev-sql-check-reason]').count() === 0)
await page.keyboard.press('Escape')
// 接口的形状:backend/app/api/evidence.py 的查询步骤带 checks(去掉了给模型的那句),指标步骤带 sql_check_failed / sql_check_reason
const CHECKS = [
{ code: 'join_unconfirmed', level: 'info', message: '「订单」与「地区」按推断的关系关联(o.region = r.code),这条关系尚未确认。', table: 'orders' },
{ code: 'fanout_sum', level: 'error', table: 'orders', column: 'amount', sql_excerpt: 'SUM(o.amount)',
message: '「订单」关联「订单明细」是一对多,对「订单」的「订单金额」求和会重复计算。请先按订单汇总明细再关联。' },
{ code: 'missing_valid_filter', level: 'warning', message: '「订单」定义了有效记录条件「status <> 9」,查询中没有按它筛选。', table: 'orders' },
]
const REASON = '所依据的查询未通过 SQL 检查(「订单」关联「订单明细」是一对多,对「订单」的「订单金额」求和会重复计算),结果不可靠'
const bad = await probeQ((id, body) => ({ ...body, chain: (body.chain ?? []).map((st) => (st.step === 'query'
? { ...st, checks: CHECKS } : st.step === 'metric' ? { ...st, sql_check_failed: true, sql_check_reason: REASON } : st)) }))
await openQ(bad.page, '1,288', { wait: '[data-ev-sql-checks]' })
const p = bad.page.locator('[data-evidence-panel]')
const items = p.locator('[data-ev-sql-checks] [data-sql-check]')
const levels = await items.evaluateAll((els) => els.map((e) => e.getAttribute('data-level')))
check('三条都列在 SQL 下面,错误在前、提示在后', levels.join(',') === 'error,warning,info', levels.join(','))
const first = await items.first().innerText().catch(() => '')
check('一条里有级别、中文规则名、说明、涉及的表和列、SQL 片段', first.includes('错误') && first.includes('一对多关联后重复计算')
&& first.includes('会重复计算') && first.includes('orders.amount') && first.includes('SUM(o.amount)'), first.replace(/\s+/g, ' '))
check('级别四通道:图标 + 文字,三档文字各不相同', (await items.locator('[data-sql-level]').allInnerTexts()).map((t) => t.trim()).join(',')
=== '错误,提醒,提示' && await items.locator('[data-sql-level] svg').count() === 3)
const tones = await items.locator('[data-sql-level]').evaluateAll((els) => els.map((e) => getComputedStyle(e).color))
const want = [await tokenColor(bad.page, 'color', 'var(--st-failed)'), await tokenColor(bad.page, 'color', 'var(--st-waiting)'),
await tokenColor(bad.page, 'color', 'var(--accent)')]
check('级别的颜色走令牌:错误 --st-failed、提醒 --st-waiting、提示 --accent', tones.join('|') === want.join('|'), tones.join('|'))
check('悬停写明依据的确证程度', (await items.first().locator('[data-sql-level]').getAttribute('title')) === '依据已确认,结果必然有误')
check('不露规则编号', !/fanout_sum|missing_valid_filter|join_unconfirmed/.test(await p.innerText()))
await bad.page.keyboard.press('Escape')
await openQ(bad.page, '45,678.5元', { wait: '[data-ev-sql-check-failed]' })
const chip = p.locator('[data-ev-sql-check-failed]')
check('指标名旁有「SQL 检查未通过」标识,悬停看原因', (await chip.innerText().catch(() => '')) === 'SQL 检查未通过'
&& await chip.getAttribute('title') === REASON, await chip.innerText().catch(() => ''))
check('原因写在指标下面', (await p.locator('[data-ev-sql-check-reason]').innerText().catch(() => '')) === REASON)
check('和「结果不完整」同一种样式(提醒色的标识)', await chip.evaluate((el) => getComputedStyle(el).color)
=== await tokenColor(bad.page, 'color', 'var(--st-waiting)') && (await chip.getAttribute('class'))?.includes('chip'))
if (SHOTS) {
for (const theme of ['dark', 'light']) {
await bad.page.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme)
await bad.page.waitForTimeout(200)
await bad.page.screenshot({ path: `${SHOTS}/evidence-sqlcheck-metric-${theme}.png` })
}
}
check('没有运行时报错', bad.errors.length === 0, bad.errors[0] ?? '')
await bad.ctx.close()
})
await section('q-narrow', '查询步骤在 360px:栏内展开、底部抽屉都不横向滚动;减少动效', async () => {
const overflow = () => page.evaluate((sel) => {
const el = document.querySelector(sel)
+58
View File
@@ -148,6 +148,20 @@ const ASK = {
edges: chain('in', 'fetch', 'done'),
}
/**
* 调用工具节点里写死的 SQL 对照数据目录查出了问题(数据目录阶段 4B):门禁的问题 code 是规则编号,field 是 args.sql。
* 参数里 sql 不在第一行,定位要在 JSON 里找到这个键
*/
const SQLWF = {
nodes: [
node('in', 'input', 0, '入口', { fields: [{ name: 'week', required: true }] }),
node('sum', 'tool', 240, '订单汇总', { tool: 'db_query__shop', args: { limit: 500, timeout: 30,
sql: 'SELECT o.region, SUM(o.amount) AS amt\nFROM orders o JOIN order_items i ON i.order_id = o.id\nGROUP BY o.region' } }),
node('done', 'output', 480, '成果', { fields: [{ name: 'table', value: '{{ nodes.sum.output }}' }] }),
],
edges: chain('in', 'sum', 'done'),
}
const wf = (id, name, graph, extra = {}) => ({
id, name, description: '检查脚本伪造的工作流', graph, tags: [], version: 3, is_template: false,
status: 'draft', published_version: null, published_by: null, run_count: 0,
@@ -164,6 +178,7 @@ const FAKES = {
'pf-defs': wf('pf-defs', '__publish_check_defaults__', { ...EASY, defaults: { approval: 'always', model: 'demo-model' } }),
'pf-legacy': wf('pf-legacy', '__upgrade_check_legacy__', LEGACY, { status: 'published', published_version: 3 }),
'pf-ask': wf('pf-ask', '__upgrade_check_ask__', ASK),
'pf-sql': wf('pf-sql', '__publish_check_sql__', SQLWF),
}
const clone = (x) => JSON.parse(JSON.stringify(x))
@@ -199,6 +214,11 @@ function lint(graph, level, { followsNever = false } = {}) {
}
for (const n of graph.nodes) {
const c = n.data.config ?? {}
// 基于数据目录的 SQL 检查(governance._lint_sql):错误级在受管档是硬性问题,已发布档只提示;没有确定性的修复
if (n.type === 'tool' && /JOIN order_items/i.test(c.args?.sql ?? '')) {
issues.push({ level: hard ? 'error' : 'warning', node_id: n.id, code: 'fanout_sum', fix: null, field: 'args.sql',
message: '「订单」关联「订单明细」是一对多,对「订单」的「订单金额」求和会重复计算。请先按订单汇总明细再关联' })
}
if (followsNever && n.type === 'agent' && !c.approval && (c.tools ?? []).length) {
const id = 'governed.default_approval_never:graph'
issues.push({ level: hard ? 'error' : 'warning', node_id: n.id, code: 'governed.default_approval_never', fix: id, field: 'approval',
@@ -806,6 +826,44 @@ await section('问题面板的发布前检查:选档、同一套修法、点
check('切回「校验」照旧是原来的问题清单', await page.locator('[data-preflight="panel"]').count() === 0)
})
await section('SQL 检查(数据目录阶段 4B):发布前检查写中文规则名,点一条定位到调用工具参数里的 SQL', async () => {
const { page, state, errors } = await open({ id: 'pf-sql' })
await openDialog(page)
const row = dialog(page).locator('[data-preflight-issue="fanout_sum"]')
await row.waitFor()
const text = await row.innerText()
check('发布弹窗:SQL 检查的问题写节点名和中文规则名,不露规则编号', text.includes('「订单汇总」') && text.includes('一对多关联后重复计算:')
&& text.includes('会重复计算') && !/fanout_sum|args\.sql/.test(text), text)
await shoot(page, 'preflight-sqlcheck', dialog(page))
await row.locator('button').first().click()
const where = () => page.evaluate(() => {
const box = document.querySelector('[data-inspector-sheet] [data-field="args"] textarea')
const line = box ? box.value.slice(0, box.value.indexOf('"sql"')).split('\n').length - 1 : -1
const height = box ? parseFloat(getComputedStyle(box).lineHeight) || 16 : 16
return { sel: window.__studio.getState().selectedId, field: !!box, line, top: box?.scrollTop ?? -1, visible: box ? box.scrollTop <= line * height : false }
})
check('点一条:弹窗关掉,选中节点并翻到「参数」,框内滚到 sql 那一行', await waitFor(page, async () => {
const w = await where()
return w.sel === 'sum' && w.field && w.line > 0 && w.visible
}) && await dialog(page).count() === 0, JSON.stringify(await where()))
// 问题面板的发布前检查:同一个落点
await page.evaluate(() => window.__studio.getState().select(null))
await page.locator('.react-flow__pane').click({ position: { x: 30, y: 30 } })
await page.keyboard.press('Alt+KeyP')
await page.locator('#dock-problems').waitFor()
await page.locator('[data-problems-mode="publish"]').click()
const panelRow = page.locator('[data-preflight="panel"] [data-preflight-issue="fanout_sum"]')
await panelRow.waitFor()
check('问题面板的发布前检查:同样写中文规则名', (await panelRow.innerText()).includes('一对多关联后重复计算:'))
await panelRow.locator('button').first().click()
check('……点一条同样落到「参数」里的 SQL', await waitFor(page, async () => {
const w = await where()
return w.sel === 'sum' && w.field && w.visible
}), JSON.stringify(await where()))
check('没有发布、没有保存', state.publishes.length === 0 && state.patches.length === 0)
check('没有运行时报错', errors.length === 0, errors[0] ?? '')
})
await section('同一个修复只画一次:validate 和门禁各报一条、几条问题共用一个图级修复', async () => {
const { page, state, errors } = await open({ id: 'pf-dup' })
await openDialog(page)
+59
View File
@@ -1652,6 +1652,65 @@ await section('画布右栏:去审批、放弃、出具横幅、回执(REQ-1
await page.evaluate(() => window.__studio.getState().select(null))
await page.waitForTimeout(200)
// 对照数据目录的 SQL 检查(数据目录阶段 4B):自查没修好的错误、不挡运行的提醒各一组,写中文规则名、级别、表和列、
// SQL 片段,不露规则编号;「定位」落到调用工具参数里的 SQL,光标放在 sql 的值开头
const SQL = 'SELECT o.region, SUM(o.amount) AS amt\nFROM orders o JOIN order_items i ON i.order_id = o.id\nGROUP BY o.region'
await page.evaluate((sql) => {
const st = window.__studio.getState()
window.__studio.setState({ nodes: [...st.nodes, { id: 'fetch', type: 'card', position: { x: 620, y: 200 },
data: { nodeType: 'tool', label: '订单汇总', config: { tool: 'db_query__shop', args: { limit: 100, sql } } } }] })
}, SQL)
const sqlErr = { level: 'error', node_id: 'fetch', edge_id: null, field: 'args.sql', code: 'fanout_sum', table: 'orders', column: 'amount',
sql_excerpt: 'SUM(o.amount)', message: '「订单」关联「订单明细」是一对多,对「订单」的「订单金额」求和会重复计算。请先按订单汇总明细再关联' }
const sqlWarn = { level: 'warning', node_id: 'fetch', edge_id: null, field: 'args.sql', code: 'missing_valid_filter', table: 'orders',
message: '「订单」定义了有效记录条件「status <> 9」,查询中没有按它筛选' }
await page.evaluate(([err, warn]) => window.__studio.setState({ copilotTurns: [{ explanation: '', error: '', phase: 'done', id: 'u8',
instruction: '按地区汇总订单金额', outcome: 'applied', diff: { added: ['fetch'], changed: [], removed: [], total: 1 },
ops: [{ op: 'add_node', node: { id: 'fetch', type: 'tool', data: { label: '订单汇总' } } },
{ op: 'check', status: 'repairing', round: 1, issues: [err] },
{ op: 'check', status: 'failed', issues: [err] },
{ op: 'final', graph: { nodes: [{ id: 'in' }, { id: 'fetch' }] }, issues: [err, warn] }] }] }), [sqlErr, sqlWarn])
await page.waitForTimeout(400)
const blocking = page.locator('[data-issue-list=""] [data-issue-code="fanout_sum"]')
const blockingText = await blocking.innerText().catch(() => '')
check('自查没修好的 SQL 问题:级别、中文规则名、涉及的表和列、说明、SQL 片段', blockingText.includes('错误') && blockingText.includes('一对多关联后重复计算')
&& blockingText.includes('orders.amount') && blockingText.includes('会重复计算') && blockingText.includes('SUM(o.amount)')
&& blockingText.includes('「订单汇总」'), blockingText.replace(/\s+/g, ' '))
const notes = page.locator('[data-issue-list="sql"]')
const notesText = await notes.innerText().catch(() => '')
check('不挡运行的提醒单列一组「SQL 检查」,没修好的错误不重复', notesText.startsWith('SQL 检查') && notesText.includes('提醒')
&& notesText.includes('未筛选有效记录') && !notesText.includes('一对多关联后重复计算'), notesText.replace(/\s+/g, ' '))
const panelText = await page.locator('[data-assistant-panel]').innerText()
check('助手面板不露规则编号', !/fanout_sum|missing_valid_filter|args\.sql/.test(panelText))
const sqlRow = page.locator('[data-turn="u8"] [data-step-code="sql_check"]')
check('过程里有一行「SQL 检查:1 处提醒」', (await sqlRow.innerText().catch(() => '')).includes('SQL 检查:1 处提醒'),
await page.locator('[data-turn="u8"]').innerText().then((t) => t.replace(/\s+/g, ' ').slice(0, 240)))
if (SHOTS) {
for (const theme of ['dark', 'light']) {
await page.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme)
await page.waitForTimeout(200)
await page.locator('[data-assistant-panel]').screenshot({ path: `${SHOTS}/stream-panel-sqlcheck-${theme}.png` })
}
await page.evaluate(() => document.documentElement.setAttribute('data-theme', 'dark'))
}
await blocking.getByRole('button', { name: /定位/ }).click()
await page.waitForTimeout(600)
const sqlFocus = await page.evaluate((sql) => {
const a = document.activeElement
const at = a instanceof HTMLTextAreaElement ? a.selectionStart : -1
return { sel: window.__studio.getState().selectedId, tag: a?.tagName ?? null,
field: a?.closest('[data-inspector-sheet] [data-field]')?.getAttribute('data-field') ?? null,
atSql: at >= 0 && (a.value.slice(at).startsWith(sql.split('\n')[0]) || a.value.slice(at).startsWith(JSON.stringify(sql).slice(1, 30))) }
}, SQL)
check('定位 SQL 检查的问题:打开调用工具的设置,光标落在「参数」里 sql 的值开头', sqlFocus.sel === 'fetch' && sqlFocus.field === 'args'
&& sqlFocus.tag === 'TEXTAREA' && sqlFocus.atSql, JSON.stringify(sqlFocus))
await page.evaluate(() => {
const st = window.__studio.getState()
st.select(null)
window.__studio.setState({ copilotTurns: [], nodes: st.nodes.filter((n) => n.id !== 'fetch') })
})
await page.waitForTimeout(200)
// 模型把工具调用写成文字、判失败:报错和提醒行给「打开『数据查询』的设置」,点了就是那个节点的属性面板
const MARKUP_ERROR = '模型以文本形式输出了工具调用的原始标记,未实际调用工具,这一步没有查询到任何数据。常见原因:节点未绑定工具,或模型、服务不支持工具调用。请在画布中为该节点绑定所需工具;如已绑定仍出现此问题,请换用支持工具调用的模型'
await page.evaluate((run) => {