数据源设置里加数据剖析:开关和五项预算带单位与范围,写明会发出的只读查询上限,保存被拒时报错落到对应的项上;导入表格的源在卡片上单独设置

This commit is contained in:
ThearchyHelios
2026-10-03 11:34:46 +08:00
parent c46ff74b8c
commit 5b83ef4ba8
5 changed files with 566 additions and 14 deletions
+63 -1
View File
@@ -43,7 +43,8 @@
*/
import type {
CatalogCardinality, CatalogMeasure, CatalogSource, CatalogStatus, CatalogTableKind, NodeType, SnapshotReasonCode, ToolTrust,
CatalogCardinality, CatalogMeasure, CatalogProfileNumberKey, CatalogSource, CatalogStatus, CatalogTableKind, NodeType, SnapshotReasonCode,
ToolTrust,
} from '../types'
import { formatNumber } from './format'
@@ -2148,3 +2149,64 @@ export const CATALOG_TEXT = {
reloadDiscardTitle: '放弃未保存的修改并重新载入?',
reloadDiscardAction: '放弃修改并重新载入',
}
// ===========================================================================
// 数据剖析(数据源设置里的开关和预算;数据目录页的「数据剖析」)
//
// 术语:
// - 「数据剖析」:对业务库发少量只读查询,核实推断的关联关系(覆盖率、基数),取状态类列的码值候选,提议业务日期。
// 动作写「剖析」,不写扫描、探测。默认关闭,按数据源开启。
// - 预算五项:查询次数上限、单条查询时限、抽样键值数、整表统计行数上限、总时长上限。标签和服务端校验报错里的叫法
// 一致(catalog_profile._NUMBER_FIELDS),保存被拒时按它认出是哪一项。
// ===========================================================================
export const PROFILE_FIELD_LABEL: Record<'enabled' | CatalogProfileNumberKey, string> = {
enabled: '开启数据剖析',
max_queries: '查询次数上限',
query_timeout_s: '单条查询时限',
sample_size: '抽样键值数',
max_scan_rows: '整表统计行数上限',
max_total_s: '总时长上限',
}
export const PROFILE_FIELD_UNIT: Record<CatalogProfileNumberKey, string> = {
max_queries: '条',
query_timeout_s: '秒',
sample_size: '个',
max_scan_rows: '行',
max_total_s: '秒',
}
export const PROFILE_FIELD_HINT: Record<CatalogProfileNumberKey, string> = {
max_queries: '一次剖析最多发出的查询条数,用完即停止',
query_timeout_s: '超过即中断这条查询;不超过数据源自身的查询时限',
sample_size: '核对一条关联关系时,从本表抽取的不同键值个数',
max_scan_rows: '不超过此行数的表才做去重计数、最小值和最大值等整表统计',
max_total_s: '一次剖析的总时长,到时即停止',
}
export const PROFILE_TEXT = {
// ---- 设置
section: '数据剖析',
enableHint: '默认关闭。开启后可在数据目录中核实推断的关联关系、取状态类列的码值候选',
risk: (queries: number, timeout: number, total: number) =>
`开启后,每次剖析会对这个数据源发出只读查询:最多 ${formatNumber(queries)} 条,单条不超过 ${formatNumber(timeout)} 秒,`
+ `总时长不超过 ${formatNumber(total)} 秒。不读取明细行,遮罩的列不取样。`,
range: (min: number, max: number, unit: string, def: number) =>
`范围 ${formatNumber(min)}–${formatNumber(max)} ${unit},默认 ${formatNumber(def)}`,
scanZero: '填 0 表示一律不做整表统计',
placeholder: (def: number) => `默认 ${formatNumber(def)}`,
rangeError: (min: number, max: number, integer: boolean) =>
`请填写 ${formatNumber(min)} 到 ${formatNumber(max)} 之间的${integer ? '整数' : '数'}`,
invalid: (n: number) => `数据剖析有 ${formatNumber(n)} 项设置不正确`,
// ---- 数据源卡片
chip: '数据剖析已开启',
chipHint: (queries: number, timeout: number) =>
`数据剖析已开启:每次最多 ${formatNumber(queries)} 条只读查询,单条不超过 ${formatNumber(timeout)} 秒`,
open: '设置数据剖析',
dialogTitle: (name: string) => `「${name}」的数据剖析设置`,
dialogBody: '设置保存在数据源上,对数据目录中的每次剖析生效。',
savedOn: (name: string) => `「${name}」已开启数据剖析`,
savedOff: (name: string) => `「${name}」已关闭数据剖析`,
saved: (name: string) => `已保存「${name}」的数据剖析设置`,
}
+65 -13
View File
@@ -2,7 +2,7 @@ import { useEffect, useId, useMemo, useRef, useState } from 'react'
import { useNavigate } from 'react-router-dom'
import {
AlertTriangle, BookMarked, ChevronRight, Database, EyeOff, FileSpreadsheet, History, Info, KeyRound, Layers, Lock, Plug, Plus,
RefreshCw, Search, Table2, Upload, X,
RefreshCw, ScanSearch, Search, Table2, Upload, X,
} from 'lucide-react'
import clsx from 'clsx'
import { ApiError, api, uploadDecision } from '../api/client'
@@ -21,9 +21,16 @@ import { checkHealth, forgetHealth, healthFromServer, setHealth, useHealth } fro
import type { HealthRecord } from '../lib/health'
import { workflowList, workflowsMentioning } from '../lib/mentions'
import {
CATALOG_TEXT, RAW_STATE_LABEL, RECIPE_ORIGIN_LABEL, RECIPE_TEXT, UPLOAD_CONVERSION_LABEL, UPLOAD_SHAPE_LABEL, UPLOAD_TEXT, VERSIONS_TEXT,
CATALOG_TEXT, PROFILE_TEXT, RAW_STATE_LABEL, RECIPE_ORIGIN_LABEL, RECIPE_TEXT, UPLOAD_CONVERSION_LABEL, UPLOAD_SHAPE_LABEL, UPLOAD_TEXT,
VERSIONS_TEXT,
} from '../lib/terms'
import { useRunClock } from '../run/useRunClock'
import { focusProfileField, ProfileSettingsDialog, ProfileSettingsSection } from './catalog/ProfileSettings'
import type { ProfileErrorKey } from './catalog/ProfileSettings'
import {
hasProfileOption, PROFILE_OPTION, profileFieldOfRejection, profileFormOf, profileOptionOf, profileProblems, profileSettingsOf,
} from './catalog/profile'
import type { ProfileForm } from './catalog/profile'
import { ImportWizard } from './import/ImportWizard'
import type { WizardEntry } from './import/ImportWizard'
import { VersionsDialog } from './import/VersionsDialog'
@@ -285,9 +292,11 @@ function SourceCard({ row, meta, onChange, onRemoved, onEdit, onReupload, onReci
// 传上来的表通常只有一两张,直接摊开;库动辄几十上百个对象,默认收着
const [open, setOpen] = useState(() => uploaded && row.table_count > 0 && row.table_count <= 3)
const [masking, setMasking] = useState(false)
const [profiling, setProfiling] = useState(false)
/** 版本页(当前版本的各期、历史版本、导入记录) */
const [versions, setVersions] = useState(false)
const masks = maskList(row.options?.[MASK_KEY])
const profile = profileSettingsOf(row.options)
const clock = useRunClock(!!busy)
const busySince = useRef(0)
useTicker(60_000)
@@ -525,12 +534,28 @@ function SourceCard({ row, meta, onChange, onRemoved, onEdit, onReupload, onReci
{masks.length ? '修改遮罩列' : '设置遮罩列'}
</button>
)}
{profile.enabled && (
<span className="chip" data-profile-on="" title={PROFILE_TEXT.chipHint(profile.max_queries, profile.query_timeout_s)}>
<ScanSearch size={10} aria-hidden /> {PROFILE_TEXT.chip}
</span>
)}
{uploaded && (
// 数据剖析的设置同理:手工登记的库在编辑框里改
<button type="button" className="text-2xs underline decoration-dotted underline-offset-2 hover:text-dim"
onClick={() => setProfiling(true)} data-edit-profile="">
{PROFILE_TEXT.open}
</button>
)}
{!!row.tools?.length && <span className="mono">{row.tools.join(' · ')}</span>}
</div>
{masking && (
<MaskColumnsDialog row={row} onClose={() => setMasking(false)}
onSaved={(next) => { setMasking(false); onChange(next); syncCatalog() }} />
)}
{profiling && (
<ProfileSettingsDialog row={row} onClose={() => setProfiling(false)}
onSaved={(next) => { setProfiling(false); onChange(next); syncCatalog() }} />
)}
{versions && (
// onChange 是父级的「更新这一行并刷新全局目录」:回滚之后表结构可能变了,检查器、问数据读的目录要跟上
<VersionsDialog source={row} onClose={() => setVersions(false)} onChange={onChange} onReload={onReload} />
@@ -940,11 +965,16 @@ interface SourceForm {
sid: string
/** options.mask_columns:证据面板展示查询原始行时遮掉的列 */
maskColumns: string[]
/** options 里除 schema / service_name / sid / mask_columns 之外的键,全部摊开可编辑 */
/** options.catalog_profile:数据剖析的开关和预算 */
profile: ProfileForm
/** 打开时 options 里已经有剖析设置:没开、也没填时也要照写(关掉之后写回 enabled: false) */
profileStored: boolean
/** options 里除 schema / service_name / sid / mask_columns / catalog_profile 之外的键,全部摊开可编辑 */
advanced: [string, string][]
}
const OWN_OPTION_KEYS = new Set(['schema', 'service_name', 'sid', MASK_KEY])
// 剖析设置是一个对象,不能摊进「高级连接参数」:那里按文字回显,保存时会写成「[object Object]」被服务端拒收
const OWN_OPTION_KEYS = new Set(['schema', 'service_name', 'sid', MASK_KEY, PROFILE_OPTION])
function toForm(source: any): SourceForm {
const o = source.options ?? {}
@@ -966,6 +996,8 @@ function toForm(source: any): SourceForm {
serviceName: o.service_name ?? (oracle ? source.database ?? '' : ''),
sid: o.sid ?? '',
maskColumns: maskList(o[MASK_KEY]),
profile: profileFormOf(o),
profileStored: hasProfileOption(o),
advanced: Object.entries(o)
.filter(([k]) => !OWN_OPTION_KEYS.has(k))
.map(([k, v]) => [k, String(v ?? '')]),
@@ -973,10 +1005,12 @@ function toForm(source: any): SourceForm {
}
function toBody(form: SourceForm, isNew: boolean, wasOracle = true): any {
const options: Record<string, string | string[]> = {}
const options: Record<string, unknown> = {}
for (const [k, v] of form.advanced) if (k.trim() && v.trim()) options[k.trim()] = v.trim()
if (form.schema.trim()) options.schema = form.schema.trim()
if (form.maskColumns.length) options[MASK_KEY] = form.maskColumns
const profile = profileOptionOf(form.profile, form.profileStored)
if (profile) options[PROFILE_OPTION] = profile
const oracle = form.kind === 'oracle'
if (oracle) {
// 只写选中的那一个,另一个不带:两处都有值时引擎听 service_name,
@@ -1005,10 +1039,12 @@ function toBody(form: SourceForm, isNew: boolean, wasOracle = true): any {
}
/**
* 422 的中文 detail 说的是哪一项。眼下只有查询时限按字段拒(后端 query_timeout_problem),
* 认不出的交回 toast
* 422 的中文 detail 说的是哪一项:查询时限(后端 query_timeout_problem);数据剖析的某一项(profile_settings_problem,
* 记为 profile.<键>,它的「单条查询时限」也带「查询时限」四个字,要先认)。认不出的交回 toast
*/
function fieldOfRejection(message: string): string | null {
const profile = profileFieldOfRejection(message)
if (profile) return `profile.${profile}`
if (/查询时限/.test(message)) return 'query_timeout_s'
return null
}
@@ -1019,10 +1055,11 @@ const FIELD_LABEL: Record<string, string> = {
/**
* 决定连到哪个库的那几项(提交体里的)。schema 不在内:它只影响探查哪一片,不影响
* 连不连得上,后端判断测连接结果还作不作数时同样不看它;遮罩列只管面板上怎么显示,也不在内
* 连不连得上,后端判断测连接结果还作不作数时同样不看它;遮罩列只管面板上怎么显示,数据剖析的设置只管剖析,
* 都不在内(服务端拼连接串时同样摘掉这几项)
*/
function connectionOf(body: any): string {
const { schema: _schema, [MASK_KEY]: _mask, ...options } = body.options ?? {}
const { schema: _schema, [MASK_KEY]: _mask, [PROFILE_OPTION]: _profile, ...options } = body.options ?? {}
return JSON.stringify([
body.kind, body.host ?? null, body.port ?? null, body.database ?? null, body.username ?? null,
options, !!body.password,
@@ -1076,6 +1113,12 @@ function SourceEditor({ source, kinds, onClose, onSaved }: {
}
const required = (key: string) => needs.includes(key)
const blocked = missing.length > 0 || !!nameError
// 剖析设置填错只拦保存,不拦测试连接(和连接无关)
const profileBad = Object.keys(profileProblems(form.profile)).length
const saveBlocked = blocked || profileBad > 0
const saveHint = blocked ? `缺少必填项:${missing.join('、') || '标识格式'}` : profileBad ? PROFILE_TEXT.invalid(profileBad) : undefined
const profileError = fieldError?.key.startsWith('profile.')
? { key: fieldError.key.slice('profile.'.length) as ProfileErrorKey, message: fieldError.message } : null
const body = toBody(form, isNew, source.kind === 'oracle')
const sig = JSON.stringify({ ...body, password: form.password })
const dirty = JSON.stringify(form) !== JSON.stringify(initial)
@@ -1097,7 +1140,7 @@ function SourceEditor({ source, kinds, onClose, onSaved }: {
}
const save = async () => {
if (blocked) return
if (saveBlocked) return
setSaving(true)
setFieldError(null)
try {
@@ -1109,7 +1152,10 @@ function SourceEditor({ source, kinds, onClose, onSaved }: {
} catch (e) {
const rejected = e instanceof ApiError && e.status === 422 ? e.message : ''
const key = rejected ? fieldOfRejection(rejected) : null
if (key) {
if (key?.startsWith('profile.')) {
setFieldError({ key, message: rejected })
focusProfileField('ds-profile', key.slice('profile.'.length) as ProfileErrorKey)
} else if (key) {
setFieldError({ key, message: rejected })
setAdvOpen(true)
requestAnimationFrame(() => document.getElementById(`ds-adv-${key}`)?.focus())
@@ -1200,8 +1246,8 @@ function SourceEditor({ source, kinds, onClose, onSaved }: {
title={blocked ? `缺少必填项:${missing.join('、') || '标识格式'}` : '使用当前配置测试连接,不保存'}>
{test.since ? <Spinner size={11} /> : <Plug size={12} aria-hidden />} 测试连接
</button>
<button className="btn btn-primary" disabled={saving || blocked} onClick={() => void save()}
title={blocked ? `缺少必填项:${missing.join('、') || '标识格式'}` : undefined}>
<button className="btn btn-primary" disabled={saving || saveBlocked} onClick={() => void save()}
title={saveHint} data-source-save="">
{saving ? <Spinner size={11} /> : null} 保存
</button>
</>
@@ -1318,6 +1364,12 @@ function SourceEditor({ source, kinds, onClose, onSaved }: {
<MaskColumnsField sourceId={source.id} synced={!!source.table_count} value={form.maskColumns}
onChange={(maskColumns) => set({ maskColumns })} />
<ProfileSettingsSection idPrefix="ds-profile" form={form.profile} serverError={profileError}
onChange={(profile) => {
if (profileError) setFieldError(null)
set({ profile })
}} />
<label className="flex cursor-pointer items-start gap-2 rounded-lg border p-2.5"
style={form.readonly ? undefined : { borderColor: 'var(--warn)', background: 'color-mix(in srgb, var(--warn) 7%, transparent)' }}>
<input type="checkbox" className="mt-0.5" checked={form.readonly}
@@ -0,0 +1,162 @@
import { useState } from 'react'
import { ShieldAlert } from 'lucide-react'
import { ApiError, api } from '../../api/client'
import type { CatalogProfileNumberKey } from '../../types'
import { Field, Modal, Spinner, toast } from '../../components/ui'
import { PROFILE_FIELD_HINT, PROFILE_FIELD_LABEL, PROFILE_FIELD_UNIT, PROFILE_TEXT as PT } from '../../lib/terms'
import {
PROFILE_DEFAULTS, PROFILE_LIMITS, PROFILE_NUMBER_KEYS, hasProfileOption, profileFieldOfRejection, profileFormOf, profileOptionOf,
profileProblems, profileSettingsOfForm, sameProfileForm, withProfileOption,
} from './profile'
import type { ProfileForm } from './profile'
// ===========================================================================
// 数据剖析的设置:开关、五项预算、风险说明。数据源编辑框里是其中一段;导入表格的源没有编辑框,数据目录页在
// 剖析被拒(未开启)时也要就地打开它,所以另有一个只改这一项的弹窗(options 的其余键原样带上)。
// 剖析会对业务库发查询,开关默认关着;开启后五项预算才展开,风险说明写的是此刻填的数。
// ===========================================================================
/** 保存被拒时的落点:某一项,或者整段(认不出是哪一项) */
export type ProfileErrorKey = 'enabled' | CatalogProfileNumberKey | 'section'
export function ProfileSettingsSection({ idPrefix, form, onChange, serverError }: {
/** 输入框 id 的前缀:报错时按 `${idPrefix}-${键}` 聚焦 */
idPrefix: string
form: ProfileForm
onChange: (next: ProfileForm) => void
/** 服务端拒收(422)的那一句,落在对应的项上 */
serverError?: { key: ProfileErrorKey; message: string } | null
}) {
const problems = profileProblems(form)
const errorOf = (k: CatalogProfileNumberKey) => (serverError?.key === k ? serverError.message : problems[k])
// 开关关着时预算收起;但填错的项(或服务端点了名的项)要看得见,不然保存被拦下却找不到原因
const open = form.enabled || Object.keys(problems).length > 0
|| (!!serverError && serverError.key !== 'enabled' && serverError.key !== 'section')
const s = profileSettingsOfForm(form)
const setValue = (k: CatalogProfileNumberKey, v: string) => onChange({ ...form, values: { ...form.values, [k]: v } })
const switchError = serverError && (serverError.key === 'enabled' || serverError.key === 'section') ? serverError.message : null
return (
<fieldset className="rounded-lg border" data-field="catalog_profile" data-profile-settings={form.enabled ? 'on' : 'off'}>
<legend className="sr-only">{PT.section}</legend>
<label className="flex cursor-pointer items-start gap-2 p-2.5">
<input id={`${idPrefix}-enabled`} type="checkbox" className="mt-0.5" checked={form.enabled}
aria-describedby={`${idPrefix}-enabled-hint`} aria-invalid={switchError ? true : undefined}
onChange={(e) => onChange({ ...form, enabled: e.target.checked })} data-profile-enabled="" />
<span className="min-w-0 text-xs">
{PT.section}
<span id={`${idPrefix}-enabled-hint`} className="ml-1.5 text-2xs leading-relaxed text-faint">{PT.enableHint}</span>
</span>
</label>
{switchError && (
<p className="px-2.5 pb-2 text-2xs leading-relaxed text-[var(--err)]" role="alert" data-profile-error="section">{switchError}</p>
)}
{open && (
<div className="space-y-3 border-t px-2.5 py-2.5">
{form.enabled && (
<p className="flex items-start gap-1.5 rounded-md px-2 py-1.5 text-2xs leading-relaxed text-dim" data-profile-risk=""
style={{ background: 'var(--st-waiting-soft)' }}>
<ShieldAlert size={12} className="mt-px shrink-0" style={{ color: 'var(--st-waiting)' }} aria-hidden />
<span>{PT.risk(s.max_queries, s.query_timeout_s, s.max_total_s)}</span>
</p>
)}
<div className="grid gap-3 sm:grid-cols-2">
{PROFILE_NUMBER_KEYS.map((k) => {
const { min, max } = PROFILE_LIMITS[k]
const unit = PROFILE_FIELD_UNIT[k]
const err = errorOf(k)
const id = `${idPrefix}-${k}`
const hint = `${PT.range(min, max, unit, PROFILE_DEFAULTS[k])}${k === 'max_scan_rows' ? `,${PT.scanZero}` : ''}。${PROFILE_FIELD_HINT[k]}`
return (
<Field key={k} htmlFor={id} label={PROFILE_FIELD_LABEL[k]} hint={hint} error={err}>
{(p) => (
<div className="relative" data-profile-field={k}>
<input {...p} className="field tnum pr-8" inputMode={PROFILE_LIMITS[k].integer ? 'numeric' : 'decimal'}
autoComplete="off" spellCheck={false} value={form.values[k]}
placeholder={PT.placeholder(PROFILE_DEFAULTS[k])}
style={err ? { borderColor: 'var(--err)' } : undefined}
onChange={(e) => setValue(k, e.target.value)} />
<span className="pointer-events-none absolute right-2.5 top-1/2 -translate-y-1/2 text-2xs text-faint" aria-hidden>
{unit}
</span>
</div>
)}
</Field>
)
})}
</div>
</div>
)}
</fieldset>
)
}
/** 聚焦保存被拒的那一项(预算收着时先等它展开) */
export function focusProfileField(idPrefix: string, key: ProfileErrorKey) {
const id = key === 'section' ? `${idPrefix}-enabled` : `${idPrefix}-${key}`
requestAnimationFrame(() => requestAnimationFrame(() => {
const el = document.getElementById(id)
el?.scrollIntoView({ block: 'nearest' })
el?.focus({ preventScroll: true })
}))
}
/**
* 只改数据剖析设置的弹窗:导入表格的源(没有编辑框)、数据目录页里剖析被拒时用。
* 保存时 options 的其余键(遮罩列、schema……)原样带上,只换剖析设置这一项
*/
export function ProfileSettingsDialog({ row, onClose, onSaved }: {
row: { id: string; name: string; options?: Record<string, unknown> | null }
onClose: () => void
onSaved: (row: any) => void
}) {
const [initial] = useState(() => profileFormOf(row.options))
const [form, setForm] = useState<ProfileForm>(initial)
const [saving, setSaving] = useState(false)
const [serverError, setServerError] = useState<{ key: ProfileErrorKey; message: string } | null>(null)
const dirty = !sameProfileForm(form, initial)
const problems = Object.keys(profileProblems(form)).length
const idPrefix = `profile-${row.id}`
const save = async () => {
if (!dirty || problems) return
setSaving(true)
setServerError(null)
try {
const options = withProfileOption(row.options, profileOptionOf(form, hasProfileOption(row.options)))
const next = await api.datasources.update(row.id, { options })
onSaved(next)
toast.ok(form.enabled === initial.enabled ? PT.saved(row.name) : form.enabled ? PT.savedOn(row.name) : PT.savedOff(row.name))
} catch (e) {
const key = e instanceof ApiError && e.status === 422 ? profileFieldOfRejection(e.message) : null
if (key) {
setServerError({ key, message: e instanceof ApiError ? e.message : '' })
focusProfileField(idPrefix, key)
} else {
toast.error(e)
}
} finally {
setSaving(false)
}
}
return (
<Modal open onClose={onClose} dirty={dirty} width={560} title={PT.dialogTitle(row.name)}
footer={(
<>
{problems > 0 && <span className="mr-auto text-2xs text-[var(--err)]" data-profile-invalid={problems}>{PT.invalid(problems)}</span>}
<button type="button" className="btn" onClick={onClose}>取消</button>
<button type="button" className="btn btn-primary" disabled={saving || !dirty || problems > 0} onClick={() => void save()}
data-profile-save="">
{saving ? <Spinner size={11} /> : null} 保存
</button>
</>
)}>
<div className="space-y-3" data-profile-dialog={row.id}>
<p className="text-2xs leading-relaxed text-faint">{PT.dialogBody}</p>
<ProfileSettingsSection idPrefix={idPrefix} form={form} serverError={serverError}
onChange={(next) => { setForm(next); setServerError(null) }} />
</div>
</Modal>
)
}
+141
View File
@@ -0,0 +1,141 @@
import type { CatalogProfileNumberKey, CatalogProfileSettings } from '../../types'
import { PROFILE_FIELD_LABEL, PROFILE_TEXT } from '../../lib/terms'
// ===========================================================================
// 数据剖析的设置:数据源 options.catalog_profile 和表单之间的来回转换、校验、保存被拒时认出是哪一项。
// 规则跟着服务端(backend/app/data/catalog_profile.py 的 ProfileSettings、profile_settings、
// profile_settings_problem)走:缺省即关闭;读的时候宽松(某一项不对就用缺省),写的时候严格(超出范围的拒收)。
// ===========================================================================
/** 数据源 options 里剖析设置的键 */
export const PROFILE_OPTION = 'catalog_profile'
export const PROFILE_NUMBER_KEYS: readonly CatalogProfileNumberKey[] = [
'max_queries', 'query_timeout_s', 'sample_size', 'max_scan_rows', 'max_total_s',
]
/** 缺省值(服务端 ProfileSettings 的默认) */
export const PROFILE_DEFAULTS: CatalogProfileSettings = {
enabled: false, max_queries: 60, query_timeout_s: 10, sample_size: 2000, max_scan_rows: 100_000, max_total_s: 120,
}
/** 各数值项的范围(服务端 _NUMBER_FIELDS):下限、上限、是否必须是整数 */
export const PROFILE_LIMITS: Record<CatalogProfileNumberKey, { min: number; max: number; integer: boolean }> = {
max_queries: { min: 1, max: 500, integer: true },
query_timeout_s: { min: 1, max: 60, integer: false },
sample_size: { min: 10, max: 10_000, integer: true },
max_scan_rows: { min: 0, max: 10_000_000, integer: true },
max_total_s: { min: 10, max: 600, integer: false },
}
/** 一个数:数字和数字写成的文字都认(同服务端 _number)。不合规返回 null */
function numberIn(value: unknown, key: CatalogProfileNumberKey): number | null {
if (value == null || typeof value === 'boolean') return null
const text = String(value).trim().replace(/,/g, '')
if (!text) return null
const n = Number(text)
const { min, max, integer } = PROFILE_LIMITS[key]
if (!Number.isFinite(n) || n < min || n > max) return null
if (integer && !Number.isInteger(n)) return null
return n
}
const rawOf = (options: Record<string, unknown> | null | undefined): Record<string, unknown> | null => {
const raw = options?.[PROFILE_OPTION]
return raw && typeof raw === 'object' && !Array.isArray(raw) ? raw as Record<string, unknown> : null
}
/** options → 生效的设置。宽松:某一项不对用缺省;开关只认 true */
export function profileSettingsOf(options: Record<string, unknown> | null | undefined): CatalogProfileSettings {
const raw = rawOf(options)
const out: CatalogProfileSettings = { ...PROFILE_DEFAULTS, enabled: raw?.enabled === true }
if (!raw) return out
for (const key of PROFILE_NUMBER_KEYS) {
const n = numberIn(raw[key], key)
if (n != null) out[key] = n
}
return out
}
// ---------------------------------------------------------------------------
// 表单:数值项是文字(输入时允许写一半),空着表示用缺省
// ---------------------------------------------------------------------------
export interface ProfileForm {
enabled: boolean
values: Record<CatalogProfileNumberKey, string>
}
export function profileFormOf(options: Record<string, unknown> | null | undefined): ProfileForm {
const raw = rawOf(options)
const values = Object.fromEntries(PROFILE_NUMBER_KEYS.map((k) => {
const v = raw?.[k]
return [k, v == null || typeof v === 'object' ? '' : String(v)]
})) as Record<CatalogProfileNumberKey, string>
return { enabled: raw?.enabled === true, values }
}
/** options 里已经有剖析设置这一项 */
export const hasProfileOption = (options: Record<string, unknown> | null | undefined): boolean => !!rawOf(options)
/**
* 表单 → 要写进 options 的设置。原来没有(stored 为假)、现在也没开、一项都没填的不写(不给 options 平白多一个键);
* 其余写开关和填了的项。填的不是数时原样交上去,由服务端拒收并说明是哪一项(表单上已经先拦过一次)
*/
export function profileOptionOf(form: ProfileForm, stored: boolean): Record<string, unknown> | undefined {
const filled = PROFILE_NUMBER_KEYS.filter((k) => form.values[k].trim())
if (!stored && !form.enabled && !filled.length) return undefined
const out: Record<string, unknown> = { enabled: form.enabled }
for (const k of filled) {
const text = form.values[k].trim().replace(/,/g, '')
const n = Number(text)
out[k] = Number.isFinite(n) ? n : form.values[k].trim()
}
return out
}
/** 表单的格式问题:键 → 说明。空对象表示可以保存 */
export function profileProblems(form: ProfileForm): Partial<Record<CatalogProfileNumberKey, string>> {
const out: Partial<Record<CatalogProfileNumberKey, string>> = {}
for (const k of PROFILE_NUMBER_KEYS) {
if (!form.values[k].trim()) continue
if (numberIn(form.values[k], k) == null) {
const { min, max, integer } = PROFILE_LIMITS[k]
out[k] = PROFILE_TEXT.rangeError(min, max, integer)
}
}
return out
}
/** 表单此刻对应的设置:没填、填错的项按缺省(风险说明据此写出「最多 N 条」) */
export function profileSettingsOfForm(form: ProfileForm): CatalogProfileSettings {
const out: CatalogProfileSettings = { ...PROFILE_DEFAULTS, enabled: form.enabled }
for (const k of PROFILE_NUMBER_KEYS) {
const n = numberIn(form.values[k], k)
if (n != null) out[k] = n
}
return out
}
export const sameProfileForm = (a: ProfileForm, b: ProfileForm): boolean =>
a.enabled === b.enabled && PROFILE_NUMBER_KEYS.every((k) => a.values[k] === b.values[k])
/**
* 服务端保存时拒收的 422(一句中文)说的是剖析设置的哪一项:
* 「数据剖析的「查询次数上限」需要填写 1 到 500 之间的整数;当前为「0」」→ max_queries。
* 说的是剖析设置、但认不出哪一项(格式不对、有认不出的项)返回 'section';说的不是剖析设置返回 null
*/
export function profileFieldOfRejection(message: string): 'enabled' | CatalogProfileNumberKey | 'section' | null {
if (!message.includes('数据剖析')) return null
const named = message.match(/「([^」]+)」/)?.[1] ?? ''
for (const k of ['enabled', ...PROFILE_NUMBER_KEYS] as const) {
if (named && named.startsWith(PROFILE_FIELD_LABEL[k])) return k
}
return 'section'
}
/** 设置里有没有剖析设置这一项(options 的其余键原样带上) */
export function withProfileOption(options: Record<string, unknown> | null | undefined, value: Record<string, unknown> | undefined): Record<string, unknown> {
const { [PROFILE_OPTION]: _old, ...rest } = options ?? {}
return value ? { ...rest, [PROFILE_OPTION]: value } : rest
}
+135
View File
@@ -2705,3 +2705,138 @@ export interface CatalogDraftOut {
/** 模型整体用不了(没有配置、已停用……):只按注释、外键和命名起草 */
model_error: string | null
}
// ===========================================================================
// 数据剖析(后端 app/data/catalog_profile.py):对业务库发少量只读查询,核实推断的关联关系、取码值候选、
// 提议业务日期。默认关闭,按数据源在 options.catalog_profile 里开启。
// ===========================================================================
/** 剖析设置里的数值项 */
export type CatalogProfileNumberKey = 'max_queries' | 'query_timeout_s' | 'sample_size' | 'max_scan_rows' | 'max_total_s'
/** 一个数据源的剖析设置(服务端 ProfileSettings.to_dict) */
export interface CatalogProfileSettings {
enabled: boolean
max_queries: number
query_timeout_s: number
sample_size: number
/** 0 表示一律不做整表统计 */
max_scan_rows: number
max_total_s: number
}
/** 整次剖析中途停下的原因:查询次数用完、总时长用完、连续多条查询失败 */
export type CatalogProfileStop = 'budget' | 'deadline' | 'failed'
/** 一张表的行数:stats 数据库的统计信息(估算);count 数到上限为止;unknown 没能得到 */
export interface CatalogProfileSize {
rows: number | null
method: 'stats' | 'count' | 'unknown' | (string & {})
/** 数到上限也没数完:至少这么多行 */
at_least: number | null
}
/** 关系的核对结论:覆盖率、基数,升为已验证(verified)或保持推断(proposed);人工确认过的只补覆盖率和基数 */
export interface CatalogProfileRelationFinding {
kind: 'relation'
path: string
target: string
columns: string[]
to_table: string
to_columns: string[]
status: CatalogStatus
confirmed: boolean
coverage: number
cardinality: CatalogCardinality | null
/** 抽了几个不同的键值、在被指向表里对上几个 */
sample: number
matched: number
summary: string
}
/** 码值候选:观察到的取值和行数,含义留给人填 */
export interface CatalogProfileCodesFinding {
kind: 'codes'
path: string
column: string
values: { value: string; rows: number }[]
rows: number
status: CatalogStatus
summary: string
}
/** 业务日期提议:表里只有一个日期类列 */
export interface CatalogProfileDateFinding {
kind: 'business_date'
path: string
column: string
min: string
max: string
status: CatalogStatus
summary: string
}
export type CatalogProfileFinding = CatalogProfileRelationFinding | CatalogProfileCodesFinding | CatalogProfileDateFinding
/** 没做的一项。detail 是可以直接显示的整句 */
export interface CatalogProfileSkip {
/** relation / codes / date / row_estimate / table */
kind: string
target: string
path: string | null
/** budget / deadline / failed / timeout / error / rejected / masked / too_large / view / unsupported / no_data / missing / high_cardinality */
reason: string
detail: string
}
export interface CatalogProfileTable {
table_name: string
queries: number
row_estimate: CatalogProfileSize | null
findings: CatalogProfileFinding[]
skipped: CatalogProfileSkip[]
/** 日期类列的取值范围 */
date_ranges: { column: string; min: string; max: string }[]
added: number
updated: number
removed: number
version: number
/** 这张表没剖析(表结构里没有)或结果没写进去(写入一直冲突) */
error: string | null
}
/** POST /datasources/{id}/catalog/profile */
export interface CatalogProfileOut {
profiled_at: string
actor: string | null
settings: CatalogProfileSettings
queries_used: number
stopped: CatalogProfileStop | null
tables: CatalogProfileTable[]
total: { added: number; updated: number; removed: number }
/** 一张表都没有剖析时的说明 */
note: string | null
}
// ===========================================================================
// 基于数据目录的 SQL 检查(后端 app/data/sqlcheck.py):七条规则,编号稳定
// ===========================================================================
export type SqlCheckCode = 'fanout_sum' | 'stock_summed' | 'join_unconfirmed' | 'ratio_aggregated' | 'missing_valid_filter'
| 'unknown_code' | 'wrong_date_column'
/** error:依据有确证、结果必然有误;warning:依据有确证、很可能有误;info:依据只是推断 */
export type SqlCheckLevel = 'error' | 'warning' | 'info'
/** 一条检查结果(证据接口查询步骤里的 checks、助手自查和发布前检查的问题)。交回模型改写用的那句不上界面,这里不收 */
export interface SqlCheckItem {
code: SqlCheckCode | (string & {})
level: SqlCheckLevel | (string & {})
/** 给人看的说明 */
message: string
/** schema_cache 里的表名 */
table: string
column?: string
relation_id?: string
sql_excerpt?: string
}