mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 00:31:58 +00:00
数据目录 F2b:剖析报告和目录页读结构化字段,不再从原话里认——服务端最小改动:码值候选的结论带 pending(还有几个含义待填写),关系的结论带 cardinality_checked;报告里「N 个含义待填写」读 pending,基数写「已用数据核实」或「子表一侧未核实」读 cardinality_checked,挑表时跳过的空表读 empty_tables 列出来;表详情的关联关系标出基数已核实;剖析说明不带日期后,码值弹窗的说明后面按这一项的 updated_at 写时间;补后端测试和检查脚本的断言(假答复的 summary 不再写待填写)
This commit is contained in:
@@ -930,7 +930,8 @@ async def _check_dates(cx: _Context, table: str, columns: list[str], total: int,
|
||||
|
||||
def _relation_finding(o: _RelationOutcome, rel: dict[str, Any], *, status: str, confirmed: bool) -> dict[str, Any]:
|
||||
"""一条关系的核实结论。previous_status 是剖析之前目录里这条关系的状态:报告按它说「升为已验证」还是「仍为已验证」
|
||||
(重新剖析、目录没变时不能说升了)。"""
|
||||
(重新剖析、目录没变时不能说升了);cardinality_checked:基数是不是用数据核实的(子表一侧数过是否唯一),
|
||||
报告据此写「已用数据核实」,不从说明的原话里认。"""
|
||||
previous = rel.get("status")
|
||||
if confirmed:
|
||||
summary = "已人工确认,只补充覆盖率和基数"
|
||||
@@ -943,8 +944,8 @@ def _relation_finding(o: _RelationOutcome, rel: dict[str, Any], *, status: str,
|
||||
return {"kind": "relation", "path": f"relations.{o.rid}", "target": o.target,
|
||||
"columns": list(rel.get("columns") or []), "to_table": rel.get("to_table"),
|
||||
"to_columns": list(rel.get("to_columns") or []), "status": status, "confirmed": confirmed,
|
||||
"previous_status": previous, "coverage": o.coverage, "cardinality": o.cardinality, "sample": o.sample,
|
||||
"matched": o.matched, "summary": summary}
|
||||
"previous_status": previous, "coverage": o.coverage, "cardinality": o.cardinality,
|
||||
"cardinality_checked": o.cardinality_checked, "sample": o.sample, "matched": o.matched, "summary": summary}
|
||||
|
||||
|
||||
def _codes_value(c: _CodesOutcome, item: dict[str, Any] | None) -> dict[str, str]:
|
||||
@@ -959,11 +960,13 @@ def _codes_value(c: _CodesOutcome, item: dict[str, Any] | None) -> dict[str, str
|
||||
|
||||
|
||||
def _codes_finding(c: _CodesOutcome, value: dict[str, str]) -> dict[str, Any]:
|
||||
"""码值候选的结论。pending:写进目录的码值里还有几个含义空着(别的来源写过的含义会沿用),报告按它说
|
||||
「N 个含义待填写」,不从 summary 的原话里认。"""
|
||||
pending = sum(1 for v in value.values() if not v.strip())
|
||||
summary = f"{len(c.values)} 个取值" + (f",{pending} 个含义待填写" if pending else "")
|
||||
return {"kind": "codes", "path": f"columns.{c.column}.codes", "column": c.column,
|
||||
"values": [{"value": k, "rows": n} for k, n in c.values], "rows": c.rows, "status": "proposed",
|
||||
"summary": summary}
|
||||
"pending": pending, "summary": summary}
|
||||
|
||||
|
||||
def _patch_confirmed(rel: dict[str, Any], o: _RelationOutcome, at: str) -> bool:
|
||||
|
||||
@@ -231,6 +231,11 @@ async def test_profile_verifies_name_relations_and_proposes_codes_and_business_d
|
||||
gate = next(f for f in visits["findings"] if f["kind"] == "relation" and f["columns"] == ["gate_id"])
|
||||
assert gate["status"] == "verified" and gate["path"] == f"relations.{_relation(notes, 'gate_id')['id']}"
|
||||
assert gate["sample"] == 8 and gate["matched"] == 8 and gate["summary"]
|
||||
# 结构化的字段:基数用数据核实过没有、码值还有几个含义待填写(界面读这些,不从原话里认)
|
||||
assert isinstance(gate["cardinality_checked"], bool)
|
||||
codes = next(f for f in visits["findings"] if f["kind"] == "codes")
|
||||
blank = sum(1 for v in notes["columns"][codes["column"]]["codes"]["value"].values() if not v.strip())
|
||||
assert codes["pending"] == blank and (f"{blank} 个含义待填写" in codes["summary"]) == (blank > 0)
|
||||
assert visits["row_estimate"] == {"rows": 1500, "method": "count", "at_least": None}
|
||||
assert visits["added"] == 2 and visits["updated"] == 2 and visits["error"] is None
|
||||
|
||||
|
||||
@@ -2541,6 +2541,12 @@ export const CATALOG_UI_TEXT = {
|
||||
publishSqlErrorBlocked: 'SQL 检查发现的错误级问题:受管级别会拦下发布,请先修改 SQL',
|
||||
publishSqlWarning: 'SQL 检查的提醒:两个发布级别都只提醒、不拦发布',
|
||||
publishSqlErrors: (n: number) => `,其中 ${formatNumber(n)} 处是 SQL 检查发现的错误级问题(受管级别会拦下)`,
|
||||
// ---- 剖析的结构化读数:基数用数据核实过没有、挑表时跳过的空表
|
||||
cardinalityChecked: '已用数据核实',
|
||||
cardinalityByStructure: '子表一侧未核实,按表结构推断',
|
||||
cardinalityCheckedShort: '已核实',
|
||||
cardinalityCheckedHint: '基数由数据剖析用数据核实(子表一侧确有重复或确实唯一),一对多关联后重复计算的检查据此报错误',
|
||||
emptyTables: (n: number) => `挑表时跳过 ${formatNumber(n)} 张空表`,
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { useState } from 'react'
|
||||
import type { CatalogItem } from '../../types'
|
||||
import { isComposing, Modal, Spinner } from '../../components/ui'
|
||||
import { formatDateTime, formatTime } from '../../lib/format'
|
||||
import { CODES_TEXT as KT } from '../../lib/terms'
|
||||
import { StatusChip } from './parts'
|
||||
import { codesComplete } from './model'
|
||||
@@ -45,8 +46,12 @@ export function CodesDialog({ column, item, saving, onSave, onClose }: {
|
||||
<StatusChip status={item.status} source={item.source} />
|
||||
</div>
|
||||
{item.note && (
|
||||
<div className="rounded border bg-bg px-2 py-1.5 text-2xs leading-relaxed text-dim">
|
||||
<div className="rounded border bg-bg px-2 py-1.5 text-2xs leading-relaxed text-dim" data-codes-note="">
|
||||
<span className="text-faint">{KT.note}:</span>{item.note}
|
||||
{/* 说明里不带日期,时间看这一项的 updated_at(按本地时间显示) */}
|
||||
{item.updated_at && (
|
||||
<span className="tnum ml-1 text-faint" title={formatDateTime(item.updated_at)} data-codes-note-at="">({formatTime(item.updated_at)})</span>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
<div className="overflow-hidden rounded-lg border">
|
||||
|
||||
@@ -287,6 +287,12 @@ function Report({ report: r, ms, filled, returnTo, onSettings, onOpenTable, onFi
|
||||
</Notice>
|
||||
)}
|
||||
{r.note && <Notice tone="info" attr={{ 'data-profile-note': '' }}>{r.note}</Notice>}
|
||||
{/* 挑表途中确知是空表、跳过的表(服务端 empty_tables):一张都没剖析时 note 已经说了,这里只在剖析了别的表时列出 */}
|
||||
{r.tables.length > 0 && (r.empty_tables?.length ?? 0) > 0 && (
|
||||
<p className="text-2xs leading-relaxed text-faint" data-profile-empty-tables={r.empty_tables!.length}>
|
||||
{UT.emptyTables(r.empty_tables!.length)}:<span className="mono">{r.empty_tables!.join('、')}</span>
|
||||
</p>
|
||||
)}
|
||||
{shown.map((t) => <TableReport key={t.table_name} t={t} filled={filled} onOpenTable={onOpenTable} onFillCodes={onFillCodes} />)}
|
||||
{quiet.length > 0 && (
|
||||
<p className="text-2xs leading-relaxed text-faint" data-profile-quiet={quiet.length}>
|
||||
@@ -418,7 +424,13 @@ function RelationLine({ f }: { f: CatalogProfileRelationFinding }) {
|
||||
<div className="tnum mt-0.5 text-2xs text-dim">
|
||||
<span data-profile-coverage="">{PT.coverage(coverageText(f.coverage))}</span>
|
||||
{' · '}
|
||||
<span data-profile-cardinality="">{f.cardinality ? CATALOG_CARDINALITY_LABEL[f.cardinality] : PT.cardinalityUnknown}</span>
|
||||
<span data-profile-cardinality="" data-checked={f.cardinality ? String(!!f.cardinality_checked) : undefined}>
|
||||
{f.cardinality ? CATALOG_CARDINALITY_LABEL[f.cardinality] : PT.cardinalityUnknown}
|
||||
{/* 基数用数据核实过没有:读服务端的 cardinality_checked,不从说明的原话里认 */}
|
||||
{f.cardinality && f.cardinality_checked !== undefined && (
|
||||
<span className="text-faint">({f.cardinality_checked ? UT.cardinalityChecked : UT.cardinalityByStructure})</span>
|
||||
)}
|
||||
</span>
|
||||
{' · '}
|
||||
{PT.sampled(f.sample, f.matched)}
|
||||
</div>
|
||||
@@ -440,9 +452,8 @@ function CodesLine({ f, fillKey: key, filled, onFill }: {
|
||||
filled: number | undefined
|
||||
onFill: () => void
|
||||
}) {
|
||||
// 服务端在 summary 里写了几个含义待填写(别的来源写过的含义会沿用);认不出时按都待填写
|
||||
const reported = Number(f.summary.match(/(\d+)\s*个含义待填写/)?.[1] ?? (f.summary.includes('待填写') ? f.values.length : 0))
|
||||
const pending = filled ?? reported
|
||||
// 几个含义待填写读服务端的 pending(别的来源写过的含义会沿用),不从 summary 的原话里认;老服务端不给时不说
|
||||
const pending = filled ?? f.pending ?? 0
|
||||
const done = filled != null && filled === 0
|
||||
return (
|
||||
<li className="text-xs" data-profile-finding="codes" data-column={f.column}>
|
||||
|
||||
@@ -757,7 +757,13 @@ function RelationList({ relations, busy, targetOf, onOpen, onReview }: {
|
||||
</td>
|
||||
<td className={cell}>
|
||||
{label(CT.relationHead.cardinality)}
|
||||
{r.cardinality ? CATALOG_CARDINALITY_LABEL[r.cardinality] : <span className="text-faint">{CT.cardinalityNone}</span>}
|
||||
<span data-relation-cardinality={r.cardinality ?? ''} data-checked={r.cardinality_checked ? 'true' : undefined}>
|
||||
{r.cardinality ? CATALOG_CARDINALITY_LABEL[r.cardinality] : <span className="text-faint">{CT.cardinalityNone}</span>}
|
||||
{/* 数据剖析用数据核实过的基数(cardinality_checked):一对多关联的检查据此给「错误」,其余的最多「提醒」 */}
|
||||
{r.cardinality && r.cardinality_checked && (
|
||||
<span className="ml-1 text-2xs text-faint" title={UT.cardinalityCheckedHint}>{UT.cardinalityCheckedShort}</span>
|
||||
)}
|
||||
</span>
|
||||
</td>
|
||||
<td className={clsx('tnum', cell, typeof r.coverage !== 'number' && '@max-[40rem]:hidden')}>
|
||||
{label(CT.relationHead.coverage)}
|
||||
|
||||
@@ -2901,6 +2901,8 @@ export interface CatalogProfileRelationFinding {
|
||||
previous_status?: CatalogStatus | null
|
||||
coverage: number
|
||||
cardinality: CatalogCardinality | null
|
||||
/** 基数是不是用数据核实的(子表一侧数过是否唯一)。老服务端不给 */
|
||||
cardinality_checked?: boolean
|
||||
/** 抽了几个不同的键值、在被指向表里对上几个 */
|
||||
sample: number
|
||||
matched: number
|
||||
@@ -2915,6 +2917,8 @@ export interface CatalogProfileCodesFinding {
|
||||
values: { value: string; rows: number }[]
|
||||
rows: number
|
||||
status: CatalogStatus
|
||||
/** 写进目录的码值里还有几个含义空着(别的来源写过的含义会沿用)。老服务端不给 */
|
||||
pending?: number
|
||||
summary: string
|
||||
}
|
||||
|
||||
|
||||
+25
-11
@@ -1346,13 +1346,13 @@ function profileReply(state, { ms = 0, extraCodes = false } = {}) {
|
||||
const before = (id) => visits.notes.relations.find((r) => r.id === id)?.status ?? null
|
||||
const previous = { gate: before(GATE_REL), member: before(MEMBER_REL) }
|
||||
notes.relations = notes.relations.map((r) => (r.id === GATE_REL ? {
|
||||
...r, source: 'profile', status: 'verified', coverage: 0.987, cardinality: 'many_to_one', updated_at: at,
|
||||
note: `数据剖析(${PROFILE_DAY}):子表抽样 300 个不同键值,父表对上 296 个,覆盖率 98.7%;被指向列是主键;子表一侧有重复值,多对一。`,
|
||||
...r, source: 'profile', status: 'verified', coverage: 0.987, cardinality: 'many_to_one', cardinality_checked: true, updated_at: at,
|
||||
note: '数据剖析:子表抽样 300 个不同键值,父表对上 296 个,覆盖率 98.7%;被指向列是主键;子表一侧有重复值,多对一。',
|
||||
} : r))
|
||||
notes.relations.push(rel(['member_id'], 'members', ['id'], 'profile', { coverage: 0.62, updated_at: at,
|
||||
note: `数据剖析(${PROFILE_DAY}):抽样覆盖率 62%,可能不是这条关系;子表抽样 200 个不同键值,父表对上 124 个;被指向列是主键。` }))
|
||||
notes.columns.status.codes = item({ 1: '已入园', 9: '已作废', 2: '' }, 'profile', 'proposed')
|
||||
notes.columns.status.codes.note = `数据剖析(${PROFILE_DAY}):统计全表,182000 行非空值共 3 个取值:1(94%)、9(5%)、2(1%)。`
|
||||
note: '数据剖析:抽样覆盖率 62%,可能不是这条关系;子表抽样 200 个不同键值,父表对上 124 个;被指向列是主键。' }))
|
||||
notes.columns.status.codes = { ...item({ 1: '已入园', 9: '已作废', 2: '' }, 'profile', 'proposed'), updated_at: at }
|
||||
notes.columns.status.codes.note = '数据剖析:统计全表,182000 行非空值共 3 个取值:1(94%)、9(5%)、2(1%)。'
|
||||
// 统计的是全表:服务端给码值候选标「已列全」
|
||||
notes.columns.status.codes.complete = true
|
||||
// extraCodes:票种列也取到码值候选(两个取值,含义都空着)——报告里有两列要填,填完一列回到报告接着填下一列
|
||||
@@ -1363,16 +1363,17 @@ function profileReply(state, { ms = 0, extraCodes = false } = {}) {
|
||||
table_name: 'visits', queries: 38, row_estimate: { rows: 182000, method: 'stats', at_least: null },
|
||||
findings: [
|
||||
{ kind: 'relation', path: `relations.${GATE_REL}`, target: 'gate_id → gates.id', columns: ['gate_id'], to_table: 'gates', to_columns: ['id'],
|
||||
status: 'verified', confirmed: false, previous_status: previous.gate, coverage: 0.987, cardinality: 'many_to_one', sample: 300, matched: 296,
|
||||
status: 'verified', confirmed: false, previous_status: previous.gate, coverage: 0.987, cardinality: 'many_to_one', cardinality_checked: true,
|
||||
sample: 300, matched: 296,
|
||||
summary: previous.gate === 'verified' ? '覆盖率 98.7%,仍为有确证' : '覆盖率 98.7%,升为有确证' },
|
||||
{ kind: 'relation', path: `relations.${MEMBER_REL}`, target: 'member_id → members.id', columns: ['member_id'], to_table: 'members',
|
||||
to_columns: ['id'], status: 'proposed', confirmed: false, previous_status: previous.member, coverage: 0.62, cardinality: 'many_to_one',
|
||||
sample: 200, matched: 124,
|
||||
cardinality_checked: false, sample: 200, matched: 124,
|
||||
summary: '抽样覆盖率 62%,保持推断' },
|
||||
{ kind: 'codes', path: 'columns.status.codes', column: 'status', values: [{ value: '1', rows: 171080 }, { value: '9', rows: 9100 },
|
||||
{ value: '2', rows: 1820 }], rows: 182000, status: 'proposed', summary: '3 个取值,1 个含义待填写' },
|
||||
{ value: '2', rows: 1820 }], rows: 182000, status: 'proposed', pending: 1, summary: '3 个取值' },
|
||||
...(extraCodes ? [{ kind: 'codes', path: 'columns.ticket_type_id.codes', column: 'ticket_type_id', values: [{ value: '11', rows: 120000 },
|
||||
{ value: '12', rows: 62000 }], rows: 182000, status: 'proposed', summary: '2 个取值,2 个含义待填写' }] : []),
|
||||
{ value: '12', rows: 62000 }], rows: 182000, status: 'proposed', pending: 2, summary: '2 个取值' }] : []),
|
||||
],
|
||||
skipped: [
|
||||
{ kind: 'codes', target: 'member_level', path: 'columns.member_level.codes', reason: 'masked', detail: 'visits.member_level 在数据源设置中被遮罩,不取值' },
|
||||
@@ -1395,6 +1396,7 @@ function profileReply(state, { ms = 0, extraCodes = false } = {}) {
|
||||
return json({
|
||||
profiled_at: at, actor: ACTOR, settings: { ...PROFILE_ON, sample_size: 2000, max_scan_rows: 100000, max_total_s: 120 },
|
||||
queries_used: 60, stopped: 'budget', tables, total: { added: 1, updated: 2, removed: 0 }, note: null,
|
||||
empty_tables: ['ext_log_001', 'ext_log_002'],
|
||||
})(route)
|
||||
}
|
||||
}
|
||||
@@ -1588,7 +1590,7 @@ await section('数据目录 · 数据剖析', async () => {
|
||||
const visitsCard = report.locator('[data-profile-table="visits"]')
|
||||
const gate = visitsCard.locator(`[data-profile-finding="relation"][data-path="relations.${GATE_REL}"]`)
|
||||
check('关系核实:覆盖率、基数、抽样规模,升为已验证', (await gate.locator('[data-profile-coverage]').innerText()) === '覆盖率 98.7%'
|
||||
&& (await gate.locator('[data-profile-cardinality]').innerText()) === '多对一' && (await gate.innerText()).includes('抽样 300 个键值,对上 296 个')
|
||||
&& (await gate.locator('[data-profile-cardinality]').innerText()) === '多对一(已用数据核实)' && (await gate.innerText()).includes('抽样 300 个键值,对上 296 个')
|
||||
&& (await gate.locator('[data-profile-outcome]').innerText()) === '升为已验证' && (await gate.getAttribute('data-status')) === 'verified',
|
||||
(await gate.innerText()).replace(/\s+/g, ' '))
|
||||
// 结论是状态说明,不是链接:带框的小标签,文字用次要文字色(不是强调蓝),不能点
|
||||
@@ -1607,6 +1609,11 @@ await section('数据目录 · 数据剖析', async () => {
|
||||
check('……结论用状态标签的样子:带框、次要文字色,不是蓝色正文、不像链接', outcomeLook.kind === 'raised' && outcomeLook.color === outcomeLook.dim
|
||||
&& outcomeLook.color !== outcomeLook.accent && outcomeLook.border === 'solid' && outcomeLook.cursor !== 'pointer' && !outcomeLook.clickable,
|
||||
JSON.stringify(outcomeLook))
|
||||
// 结构化的读数:基数核实读 cardinality_checked,几个含义待填写读 pending(假答复的 summary 里故意不写),空表读 empty_tables
|
||||
check('……基数用数据核实过没有,读 cardinality_checked', (await gate.locator('[data-profile-cardinality]').getAttribute('data-checked')) === 'true'
|
||||
&& (await visitsCard.locator(`[data-path="relations.${MEMBER_REL}"] [data-profile-cardinality]`).innerText()) === '多对一(子表一侧未核实,按表结构推断)')
|
||||
check('……挑表时跳过的空表读 empty_tables', (await report.locator('[data-profile-empty-tables="2"]').innerText().catch(() => ''))
|
||||
=== '挑表时跳过 2 张空表:ext_log_001、ext_log_002')
|
||||
const member = visitsCard.locator(`[data-profile-finding="relation"][data-path="relations.${MEMBER_REL}"]`)
|
||||
check('……覆盖率不够的保持推断,写明原因', (await member.locator('[data-profile-outcome]').innerText()) === '保持推断:覆盖率不足,可能不是这条关系'
|
||||
&& (await member.locator('[data-profile-coverage]').innerText()) === '覆盖率 62%')
|
||||
@@ -1635,6 +1642,11 @@ await section('数据目录 · 数据剖析', async () => {
|
||||
(await gateRow.innerText().catch(() => '')).replace(/\s+/g, ' '))
|
||||
await openMark(page, `relations.${GATE_REL}`)
|
||||
check('……点开来源:写明抽样规模和覆盖率的剖析说明', (await panel(page).innerText()).includes('子表抽样 300 个不同键值,父表对上 296 个'))
|
||||
// 说明里不带日期:时间看这一项的 updated_at(按本地时间,2026-10-03 03:00 UTC 是 11:00)
|
||||
check('……说明不带日期,时间按这一项的 updated_at 写', !/\d{4}-\d{2}-\d{2}/.test(await panel(page).locator('.whitespace-pre-wrap').innerText())
|
||||
&& /更新于 (2026\/)?10\/3 11:00/.test(await panel(page).innerText()), (await panel(page).innerText()).replace(/\s+/g, ' '))
|
||||
check('……关系表上写基数已用数据核实', await page.locator(`[data-relation="${GATE_REL}"] [data-relation-cardinality][data-checked="true"]`).count() === 1
|
||||
&& (await page.locator(`[data-relation="${GATE_REL}"] [data-relation-cardinality]`).innerText()).includes('已核实'))
|
||||
await page.keyboard.press('Escape')
|
||||
const statusCodes = page.locator('[data-column="status"] [data-cell="codes"]')
|
||||
check('……码值的含义空着写「含义待填写」,给「填写含义」', (await statusCodes.locator('[data-code="2"] [data-code-pending]').innerText()) === '含义待填写'
|
||||
@@ -1650,7 +1662,9 @@ await section('数据目录 · 数据剖析', async () => {
|
||||
check('码值弹窗:每个码值一个输入框,空着的写「含义待填写」,光标在它上面', await codesBox.locator('input[data-code]').count() === 3
|
||||
&& (await codesBox.locator('input[data-code="2"]').getAttribute('placeholder')) === '含义待填写'
|
||||
&& await until(async () => page.evaluate(() => document.activeElement?.getAttribute('data-code') === '2')))
|
||||
check('……写明剖析说明(各取值的占比)', (await codesBox.innerText()).includes('1(94%)、9(5%)、2(1%)'))
|
||||
check('……写明剖析说明(各取值的占比),时间按这一项的 updated_at', (await codesBox.innerText()).includes('1(94%)、9(5%)、2(1%)')
|
||||
&& /^((2026\/)?10\/3 11:00)$/.test(await codesBox.locator('[data-codes-note-at]').innerText().catch(() => '')),
|
||||
await codesBox.locator('[data-codes-note]').innerText().catch(() => ''))
|
||||
check('……没填时保存禁用', await page.locator('[data-codes-save]').isDisabled())
|
||||
check('……写明当前已列出全部取值(剖析统计了全表),可以取消', await codesBox.locator('[data-codes-complete]').isChecked()
|
||||
&& (await codesBox.innerText()).includes('已列出全部取值') && (await statusCodes.locator('[data-codes-complete-mark]').count()) === 1)
|
||||
|
||||
Reference in New Issue
Block a user