From 03a9684a187160295aef0d1e1551619f36ddb305 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Thu, 15 Oct 2026 18:14:14 +0800 Subject: [PATCH 1/4] =?UTF-8?q?=E6=95=B0=E6=8D=AE=E5=89=96=E6=9E=90?= =?UTF-8?q?=E5=89=8D=E7=AB=AF=E6=94=B9=E6=88=90=E9=80=90=E8=A1=A8=E7=9A=84?= =?UTF-8?q?=E5=90=8E=E5=8F=B0=E4=BB=BB=E5=8A=A1=EF=BC=9A=E5=AF=B9=E8=AF=9D?= =?UTF-8?q?=E6=A1=86=E7=9C=8B=E5=BE=97=E5=88=B0=E6=AF=8F=E5=BC=A0=E8=A1=A8?= =?UTF-8?q?=E7=9A=84=E8=BF=9B=E5=BA=A6=EF=BC=8C=E5=8F=AF=E4=BB=A5=E5=81=9C?= =?UTF-8?q?=E6=AD=A2=E3=80=81=E9=87=8D=E8=AF=95=E6=B2=A1=E5=81=9A=E5=AE=8C?= =?UTF-8?q?=E7=9A=84=E8=A1=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 以前剖析是一次同步长请求,进行中只有一条已用时长的横条、停不下来,关掉页面报告就没了;预算按整次剖析算,几十张表 往往只做完前两三张。服务端改成逐表的后台任务(profile-jobs)后,界面照起草任务的做法重做: - types、api:按接口约定加剖析任务、逐表行、逐表详情(检查清单、发现、发过的查询)、清单行上的最近一次剖析和 profile_due / profile_checks、目录顶上的 active_profile_job,起草任务加 waiting_for;删掉同步接口 profile() 和只给它用的 类型;设置换成每张表最多查询数、每张表时长上限两项新键 - profileJob.ts(新):useProfileJob,进行中每 1.5 秒取一次,状态只往前走,关掉对话框照常进行,结束时右下角提示 - ProfileDialog 重做:开始前四种范围和个数、按表写的预算和这次最多约几条查询、只读一条一条发可随时停止的说明,同源有 起草任务时「排队剖析」;进行中逐表一行(状态、各类检查进度、查询数、用时、正在做哪一项),点开看检查清单、发现和发过的 查询;结束后汇总、止损和停止的原因照实写、按有发现 / 有跳过 / 失败筛、重试没做完的表;409 六种情况各有说法和下一步 - 目录页头部起草、剖析各一个进度入口;表清单加「剖析」一格和「需要剖析」开关(地址里记 profile=due);表详情加 「数据剖析」一节和「剖析这张表」 - 起草排在剖析后面时写「排在剖析任务后面,剖析结束后自动开始」 - 文案:去掉「剖析无法中途停止」「总时长上限」等已不成立的说法,补上新的跳过原因和任务状态的叫法 --- frontend/src/api/client.ts | 47 +- frontend/src/lib/terms.ts | 303 ++++-- frontend/src/pages/DataSourcesTab.tsx | 2 +- frontend/src/pages/catalog/CatalogPage.tsx | 256 +++-- frontend/src/pages/catalog/DraftDialog.tsx | 19 +- frontend/src/pages/catalog/ProfileDialog.tsx | 967 +++++++++++++----- .../src/pages/catalog/ProfileSettings.tsx | 4 +- frontend/src/pages/catalog/TableDetail.tsx | 173 +++- frontend/src/pages/catalog/TableIndex.tsx | 49 +- frontend/src/pages/catalog/draftJob.ts | 7 + frontend/src/pages/catalog/model.ts | 26 +- frontend/src/pages/catalog/profile.ts | 149 ++- frontend/src/pages/catalog/profileJob.ts | 162 +++ frontend/src/types.ts | 229 ++++- 14 files changed, 1885 insertions(+), 508 deletions(-) create mode 100644 frontend/src/pages/catalog/profileJob.ts diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index 960078c3..f6f12789 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -1,6 +1,6 @@ import type { ActivateSnapshotBody, ActivateSnapshotOut, AiPreview, CatalogAssistantEvent, CatalogAssistantRequest, CatalogDetail, CatalogDraftJob, - CatalogImpact, CatalogImportReport, CatalogList, CatalogNotes, CatalogPatchPreview, CatalogPatchSubmit, CatalogProfileOut, + CatalogImpact, CatalogImportReport, CatalogList, CatalogNotes, CatalogPatchPreview, CatalogPatchSubmit, CatalogProfileJob, CatalogProfileTaskDetail, CatalogReviewAction, CatalogReviewBatchItem, CatalogReviewBatchResult, CatalogReviewKind, CatalogReviewProgress, CatalogReviewQueue, CatalogTableUsage, CommitOut, CurrentRecipe, EditPreview, EditRequest, ImportRecord, @@ -466,12 +466,44 @@ export const api = { putGuidance: (sourceId: string, guidance: string) => put<{ guidance: string | null }>(`${catalogBase(sourceId)}/guidance`, { guidance }), /** - * 同步剖析:对业务库发少量只读查询,核对关系、取码值候选,结论写进目录。不给 tables 时按使用次数取前 10 张 - * 有待核实关系的表。没开启、数据源停用、没有表结构、同一个数据源正在剖析、快照被改动时回 409(detail 是一句中文) + * 剖析任务(后台进行:逐表对业务库发只读查询,一条一条发,每张表一结束就写进目录)。剖析会对业务库发查询,默认关闭, + * 按数据源开启。和同一个数据源的起草任务按建立顺序排队(起草在排队、进行中时剖析建成 queued、waiting_for 为 draft) */ - profile: (sourceId: string, body: { tables?: string[] }, opts?: RequestOptions) => - request(`/datasources/${encodeURIComponent(sourceId)}/catalog/profile`, - { method: 'POST', body: JSON.stringify(body), ...opts }), + profileJobs: { + /** + * 建任务:tables 非空时只剖析这些表(最多 500 张,按给的顺序,表结构里没有、视图、没有可剖析的项照样建进任务、记跳过), + * 否则按 scope(缺省 needed:需要剖析的表;all:全部可剖析的表)由服务端挑。202。 + * 409 带机读码:profile_disabled、datasource_inactive、schema_missing、snapshot_tampered、profile_busy(这个源已有排队、 + * 进行中的剖析任务,detail 写明进行到第几张)、profile_nothing(按范围挑出 0 张表,detail 写明原因和建议) + */ + create: (sourceId: string, body: { tables?: string[]; scope?: 'needed' | 'all' }) => + post<{ job: CatalogProfileJob }>(`${catalogBase(sourceId)}/profile-jobs`, body), + /** 排队或进行中的那个剖析任务;没有为 null */ + active: (sourceId: string, opts?: RequestOptions) => + get<{ job: CatalogProfileJob | null }>(`${catalogBase(sourceId)}/profile-jobs/active`, opts), + /** 最近建的一个剖析任务(任何状态);没有为 null */ + latest: (sourceId: string, opts?: RequestOptions) => + get<{ job: CatalogProfileJob | null }>(`${catalogBase(sourceId)}/profile-jobs/latest`, opts), + /** 一个剖析任务的当前状态;不属于这个数据源回 404 */ + get: (sourceId: string, jobId: string, opts?: RequestOptions) => + get<{ job: CatalogProfileJob }>(`${catalogBase(sourceId)}/profile-jobs/${encodeURIComponent(jobId)}`, opts), + /** 任务里一张表的完整结果:检查清单、发现、跳过的项、日期范围、发过的查询。找不到 404 */ + task: (sourceId: string, jobId: string, seq: number, opts?: RequestOptions) => + get<{ task: CatalogProfileTaskDetail }>( + `${catalogBase(sourceId)}/profile-jobs/${encodeURIComponent(jobId)}/tasks/${encodeURIComponent(String(seq))}`, opts), + /** + * 停止:排队的直接停;进行中的那条查询由数据库按单条时限截止,这张表已做完的项照常写进目录、没做的保留旧结论, + * 后面的表记已停止。回停止之后的状态;已经结束的回 409(catalog_profile_finished) + */ + cancel: (sourceId: string, jobId: string) => + post<{ job: CatalogProfileJob }>(`${catalogBase(sourceId)}/profile-jobs/${encodeURIComponent(jobId)}/cancel`), + /** + * 重试没做完的表:新任务只含原任务里失败、部分完成、已停止的表(按原顺序)。202;没有可重试的表 422; + * 这个源已有排队、进行中的剖析任务 409(profile_busy) + */ + retry: (sourceId: string, jobId: string) => + post<{ job: CatalogProfileJob }>(`${catalogBase(sourceId)}/profile-jobs/${encodeURIComponent(jobId)}/retry`), + }, // ---- 审阅队列(只列推断项,按表的使用次数、这一项对数字的影响和来源排好) /** 一页待确认项。kind:全部 / 影响数字的项 / 名称和含义;table 只看这一张表 */ reviewQueue: (sourceId: string, params: { limit?: number; offset?: number; kind?: CatalogReviewKind; table?: string | null }, @@ -512,7 +544,8 @@ export const api = { importPreview: (sourceId: string, body: { content: unknown; include_guidance: boolean }) => post(`${catalogBase(sourceId)}/import/preview`, body), /** - * 导入:规则同预览,逐表乐观锁,一张表失败不影响别的。有排队、进行中的起草任务时 409(catalog_draft_busy); + * 导入:规则同预览,逐表乐观锁,一张表失败不影响别的。有排队、进行中的起草任务时 409(catalog_draft_busy),剖析任务时 + * 409(profile_busy); * 文件格式不对 422(detail 是一句中文) */ importCatalog: (sourceId: string, body: { content: unknown; include_guidance: boolean }) => diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index 979c7541..89763c3a 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -2485,6 +2485,10 @@ export const CATALOG_DRAFT_TEXT = { : total === 1 ? '表结构中已没有这张表,不能起草' : `表结构中已没有这 ${formatNumber(total)} 张表,不能起草`), // ---- 进行中 queued: '排队中,等待开始…', + /** 同一个数据源的起草和剖析按建立顺序排队:排在剖析任务后面 */ + waitingProfile: '排在剖析任务后面,剖析结束后自动开始', + /** 设置阶段:这个数据源有剖析任务在排队、进行中,起草会排在它后面 */ + queueHint: '这个数据源有剖析任务在进行:起草会排在它后面,剖析结束后自动开始', running: '正在起草…', // 服务端点了停止就取消进行中的模型调用(不等它答完,一次调用可能要几分钟),只等手上正在写入目录的那张表写完 // 宽表每写完一段就写进目录:停下来时已写进的段保留,只停掉进行中的调用 @@ -2578,6 +2582,13 @@ export const CATALOG_DRAFT_TEXT = { // ---- 目录页上方的进度、结束提示 strip: (done: number, total: number) => `正在起草:已完成 ${formatNumber(done)} / ${formatNumber(total)} 张表`, stripQueued: (total: number) => `起草任务排队中,共 ${formatNumber(total)} 张表`, + stripQueuedProfile: (total: number) => `起草任务排在剖析任务后面,剖析结束后自动开始(共 ${formatNumber(total)} 张表)`, + /** 页头的进度入口:进行中写「起草 3/20」,排队写「起草 排队中」 */ + headerRunning: (done: number, total: number) => `起草 ${formatNumber(done)}/${formatNumber(total)}`, + headerQueued: '起草 排队中', + headerRunningHint: (done: number, total: number) => `起草进行中:已完成 ${formatNumber(done)} / ${formatNumber(total)} 张表,点击查看进度`, + headerQueuedHint: '起草任务排队中,点击查看', + headerQueuedProfile: '起草任务排在剖析任务后面,剖析结束后自动开始,点击查看', stripOpen: '查看进度', toast: { succeeded: (summary: string) => `数据目录起草完成:${summary}`, @@ -2752,47 +2763,51 @@ export const SCHEMA_PARTIAL_TEXT = { } // =========================================================================== -// 数据剖析(数据源设置里的开关和预算;数据目录页的「数据剖析」) +// 数据剖析(数据源设置里的开关和预算;数据目录页的「数据剖析」:后台任务,逐表进行) // // 术语: -// - 「数据剖析」:对业务库发少量只读查询,核实推断的关联关系(覆盖率、基数),取状态类列的码值候选,提议业务日期。 +// - 「数据剖析」:对业务库发只读查询,核实推断的关联关系(覆盖率、基数),取状态类列的码值候选,提议业务日期。 // 动作写「剖析」,不写扫描、探测。默认关闭,按数据源开启。 -// - 预算五项:查询次数上限、单条查询时限、抽样键值数、整表统计行数上限、总时长上限。标签和服务端校验报错里的叫法 -// 一致(catalog_profile._NUMBER_FIELDS),保存被拒时按它认出是哪一项。 +// - 剖析是后台任务:逐张表进行,每张表剖析完就写进目录;查询一条一条地发,可以随时停止。关掉对话框照常进行。 +// - 预算按表算:「每张表最多查询数」「每张表时长上限」用完只停这张表,下一张表重新计数;另有单条查询时限、抽样键值数、 +// 整表统计行数上限。标签和服务端校验报错里的叫法一致(catalog_profile._NUMBER_FIELDS),保存被拒时按机读码认出是哪一项。 +// - 没有整次总量上限:任务自己在连续失败、多数超时时停下(止损),写明原因。 +// - 「没查成」≠ 结论错了:因为次数、时长、超时、出错没查成的项保留剖析以前的结论,可以「重试没做完的表」。 +// - 和起草同一套状态词:排队、进行中、完成、部分完成、失败、已停止、服务重启中断。同一个数据源的起草和剖析排队进行。 // =========================================================================== export const PROFILE_FIELD_LABEL: Record<'enabled' | CatalogProfileNumberKey, string> = { enabled: '开启数据剖析', - max_queries: '查询次数上限', + table_max_queries: '每张表最多查询数', + table_max_s: '每张表时长上限', query_timeout_s: '单条查询时限', sample_size: '抽样键值数', max_scan_rows: '整表统计行数上限', - max_total_s: '总时长上限', } export const PROFILE_FIELD_UNIT: Record = { - max_queries: '条', + table_max_queries: '条', + table_max_s: '秒', query_timeout_s: '秒', sample_size: '个', max_scan_rows: '行', - max_total_s: '秒', } export const PROFILE_FIELD_HINT: Record = { - max_queries: '一次剖析最多发出的查询条数,用完即停止', - query_timeout_s: '超过即中断这条查询;不超过数据源自身的查询时限', + table_max_queries: '剖析一张表最多发出的查询条数,用完只停这张表,下一张表重新计数', + table_max_s: '剖析一张表最多用的时长,到时只停这张表,下一张表重新计时', + query_timeout_s: '超过即中断这条查询;不超过数据源自身的查询时限,也不超过这张表剩下的时长', sample_size: '核对一条关联关系时,从本表抽取的不同键值个数', max_scan_rows: '不超过此行数的表才做去重计数、最小值和最大值等整表统计', - max_total_s: '一次剖析的总时长,到时即停止', } export const PROFILE_TEXT = { // ---- 设置 section: '数据剖析', enableHint: '默认关闭。开启后可在数据目录中核实推断的关联关系、取状态类列的码值候选', - risk: (queries: number, timeout: number, total: number) => - `开启后,每次剖析会对这个数据源发出只读查询:最多 ${formatNumber(queries)} 条,单条不超过 ${formatNumber(timeout)} 秒,` - + `总时长不超过 ${formatNumber(total)} 秒。不读取明细行,遮罩的列不取样。`, + risk: (queries: number, perTable: number, timeout: number) => + `开启后,剖析会对这个数据源发出只读查询,一条一条地发,可随时停止:每张表最多 ${formatNumber(queries)} 条、` + + `${formatNumber(perTable)} 秒,单条不超过 ${formatNumber(timeout)} 秒。不读取明细行,遮罩的列不取样。`, range: (min: number, max: number, unit: string, def: number) => `范围 ${formatNumber(min)}–${formatNumber(max)} ${unit},默认 ${formatNumber(def)}`, scanZero: '填 0 表示一律不做整表统计', @@ -2802,70 +2817,160 @@ export const PROFILE_TEXT = { invalid: (n: number) => `数据剖析有 ${formatNumber(n)} 项设置不正确`, // ---- 数据源卡片 chip: '数据剖析已开启', - chipHint: (queries: number, timeout: number) => - `数据剖析已开启:每次最多 ${formatNumber(queries)} 条只读查询,单条不超过 ${formatNumber(timeout)} 秒`, + chipHint: (queries: number, perTable: number, timeout: number) => + `数据剖析已开启:每张表最多 ${formatNumber(queries)} 条只读查询、${formatNumber(perTable)} 秒,单条不超过 ${formatNumber(timeout)} 秒`, open: '设置数据剖析', dialogTitle: (name: string) => `「${name}」的数据剖析设置`, - dialogBody: '设置保存在数据源上,对数据目录中的每次剖析生效。', + dialogBody: '设置保存在数据源上,对之后新建的剖析生效;进行中的剖析按开始时的设置。', savedOn: (name: string) => `「${name}」已开启数据剖析`, savedOff: (name: string) => `「${name}」已关闭数据剖析`, saved: (name: string) => `已保存「${name}」的数据剖析设置`, - // ---- 数据目录页:范围与确认 + + // ---- 数据目录页:入口 action: '数据剖析', - actionHint: '对业务库发少量只读查询:核实推断的关联关系、取码值候选、提议业务日期', + actionHint: '对业务库发只读查询:核实推断的关联关系、取码值候选、提议业务日期', profileSelected: (n: number) => `剖析所选(${formatNumber(n)})`, + /** 页头的进度入口:进行中写「剖析 12/40」,排队写「剖析 排队中」 */ + headerRunning: (done: number, total: number) => `剖析 ${formatNumber(done)}/${formatNumber(total)}`, + headerQueued: '剖析 排队中', + headerRunningHint: (done: number, total: number) => `数据剖析进行中:已完成 ${formatNumber(done)} / ${formatNumber(total)} 张表,点击查看进度`, + headerQueuedHint: '剖析任务排队中,点击查看', + headerQueuedDraft: '剖析任务排在起草任务后面,起草结束后自动开始,点击查看', + + // ---- 开始前:范围、预算 title: '数据剖析', - intro: '对业务库发少量只读查询,核实推断的关联关系(覆盖率、基数),取状态类列的码值候选,提议业务日期。' - + '结论写入数据目录;已确认、已驳回的项不会被改动。', + intro: '逐张表对业务库发只读查询,核实推断的关联关系(覆盖率、基数),取状态类列的码值候选,提议业务日期。' + + '每张表剖析完就写进数据目录;已确认、已驳回的项不会被改动。', scope: '剖析范围', scopeSelected: (n: number) => `已选的 ${formatNumber(n)} 张表`, scopeFiltered: (n: number) => `当前筛选结果中的 ${formatNumber(n)} 张表`, - scopeDefault: '使用次数最多、有待核实关联关系的表', - scopeDefaultHint: (n: number) => `由服务端挑选,至多 ${formatNumber(n)} 张`, - scopeTooMany: (max: number) => `超过 ${formatNumber(max)} 张,请缩小范围`, - budget: '本次预算', - budgetQueries: (queries: number, timeout: number) => - `最多 ${formatNumber(queries)} 条只读查询,单条不超过 ${formatNumber(timeout)} 秒`, - budgetTotal: (total: number) => `总时长不超过 ${formatNumber(total)} 秒,到时即停止`, + scopeNeeded: (n: number) => `需要剖析的 ${formatNumber(n)} 张表`, + scopeNeededHint: '从没剖析过、上次没做完,或剖析之后又出现了待核实关联关系的表', + scopeNeededNone: '没有需要剖析的表:都剖析过、上次都做完了,也没有新的待核实关联关系', + scopeAll: (n: number) => `全部 ${formatNumber(n)} 张表`, + scopeAllHint: '有可剖析项的表,不含视图', + /** 已选、筛选结果里有剖析不了的表:不交给服务端,写明有几张 */ + scopeLeftOut: (n: number) => `另有 ${formatNumber(n)} 张不剖析(视图、表结构中已没有,或没有可剖析的项)`, + scopeEmpty: '这个范围里没有可剖析的表', + scopeTooMany: (max: number) => `一次最多剖析 ${formatNumber(max)} 张表,请缩小范围`, + budget: '预算', + budgetFrom: (name: string) => `按「${name}」的数据剖析设置`, + budgetPerTable: (queries: number, secs: number) => + `每张表最多 ${formatNumber(queries)} 条查询、${formatNumber(secs)} 秒,用完只停这张表,下一张表重新计数`, + budgetQuery: (secs: number) => `单条查询不超过 ${formatNumber(secs)} 秒`, budgetSample: (n: number) => `每条关联关系抽样 ${formatNumber(n)} 个键值`, budgetScan: (n: number) => (n > 0 ? `行数超过 ${formatNumber(n)} 的表不做整表统计` : '不做整表统计'), - budgetFrom: (name: string) => `按「${name}」的数据剖析设置`, + /** 上限的估算:表数 × 每张表的上限 */ + budgetTotal: (tables: number, queries: number, span: string) => + `这次 ${formatNumber(tables)} 张表,最多约 ${formatNumber(queries)} 条查询、${span}`, editSettings: '修改设置', - noCancel: '剖析在服务端一次完成,开始后无法中途停止。', + readOnly: '剖析会对业务库发只读查询,一条一条地发,可随时停止;停止时已做完的项照常写进目录,没做的保留原来的结论。', start: '开始剖析', - again: '重新剖析', + /** 同一个数据源有起草任务在排队、进行中:剖析排在它后面 */ + queue: '排队剖析', + queueHint: '这个数据源有起草任务在进行:剖析会排在它后面,起草任务结束后自动开始', disabledTitle: '这个数据源未开启数据剖析', disabledBody: '剖析会对业务库发出只读查询,需要先在数据源设置中开启。', enable: '开启数据剖析', - // ---- 进行中 - running: '正在剖析…', - elapsed: (used: string, total: string) => `已用时 ${used}(总时长上限 ${total})`, - runningHint: '剖析在服务端一次完成,无法中途停止。关闭此窗口不影响剖析,完成后会提示结果。', - background: '在后台继续', - headerRunning: (used: string) => `正在剖析 ${used}`, - headerRunningHint: '数据剖析进行中,点击查看', - leavePage: '离开本页不会中止剖析,结论仍会写入数据目录,但不再显示本次的报告。', - // ---- 报告 - done: '剖析完成', - stoppedTitle: (reason: string) => `剖析已停止:${reason}`, - summary: (used: number, max: number, tables: number) => - `用了 ${formatNumber(used)} / ${formatNumber(max)} 条查询,剖析 ${formatNumber(tables)} 张表`, + /** 上一次剖析(没做完的才提),可以点开 */ + last: (state: string, at: string) => `上一次剖析(${at}):${state}`, + lastOpen: '查看', + startFailed: (reason: string) => `未能开始剖析:${reason}`, + + // ---- 进行中、结束:任务 + jobTitle: { + queued: '排队中,等待开始…', + running: '正在剖析…', + succeeded: '剖析完成', + partial: '剖析完成,部分表没做完', + failed: '剖析失败', + cancelled: '已停止', + interrupted: '服务重启,剖析中断', + } as Record, + waitingDraft: '排在起草任务后面,起草结束后自动开始', + // 停止时正在执行的那条查询由数据库按单条时限截止,不是立刻停 + stopping: '正在停止:等正在执行的那条查询结束(不超过单条查询时限),已做完的项照常写进目录…', + progress: (done: number, total: number) => `${formatNumber(done)} / ${formatNumber(total)} 张表`, + queriesSent: (n: number) => `已发 ${formatNumber(n)} 条查询`, + queriesBad: (timeouts: number, errors: number) => [ + timeouts ? `超时 ${formatNumber(timeouts)} 条` : '', errors ? `失败 ${formatNumber(errors)} 条` : '', + ].filter(Boolean).join('、'), + elapsed: (span: string) => `用时 ${span}`, + hide: '在后台继续', + hideHint: '关闭对话框不会停止剖析,页头显示进度;每张表剖析完就写进目录', + stop: '停止', + busyShown: '这个数据源已有剖析任务在进行,下面显示的是它的进度', + createdBy: (who: string) => `由「${who}」发起`, + /** 任务级止损 */ + stopTitle: { failed: '连续多条查询失败,剖析已停止', slow: '数据库响应慢,剖析已停止' } as Record, + stopShort: { failed: '连续多条查询失败', slow: '数据库响应慢' } as Record, + stopNext: { + failed: '后面的表没有剖析。请先在数据源卡片上测试连接、确认账号有只读权限,再重试没做完的表。', + slow: '后面的表没有剖析。可以在业务库不忙的时候重试没做完的表,或在剖析设置中调低单条查询时限、每张表最多查询数。', + } as Record, + cancelledNote: '做完的项已写进数据目录,没做的项保留原来的结论;后面的表没有剖析,可以重试没做完的表。', + interruptedNote: '服务重启时这次剖析还没有完成:做完的项已写进数据目录,没做的项保留原来的结论。不会自动续上,可以重试没做完的表。', + /** 结束后的汇总:「40 张表:完成 33,部分完成 4,跳过 2,失败 1」 */ + counts: (total: number, c: { ok: number; partial: number; skipped: number; failed: number; cancelled: number }) => + `${formatNumber(total)} 张表:${[ + `完成 ${formatNumber(c.ok)}`, + c.partial ? `部分完成 ${formatNumber(c.partial)}` : '', + c.skipped ? `跳过 ${formatNumber(c.skipped)}` : '', + c.failed ? `失败 ${formatNumber(c.failed)}` : '', + c.cancelled ? `已停止 ${formatNumber(c.cancelled)}` : '', + ].filter(Boolean).join(',')}`, changes: (added: number, updated: number, removed: number) => - `新增 ${formatNumber(added)} 项、更新 ${formatNumber(updated)} 项${removed ? `、删除 ${formatNumber(removed)} 项` : ''}`, + `目录新增 ${formatNumber(added)} 项、更新 ${formatNumber(updated)} 项${removed ? `、删除 ${formatNumber(removed)} 项` : ''}`, noChange: '目录没有变化', - toastDone: (summary: string) => `数据剖析完成:${summary}`, - toastStopped: (reason: string) => `数据剖析已停止:${reason}`, - toastFailed: '数据剖析未能完成', - viewReport: '查看报告', + /** + * 发现的合计(任务、逐表共用):核实了几条关系、几条仍为推断,取到几列码值(几个含义待填写),提议了业务日期。 + * 一项都没有为空串 + */ + findings: (s: { relations_checked: number; relations_verified: number; codes: number; codes_pending: number; business_dates: number }, + perTable = false) => [ + s.relations_verified ? `核实 ${formatNumber(s.relations_verified)} 条关系` : '', + s.relations_checked > s.relations_verified ? `${formatNumber(s.relations_checked - s.relations_verified)} 条关系仍为推断` : '', + s.codes ? `取到 ${formatNumber(s.codes)} 列码值候选${s.codes_pending ? `(${formatNumber(s.codes_pending)} 个含义待填写)` : ''}` : '', + s.business_dates ? (perTable ? '提议业务日期' : `提议 ${formatNumber(s.business_dates)} 张表的业务日期`) : '', + ].filter(Boolean).join(','), + noFindings: '没有新的发现', + filterLabel: '筛选逐表结果', + filter: { all: '全部', findings: '有发现', skipped: '有跳过', failed: '失败' } as Record, + filterNone: '没有符合的表', + retry: (n: number) => `重试没做完的表(${formatNumber(n)} 张)`, + retryHint: '新建一次剖析,只含失败、部分完成和已停止的表;已写进目录的结论不受影响', + again: '重新选择范围', close: '完成', + tasks: '逐表进度', + + // ---- 逐表一行 + taskStatus: { + pending: '排队', running: '剖析中', ok: '完成', partial: '部分完成', failed: '失败', skipped: '跳过', cancelled: '已停止', + } as Record, + checkKind: { relation: '关系', codes: '码值', date: '日期' } as Record, + kindProgress: (label: string, done: number, total: number) => `${label} ${formatNumber(done)}/${formatNumber(total)}`, tableQueries: (n: number) => `${formatNumber(n)} 条查询`, + /** 这张表的查询次数用完了(等于每张表最多查询数) */ + tableQueriesUsedUp: (n: number) => `${formatNumber(n)} 条查询(已用完)`, + unfinished: (n: number) => `${formatNumber(n)} 项没查成`, + skippedN: (n: number) => `跳过 ${formatNumber(n)} 项`, + expandLabel: (table: string) => `查看 ${table} 的检查清单和发过的查询`, + + // ---- 逐表展开、表详情:检查清单、发现、发过的查询 + loading: '正在载入…', + loadFailed: (reason: string) => `未能载入这张表的剖析结果:${reason}`, + checksTitle: (done: number, total: number) => `检查清单(${formatNumber(done)} / ${formatNumber(total)} 项)`, + checkState: { pending: '等待', running: '进行中', done: '完成', skipped: '跳过' } as Record, + checkQueries: (n: number) => `${formatNumber(n)} 条查询`, + rowsTitle: '行数', rowsStats: (n: number) => `约 ${formatNumber(n)} 行`, rowsCount: (n: number) => `${formatNumber(n)} 行`, rowsAtLeast: (n: number) => `超过 ${formatNumber(n - 1)} 行`, + rowsUnknown: '没能得到行数', rowsHint: { stats: '数据库统计信息中的估算值', count: '计数到整表统计行数上限为止' } as Record, + rowsSource: { stats: '来自数据库统计信息', count: '数到整表统计行数上限为止' } as Record, openTable: '打开', openTableLabel: (name: string) => `打开「${name}」`, - noFindings: '没有新的发现', findingRelations: '关联关系', findingCodes: '码值候选', findingDate: '业务日期', @@ -2887,52 +2992,101 @@ export const PROFILE_TEXT = { dateRanges: '日期列的取值范围', skippedTitle: (n: number) => `跳过 ${formatNumber(n)} 项`, moreSkipped: (n: number) => `…另有 ${formatNumber(n)} 项`, - tableError: '这张表未能剖析', - // ---- 无法开始(409) + queriesTitle: (n: number) => `看发过的查询(${formatNumber(n)} 条)`, + queriesHint: 'SQL 里的取值都换成了「?」:抽样到的键值不保存在 AgentLab 中;遮罩的列不出现在任何查询里', + queryKind: { row_estimate: '行数估算', relation: '关系', codes: '码值', date: '日期' } as Record, + queryOutcome: { ok: '成功', timeout: '超时', error: '失败', rejected: '未通过只读守卫' } as Record, + queryRows: (n: number) => `${formatNumber(n)} 行`, + queryMs: (ms: number) => `${formatNumber(ms)} 毫秒`, + + // ---- 结束时的提示(页面右下角,对话框关着时) + toast: { + succeeded: (text: string) => (text ? `数据剖析完成:${text}` : '数据剖析完成'), + partial: (n: number) => `数据剖析完成,${formatNumber(n)} 张表没做完`, + failed: (reason: string) => (reason ? `数据剖析失败:${reason}` : '数据剖析失败'), + stopped: (title: string) => `数据剖析已停止:${title}`, + cancelled: '数据剖析已停止', + interrupted: '服务重启,数据剖析中断', + }, + viewReport: '查看报告', + + // ---- 开始不了(409) blocked: { disabled: '未开启数据剖析', inactive: '数据源已停用', noSchema: '还没有表结构', - busy: '这个数据源正在剖析', + busy: '这个数据源已有剖析任务', tampered: '数据文件核对未通过', + nothing: '没有可剖析的表', other: '无法开始剖析', } as Record, blockedNext: { disabled: '开启后即可在这里剖析。', inactive: '启用后回到这里重试。', noSchema: '探查完成后回到这里重试。', - busy: '同一个数据源同一时间只能进行一次剖析。', + busy: '同一个数据源同一时间只有一个剖析任务:可以查看它的进度,等它结束后再开始新的剖析。', tampered: '请重新上传表格,或联系管理员核对数据文件。', + nothing: '可以改选「全部表」,或在表清单中选定要剖析的表。', other: '请稍后重试。', } as Record, + busyAt: (who: string | null, done: number, total: number) => + `${who ? `由「${who}」发起,` : ''}已完成 ${formatNumber(done)} / ${formatNumber(total)} 张表`, + viewActive: '查看它的进度', + chooseAll: '改选全部表', goSource: '前往数据源', - retry: '重试', - failedTitle: '剖析未能完成', + retryStart: '重试', + failedTitle: '剖析未能开始', backToSetup: '返回', + + // ---- 表清单的「剖析」一列和筛选 + listNever: '未剖析', + listNoChecks: '无可剖析项', + listNoChecksHint: '按当前的目录和表结构,这张表没有可剖析的项(关联关系候选、像码值的列、日期列)', + listAgo: (ago: string) => `${ago}剖析`, + listNewRelations: '有新关系待剖析', + listStatus: { + pending: '排队剖析', running: '剖析中', partial: '剖析部分完成', failed: '剖析失败', skipped: '剖析已跳过', cancelled: '剖析已停止', + } as Record, + listHint: (status: string, at: string) => `上次剖析(${at}):${status}`, + dueReason: { + never: '从没剖析过', + unfinished: '上次剖析没做完', + new_relations: '剖析之后目录里出现了新的待核实关联关系', + } as Record, + dueFilter: '需要剖析', + dueFilterHint: '从没剖析过、上次没做完,或剖析之后又出现了待核实关联关系的表', + + // ---- 表详情的「数据剖析」一节 + detailHint: '对业务库发只读查询得到的观测:行数、日期范围、核实了什么、跳过了什么', + detailNever: '这张表还没有剖析过。', + detailNoChecks: '按当前的目录和表结构,这张表没有可剖析的项(关联关系候选、像码值的列、日期列)。', + detailView: '视图不剖析。', + detailMissing: '表结构中已没有这张表,不能剖析。', + detailLast: (at: string) => `上次剖析:${at}`, + detailBy: (who: string) => `${who} 发起`, + detailJob: '查看这次剖析', + detailPending: '在剖析任务里排队,等待开始', + detailRunning: '正在剖析这张表', + profileThis: '剖析这张表', + profileThisHint: '新建一次剖析,只含这张表', + profileThisDisabled: '这个数据源未开启数据剖析,开启后才能剖析', + profileThisStarted: (table: string) => `已开始剖析 ${table}`, + profileThisQueued: (table: string) => `已排队剖析 ${table}:起草任务结束后自动开始`, + busyThisIn: (pos: number, total: number) => `这张表在其中,排第 ${formatNumber(pos)} / ${formatNumber(total)} 张`, + busyThisNotIn: '这张表不在其中,等它结束后再剖析这张表', + viewProgress: '查看进度', } -/** 整次剖析中途停下的原因 */ -export const PROFILE_STOP_LABEL: Record = { - budget: '查询次数已用完', - deadline: '总时长已用完', - failed: '连续多条查询失败', -} - -/** 停下之后怎么办 */ -export const PROFILE_STOP_NEXT: Record = { - budget: '其余检查未执行。可在数据剖析设置中调高查询次数上限,或缩小剖析范围后重新剖析。', - deadline: '其余检查未执行。可在数据剖析设置中调高总时长上限,或缩小剖析范围后重新剖析。', - failed: '其余检查未执行。请先在数据源卡片上测试连接,确认可用后重新剖析。', -} - -/** 跳过一项的原因(skipped.reason)。整句说明在 skipped.detail 里 */ +/** 跳过一项的原因(skipped.reason、检查项的 reason)。整句说明在 detail 里 */ export const PROFILE_SKIP_REASON_LABEL: Record = { - budget: '查询次数用完', - deadline: '总时长用完', + budget: '本表查询次数用完', + deadline: '本表时长用完', failed: '连续失败', timeout: '查询超时', error: '查询失败', - rejected: '未通过安全守卫', + rejected: '未通过只读守卫', + slow: '数据库响应慢', + cancelled: '已停止', masked: '列已遮罩', too_large: '表太大', view: '视图', @@ -2940,6 +3094,7 @@ export const PROFILE_SKIP_REASON_LABEL: Record = { no_data: '没有数据', missing: '表结构中没有', high_cardinality: '取值太多', + nothing: '没有可剖析的项', } /** 跳过的是哪一类检查(skipped.kind) */ diff --git a/frontend/src/pages/DataSourcesTab.tsx b/frontend/src/pages/DataSourcesTab.tsx index c7da8429..a7585768 100644 --- a/frontend/src/pages/DataSourcesTab.tsx +++ b/frontend/src/pages/DataSourcesTab.tsx @@ -536,7 +536,7 @@ function SourceCard({ row, meta, onChange, onRemoved, onEdit, onReupload, onReci )} {profile.enabled && ( - + {PROFILE_TEXT.chip} )} diff --git a/frontend/src/pages/catalog/CatalogPage.tsx b/frontend/src/pages/catalog/CatalogPage.tsx index 09a1ddc0..c8b0b738 100644 --- a/frontend/src/pages/catalog/CatalogPage.tsx +++ b/frontend/src/pages/catalog/CatalogPage.tsx @@ -6,30 +6,34 @@ import { } from 'lucide-react' import clsx from 'clsx' import { ApiError, api } from '../../api/client' -import type { CatalogDetail, CatalogList, CatalogReviewItem, CatalogReviewKind, CatalogTableRow } from '../../types' -import { EmptyState, ErrorState, PageHeader, Skeleton, Spinner, rovingTarget, toast, useTicker } from '../../components/ui' +import type { + CatalogDetail, CatalogList, CatalogProfileTaskStatus, CatalogReviewItem, CatalogReviewKind, CatalogTableRow, +} from '../../types' +import { EmptyState, ErrorState, PageHeader, Skeleton, Spinner, rovingTarget, toast } from '../../components/ui' import { useCatalog, useDatasources, useOnReconnect } from '../../store/catalog' import { errorMessage } from '../../lib/errors' import { formatNumber } from '../../lib/format' import { - CATALOG_DRAFT_TEXT as DT, CATALOG_REVIEW_TEXT as RT, CATALOG_TEXT as CT, CATALOG_TRANSFER_TEXT as XT, PROFILE_STOP_LABEL, - PROFILE_TEXT as PT, SCHEMA_PARTIAL_TEXT as SPT, + CATALOG_DRAFT_TEXT as DT, CATALOG_REVIEW_TEXT as RT, CATALOG_TEXT as CT, CATALOG_TRANSFER_TEXT as XT, PROFILE_TEXT as PT, + SCHEMA_PARTIAL_TEXT as SPT, } from '../../lib/terms' import { StatusLegend, SystemNotesNotice } from './parts' import { DraftDialog } from './DraftDialog' -import { isJobActive, jobBlank, retryableCount, useDraftJob } from './draftJob' +import { isJobActive, jobBlank, retryableCount, useDraftJob, waitsForProfile } from './draftJob' import type { DraftPreset, DraftStartBody } from './draftJob' import { ProfileDialog } from './ProfileDialog' -import type { ProfileJob } from './ProfileDialog' import { ProfileSettingsDialog } from './ProfileSettings' -import { fillKey, profileBlockOf, profileSettingsOf, secondsText } from './profile' +import { fillKey, profileSettingsOf } from './profile' +import { retryableTasks, useProfileJob } from './profileJob' +import type { ProfileStartBody } from './profileJob' import { TableDetail } from './TableDetail' +import type { TableProfileProps } from './TableDetail' import { TableIndex } from './TableIndex' import { ReviewOverview, ReviewQueue } from './ReviewQueue' import { ImportDialog } from './TransferDialog' import { itemKey, reviewParamsOf, writeReviewParams } from './review' import type { CatalogView } from './review' -import { filterCounts, listParamsOf, progressOf, rowFromDetail, usageSummary, visibleRows, writeListParams } from './model' +import { dueCount, filterCounts, listParamsOf, progressOf, rowFromDetail, usageSummary, visibleRows, writeListParams } from './model' import type { ListFilter, ListParams, ListSort } from './model' // =========================================================================== @@ -47,6 +51,10 @@ import type { ListFilter, ListParams, ListSort } from './model' // 左栏有两个视图:表清单(逐表看)和审阅队列(view=review:跨表按优先级列出推断项,成批确认、驳回,筛选记在 kind / // rtable 里)。从队列点开一项,右栏是所在表的详情,地址带着那一项(item),详情滚到那一项、列出用过这张表的运行。 // 页头的「导出目录」「导入目录」把整理好的目录带到另一个实例(导入先预览,确认后才写)。 +// +// 起草、剖析都是服务端的后台任务(draftJob.ts、profileJob.ts),状态在这一页上:关掉对话框照常进行,页头的「起草」「数据 +// 剖析」换成进度入口(「起草 3/20」「剖析 12/40」「剖析 排队中」),点开对应的对话框;结束时右下角提示,可以回到结果。 +// 同一个数据源的起草和剖析按建立顺序排队:一个在排队、进行中时,另一个建出来是排队,前一个结束后自动开始。 // =========================================================================== export function CatalogPage() { @@ -107,10 +115,13 @@ export function CatalogPage() { const { query, filter, sort } = list /** 只看运行中查询过的表:从顶部摘要点进来时打开,筛选结果和摘要的数字一致 */ const usedOnly = list.used + /** 只看需要剖析的表(和剖析范围「需要剖析的表」同一个口径) */ + const dueOnly = !!list.due const setQuery = useCallback((q: string) => { typedAt.current = Date.now(); setList((c) => ({ ...c, query: q })) }, []) const setFilter = useCallback((f: ListFilter) => setList((c) => ({ ...c, filter: f })), []) const setSort = useCallback((v: ListSort) => setList((c) => ({ ...c, sort: v })), []) const setUsedOnly = useCallback((v: boolean) => setList((c) => ({ ...c, used: v })), []) + const setDueOnly = useCallback((v: boolean) => setList((c) => ({ ...c, due: v })), []) /** * 跳到别的表、回到清单时带上清单的查询参数(按页面上的算:搜索词可能还没写进地址)。审阅视图的参数(view、kind、rtable) * 照旧带着,从审阅队列打开的那一项(item)不带:换了表就不是那一项了 @@ -133,21 +144,25 @@ export function CatalogPage() { const [draftOpen, setDraftOpen] = useState<{ view: 'setup' | 'job'; busy?: boolean; preset?: DraftPreset } | null>(null) /** 起草写进了这些表:详情据此重新载入(编辑中的不动,保存时服务端会用版本号拦下) */ const [drafted, setDrafted] = useState<{ seq: number; tables: Set }>({ seq: 0, tables: new Set() }) - const [profiling, setProfiling] = useState(false) + /** + * 剖析对话框开着没有、打开时显示什么:setup 选范围开始;job 显示页面跟着的任务(进行中的进度、刚结束的结果)。 + * scope:从表清单的「剖析所选」打开,范围默认是已选的表 + */ + const [profileOpen, setProfileOpen] = useState<{ view: 'setup' | 'job'; scope?: 'selected' } | null>(null) const [profileSettingsOpen, setProfileSettingsOpen] = useState(false) - /** 一次剖析的状态。关掉弹窗剖析照常进行:状态留在页面上,页头有入口,完成后提示并可再打开报告 */ - const [job, setJob] = useState<(ProfileJob & { sourceId: string }) | null>(null) /** 剖析设置刚存过:数据源列表重取回来之前按存下的这份算(预算、开没开) */ const [savedOptions, setSavedOptions] = useState<{ sourceId: string; options: Record } | null>(null) /** - * 从剖析报告点「填写含义」:报告先收起,打开那张表并弹出那一列的码值;码值弹窗关上(保存或取消)后回到报告, - * 焦点落在下一列的「填写含义」上,一列接一列地填 + * 从剖析结果点「填写含义」:对话框先收起,打开那张表并弹出那一列的码值;码值弹窗关上(保存或取消)后回到结果, + * 那张表照旧展开,焦点落在下一列的「填写含义」上,一列接一列地填 */ const [fillCodes, setFillCodes] = useState<{ table: string; column: string; seq: number } | null>(null) - /** 回到报告时从哪一列接着往下 */ + /** 回到结果时从哪一列接着往下 */ const [returnTo, setReturnTo] = useState<{ table: string; column: string } | null>(null) - const profilingRef = useRef(profiling) - profilingRef.current = profiling + /** 从结果去填了含义的码值列:保存后还剩几个待填写(按任务记,换了任务就不算) */ + const [filled, setFilled] = useState<{ jobId: string; map: Record } | null>(null) + const profileOpenRef = useRef(profileOpen) + profileOpenRef.current = profileOpen const alive = useRef(true) // 开发模式的严格模式会先卸载再挂载一次:挂载时要重新置为真 useEffect(() => { @@ -158,8 +173,9 @@ export function CatalogPage() { sourceRef.current = sourceId const rows = useMemo(() => data?.tables ?? [], [data]) - const visible = useMemo(() => visibleRows(rows, query, filter, sort, usedOnly), [rows, query, filter, sort, usedOnly]) - const counts = useMemo(() => filterCounts(rows, query, usedOnly), [rows, query, usedOnly]) + const visible = useMemo(() => visibleRows(rows, query, filter, sort, usedOnly, dueOnly), [rows, query, filter, sort, usedOnly, dueOnly]) + const counts = useMemo(() => filterCounts(rows, query, usedOnly, dueOnly), [rows, query, usedOnly, dueOnly]) + const dueTotal = useMemo(() => dueCount(rows, query, usedOnly), [rows, query, usedOnly]) const base = `/data/catalog/${encodeURIComponent(sourceId)}` const openTable = useCallback((t: string) => navigate(`${base}/${encodeURIComponent(t)}${searchRef.current}`), [navigate, base]) const backToList = useCallback(() => navigate(`${base}${searchRef.current}`), [navigate, base]) @@ -232,7 +248,7 @@ export function CatalogPage() { }, []) // 上方的进度:跟着的任务在进行就按它;跟着的就是清单说的那一个、已经结束了,就不信清单(清单还没重取回来) const listActive = isJobActive(data?.active_job) ? data?.active_job ?? null : null - const stripJob: { done: number; total: number; status: string } | null = draftJob && (draftActive || draftJob.id === listActive?.id) + const stripJob: { done: number; total: number; status: string; waiting_for?: string | null } | null = draftJob && (draftActive || draftJob.id === listActive?.id) ? (draftActive ? draftJob : null) : listActive // 调用接连没写出来的提醒:关着对话框时也要看得到(上方的进度带着它) @@ -240,87 +256,105 @@ export function CatalogPage() { const options = savedOptions?.sourceId === sourceId ? savedOptions.options : source?.options const profileSettings = useMemo(() => profileSettingsOf(options), [options]) - const myJob = job?.sourceId === sourceId ? job : null - const jobRunning = myJob?.phase === 'running' - useTicker(1000, jobRunning) /** - * 剖析:同步请求,做完才返回。期间关掉弹窗不影响;回来时页面已经换了数据源或卸载了,只提示、不再改这一页。 - * 写进去的表重新载入清单和正在看的详情(编辑中的不动,同起草) + * 剖析任务(后台进行,状态在页面上):每张表一结束就重取清单、正在看的表在里面就重新载入(编辑中的不动,保存时服务端 + * 会用版本号拦下);结束时重取一次清单。对话框关着时结束的,右下角提示并给「查看报告」 */ - const startProfile = useCallback(async (tables: string[] | null) => { + const profile = useProfileJob(sourceId, data?.active_profile_job, { + onTablesEnded: (tables) => { + setDrafted((cur) => ({ seq: cur.seq + 1, tables: new Set(tables) })) + void load() + }, + onFinished: (job) => { + void load() + if (profileOpenRef.current) return + const src = job.source_id + const opts = { + key: `catalog-profile:${src}`, duration: 10000, + action: { label: PT.viewReport, onClick: () => { if (sourceRef.current === src) setProfileOpen({ view: 'job' }) } }, + } + if (job.stop_reason) toast.warn(PT.toast.stopped(PT.stopShort[job.stop_reason] ?? ''), opts) + else if (job.status === 'succeeded') toast.ok(PT.toast.succeeded(PT.findings(job.summary)), opts) + else if (job.status === 'partial') toast.warn(PT.toast.partial(retryableTasks(job).length), opts) + else if (job.status === 'failed') toast.error(PT.toast.failed(job.error ?? ''), opts) + else if (job.status === 'interrupted') toast.warn(PT.toast.interrupted, opts) + else toast.warn(PT.toast.cancelled, opts) + }, + }) + const profileJob = profile.job + const profileActive = isJobActive(profileJob) + // 页头的剖析进度:同起草,跟着的任务在进行就按它;跟着的就是清单说的那一个、已经结束了,就不信清单 + const listProfile = isJobActive(data?.active_profile_job) ? data?.active_profile_job ?? null : null + const profileStrip: { done: number; total: number; status: string; waiting_for?: string | null } | null = + profileJob && (profileActive || profileJob.id === listProfile?.id) ? (profileActive ? profileJob : null) : listProfile + /** 跟着的剖析任务里各表的状态(表清单的「剖析」一格据此写排队、剖析中) */ + const profileLive = useMemo(() => new Map( + profileActive && profileJob ? profileJob.tasks.map((t) => [t.table_name, t.status]) : []), [profileActive, profileJob]) + /** 打开剖析对话框:有进行中的任务就直接显示它;上一次的结果看过了,再点就是重新开始 */ + const openProfile = useCallback(() => setProfileOpen({ view: profileActive ? 'job' : 'setup' }), [profileActive]) + const openProfileSelected = useCallback(() => setProfileOpen({ view: profileActive ? 'job' : 'setup', scope: 'selected' }), [profileActive]) + const startProfile = useCallback(async (body: ProfileStartBody) => { + const next = await profile.start(body) + // 选中的表已经交给剖析任务了:清空,免得接着点「剖析所选」又剖析一遍 + if (body.tables) setSelected(new Set()) + return next + }, [profile.start]) + /** 表详情里「剖析这张表」:只含这张表 */ + const profileTable = useCallback((t: string) => profile.start({ tables: [t] }), [profile.start]) + /** 打开剖析对话框显示某个任务:页面跟着的那一个直接打开;别的(表详情里「查看这次剖析」)先取来 */ + const openProfileJob = useCallback(async (jobId: string | null) => { + if (!jobId || profile.job?.id === jobId) { + setProfileOpen({ view: 'job' }) + return + } const src = sourceId - const startedAt = Date.now() - setJob({ sourceId: src, phase: 'running', startedAt, tables, settings: profileSettings }) - let next: ProfileJob try { - const report = await api.dataCatalog.profile(src, tables ? { tables } : {}) - next = { phase: 'done', report, ms: Date.now() - startedAt } + const { job } = await api.dataCatalog.profileJobs.get(src, jobId) + if (sourceRef.current !== src) return + profile.track(job) + setProfileOpen({ view: 'job' }) } catch (e) { - next = e instanceof ApiError && e.status === 409 - ? { phase: 'blocked', kind: profileBlockOf(e), message: e.message, tables } - : { phase: 'failed', error: e, tables } + toast.error(e) } - const here = alive.current && sourceRef.current === src - if (here) setJob({ ...next, sourceId: src }) - if (next.phase === 'done') { - const r = next.report - const touched = r.tables.filter((t) => !t.error && t.added + t.updated + t.removed > 0).map((t) => t.table_name) - if (here) { - if (touched.length) setDrafted((cur) => ({ seq: cur.seq + 1, tables: new Set(touched) })) - void load() - } - // 弹窗开着就看报告;关了(在后台继续)才提示,并给「查看报告」 - if (!here || !profilingRef.current) { - const summary = PT.summary(r.queries_used, r.settings.max_queries, r.tables.length) - const text = r.stopped ? PT.toastStopped(PROFILE_STOP_LABEL[r.stopped] ?? r.stopped) : PT.toastDone(summary) - const show = r.stopped ? toast.warn : toast.ok - show(text, here ? { key: `profile:${src}`, duration: 10000, action: { label: PT.viewReport, onClick: () => setProfiling(true) } } - : { key: `profile:${src}` }) - } - } else if (!here || !profilingRef.current) { - toast.error(next.phase === 'blocked' ? next.message : next.error, here - ? { key: `profile:${src}`, action: { label: PT.viewReport, onClick: () => setProfiling(true) } } : { key: `profile:${src}` }) - } - }, [sourceId, profileSettings, load]) - - const openProfile = useCallback(() => { - // 上一次的结果(报告、被拒)看过了,再点就是重新开始;正在剖析时打开的是进度 - setJob((cur) => (cur && cur.phase === 'running' && cur.sourceId === sourceId ? cur : null)) - setProfiling(true) - }, [sourceId]) + }, [sourceId, profile.job?.id, profile.track]) + const tableProfile: TableProfileProps = useMemo(() => ({ + job: profileJob, onStart: profileTable, onShowActive: profile.showActive, onSettings: () => setProfileSettingsOpen(true), + onOpenJob: (id: string | null) => void openProfileJob(id), + }), [profileJob, profileTable, profile.showActive, openProfileJob]) const onProfileSettingsSaved = useCallback((row: any) => { setProfileSettingsOpen(false) setSavedOptions({ sourceId, options: row?.options ?? {} }) void useCatalog.getState().reload('datasources') - // 因为没开启被拒的:开了就回到选范围,可以直接开始 - setJob((cur) => (cur && cur.phase === 'blocked' && cur.kind === 'disabled' ? null : cur)) }, [sourceId]) const openFromReport = useCallback((t: string) => { - setProfiling(false) + setProfileOpen(null) openTable(t) }, [openTable]) const fillFromReport = useCallback((t: string, column: string) => { - setProfiling(false) + setProfileOpen(null) setFillCodes((cur) => ({ table: t, column, seq: (cur?.seq ?? 0) + 1 })) openTable(t) }, [openTable]) - const jobRef = useRef(job) - jobRef.current = job - /** 从报告打开的码值弹窗关上了:记下这一列还剩几个含义待填写(报告里跟着改),回到报告 */ + const profileJobRef = useRef(profileJob) + profileJobRef.current = profileJob + /** + * 码值弹窗关上了:是从剖析结果打开的,就记下这一列还剩几个含义待填写(结果里跟着改),回到结果。表详情的「数据剖析」 + * 一节里点的「填写含义」不回对话框 + */ const onFillDone = useCallback((seq: number, pending: number | null) => { const fill = fillCodes?.seq === seq ? fillCodes : null - const cur = jobRef.current - if (!fill || !alive.current || cur?.phase !== 'done' || cur.sourceId !== sourceRef.current) return + const cur = profileJobRef.current + if (!fill || !alive.current || !cur || cur.source_id !== sourceRef.current) return if (pending != null) { - setJob((j) => (j && j.phase === 'done' ? { ...j, filled: { ...j.filled, [fillKey(fill.table, fill.column)]: pending } } : j)) + setFilled((f) => ({ jobId: cur.id, map: { ...(f?.jobId === cur.id ? f.map : {}), [fillKey(fill.table, fill.column)]: pending } })) } setReturnTo({ table: fill.table, column: fill.column }) - setProfiling(true) + setProfileOpen({ view: 'job' }) }, [fillCodes]) // 上一张 / 下一张按清单当前的筛选和排序走;正在看的表不在筛选结果里(刚确认完、从关联关系跳过来)时按全部表走 @@ -394,7 +428,7 @@ export function CatalogPage() { } }, [table, navigate, base, params]) - const filtered = !!query.trim() || filter !== 'all' || usedOnly + const filtered = !!query.trim() || filter !== 'all' || usedOnly || dueOnly const empty = rows.length > 0 && rows.every((r) => progressOf(r.counts) === 'none') const hasTables = rows.length > 0 const draftButton = (primary = false) => ( @@ -402,23 +436,32 @@ export function CatalogPage() { {CT.draft} ) - // 页头的「数据剖析」:剖析进行中换成计时,点开看进度(关掉弹窗剖析照常进行) - const headerProfile = jobRunning && myJob?.phase === 'running' - ? ( - - ) - : ( - - ) - // 页头的「起草」窄屏只留图标,标题才放得下 + // 页头的「数据剖析」「起草」:任务在排队、进行中时换成进度入口(「剖析 12/40」「剖析 排队中」),点开对应的对话框 + // (关掉对话框任务照常进行);排在另一种任务后面时悬停写明。窄屏只留图标,标题才放得下(进度在读屏的名称里) + const profileQueued = profileStrip?.status === 'queued' + const profileLabel = profileStrip ? (profileQueued ? PT.headerQueued : PT.headerRunning(profileStrip.done, profileStrip.total)) : PT.action + const profileTitle = profileStrip + ? (profileQueued ? (profileStrip.waiting_for === 'draft' ? PT.headerQueuedDraft : PT.headerQueuedHint) : PT.headerRunningHint(profileStrip.done, profileStrip.total)) + : PT.actionHint + // 最近查询多数超时的提醒:对话框关着时也要看得到(页头的入口换成提醒色,悬停写全文) + const profileNotice = profileActive ? profileJob?.notice?.trim() || null : null + const headerProfile = ( + + ) + const draftQueued = stripJob?.status === 'queued' + const draftLabel = stripJob ? (draftQueued ? DT.headerQueued : DT.headerRunning(stripJob.done, stripJob.total)) : CT.draft + const draftTitle = stripJob + ? (draftQueued ? (waitsForProfile(stripJob) ? DT.headerQueuedProfile : DT.headerQueuedHint) : DT.headerRunningHint(stripJob.done, stripJob.total)) + : CT.draft const headerDraft = ( - ) @@ -481,7 +524,12 @@ export function CatalogPage() { active={table} onOpen={openTable} onDraft={openDraft} - onProfile={openProfile} + onProfile={openProfileSelected} + profileOn={profileSettings.enabled} + profileLive={profileLive} + dueOnly={dueOnly} + onDueOnly={setDueOnly} + dueTotal={dueTotal} /> )} @@ -493,7 +541,8 @@ export function CatalogPage() { + profileEnabled={profileSettings.enabled} focusPath={inReview ? review.item || null : null} usageOpen={inReview} + profile={tableProfile} /> ) : inReview ? @@ -554,6 +603,7 @@ export function CatalogPage() { initialBusy={!!draftOpen.busy} preset={draftOpen.preset ?? null} stopping={draft.stopping} + profileActive={!!profileStrip} onStart={startDraft} onCancel={draft.cancel} onRetry={draft.retry} @@ -562,23 +612,32 @@ export function CatalogPage() { onClose={() => setDraftOpen(null)} /> )} - {profiling && data && ( + {profileOpen && data && ( void startProfile(tables)} - onClose={() => { setProfiling(false); setReturnTo(null) }} + job={profileJob} + initial={profileOpen.view} + initialScope={profileOpen.scope ?? null} + draftActive={!!stripJob} + stopping={profile.stopping} + filled={filled && filled.jobId === profileJob?.id ? filled.map : {}} returnTo={returnTo} + onStart={startProfile} + onCancel={profile.cancel} + onRetry={profile.retry} + onShowActive={profile.showActive} + onTrack={profile.track} + onClose={() => { setProfileOpen(null); setReturnTo(null) }} onSettings={() => setProfileSettingsOpen(true)} onGoSource={() => navigate(back)} onOpenTable={openFromReport} onFillCodes={fillFromReport} - onReset={() => setJob(null)} /> )} {importing && ( @@ -597,11 +656,12 @@ export function CatalogPage() { * notice:调用接连没写出来时服务端给的提醒,跟着写一行(全文在悬停里,对话框里也有) */ function DraftStrip({ job, notice, onOpen }: { - job: { done: number; total: number; status: string } + job: { done: number; total: number; status: string; waiting_for?: string | null } notice: string | null onOpen: () => void }) { - const text = job.status === 'queued' ? DT.stripQueued(job.total) : DT.strip(job.done, job.total) + // 排在剖析任务后面(同一个数据源的起草和剖析按建立顺序排队):写明剖析结束后自动开始 + const text = job.status === 'queued' ? (waitsForProfile(job) ? DT.stripQueuedProfile(job.total) : DT.stripQueued(job.total)) : DT.strip(job.done, job.total) return (
export function DraftDialog({ sourceId, sourceDescription, guidance, rows, visible, selected, filtered, job, initial, initialBusy = false, preset: initialPreset = null, - stopping, onStart, onCancel, onRetry, onTrack, onGuidanceSaved, onClose, + stopping, profileActive = false, onStart, onCancel, onRetry, onTrack, onGuidanceSaved, onClose, }: { sourceId: string /** 数据源说明:用模型起草时连同起草要求一起交给模型 */ @@ -74,6 +77,8 @@ export function DraftDialog({ preset?: DraftPreset | null /** 点了停止、服务端还没停下来 */ stopping: boolean + /** 同一个数据源有剖析任务在排队、进行中:起草会排在它后面 */ + profileActive?: boolean /** 建任务;409 已有任务时页面改取进行中的那一个(busy 为真) */ onStart: (body: DraftStartBody) => Promise<{ job: CatalogDraftJob; busy: boolean }> onCancel: () => Promise @@ -367,6 +372,11 @@ export function DraftDialog({
)} + {profileActive && ( +

+ {DT.queueHint} +

+ )} {startError && {startError}} )} @@ -451,6 +461,11 @@ function JobView({ job, running, stopping, busy, blankTables, onRedraftBlank }: aria-valuemin={0} aria-valuemax={job.total} aria-valuenow={value}>
+ {waitsForProfile(job) && ( +

+ {DT.waitingProfile} +

+ )} {notice && {notice}}

{summary}

{(job.model || usage.calls > 0 || failures) && ( diff --git a/frontend/src/pages/catalog/ProfileDialog.tsx b/frontend/src/pages/catalog/ProfileDialog.tsx index 028f2d65..c3ad94e1 100644 --- a/frontend/src/pages/catalog/ProfileDialog.tsx +++ b/frontend/src/pages/catalog/ProfileDialog.tsx @@ -1,60 +1,90 @@ import { useEffect, useMemo, useRef, useState } from 'react' import type { ReactNode } from 'react' -import { ArrowDown, ArrowUp, ArrowUpRight, CircleCheck, Equal, Minus, OctagonX, ScanSearch, ShieldAlert, TriangleAlert, UserCheck } from 'lucide-react' +import { + ArrowDown, ArrowUp, ArrowUpRight, ChevronRight, CircleCheck, CircleDashed, CircleMinus, CircleSlash, CircleX, Equal, Info, Minus, + RotateCcw, ScanSearch, ShieldAlert, TriangleAlert, UserCheck, +} from 'lucide-react' import type { LucideIcon } from 'lucide-react' import clsx from 'clsx' +import { ApiError, api } from '../../api/client' import type { - CatalogProfileCodesFinding, CatalogProfileDateFinding, CatalogProfileOut, CatalogProfileRelationFinding, CatalogProfileSettings, - CatalogProfileSize, CatalogProfileSkip, CatalogProfileTable, CatalogTableRow, + CatalogProfileCheck, CatalogProfileCodesFinding, CatalogProfileDateFinding, CatalogProfileJob, CatalogProfileQuery, + CatalogProfileRelationFinding, CatalogProfileSettings, CatalogProfileSkip, CatalogProfileTaskDetail, CatalogProfileTaskRow, + CatalogProfileTaskStatus, CatalogTableRow, } from '../../types' -import { ErrorState, Modal, Notice, Spinner, useTicker } from '../../components/ui' +import { Modal, Notice, Spinner, useRadioGroup, useTicker } from '../../components/ui' +import { errorMessage } from '../../lib/errors' +import { formatDateTime, formatNumber, formatTime } from '../../lib/format' import { - CATALOG_CARDINALITY_LABEL, CATALOG_TEXT as CT, CATALOG_UI_TEXT as UT, PROFILE_SKIP_KIND_LABEL, PROFILE_SKIP_REASON_LABEL, PROFILE_STOP_LABEL, PROFILE_STOP_NEXT, PROFILE_TEXT as PT, + CATALOG_CARDINALITY_LABEL, CATALOG_TEXT as CT, CATALOG_UI_TEXT as UT, PROFILE_SKIP_KIND_LABEL, PROFILE_SKIP_REASON_LABEL, PROFILE_TEXT as PT, } from '../../lib/terms' import { STATUS_TONE, StatusChip } from './parts' -import { PROFILE_DEFAULT_TABLES, PROFILE_MAX_TABLES, coverageText, fillKey, secondsText } from './profile' +import { + PROFILE_MAX_TABLES, PROFILE_RETRYABLE, UNFINISHED_REASONS, coverageText, fillKey, isProfilable, profileBlockOf, profileScopeTables, + secondsText, sizeText, spanText, +} from './profile' import type { ProfileBlock } from './profile' +import { isJobActive } from './draftJob' +import { isProfileBusy, retryableTasks } from './profileJob' +import type { ProfileStartBody } from './profileJob' // =========================================================================== -// 数据目录页的「数据剖析」。接口是同步的:一次请求做完整个剖析才返回,服务端不支持中途停止,所以这里不给「停止」, -// 进行中只写已用时长和总时长上限,并明说无法中途停止。关掉窗口剖析照常进行(状态在页面上,页头有进度入口), -// 完成后弹提示、可以再打开报告。 +// 数据目录页的「数据剖析」。剖析是服务端的后台任务(profileJob.ts):这里选范围、看预算、建任务;之后显示页面跟着的 +// 那个任务(进行中的逐表进度、结束后的结果),任务本身的状态在页面上,关掉对话框照常进行。照起草对话框的结构: // -// 四段:选范围、看预算、确认开始 → 进行中 → 报告(用了几条查询、为什么停、每张表的发现和跳过的项); -// 开始不了(409)时写明是哪一种情况和下一步,未开启的就地打开剖析设置。 +// 开始前:范围(已选 / 当前筛选 / 需要剖析的表(默认)/ 全部表,个数按清单上每行的 profile_due、profile_checks 算)、按表写的 +// 预算和这次最多约几条查询、醒目一句「只读查询、一条一条发、可随时停止」;同一个数据源有起草任务时按钮写「排队剖析」。 +// 进行中:几张 / 共几张、已发几条查询、用时,按表数的真进度条,停止;逐表一行(状态、各类检查的进度、查询数、用时、正在做 +// 哪一项),点一行展开这张表的检查清单、发现和发过的查询。 +// 结束后:汇总(各状态几张、核实了什么)、止损和停止的原因照实写、按「有发现 / 有跳过 / 失败」筛、重试没做完的表;码值的 +// 「填写含义」照旧(填完回到这里,接着下一列)。 +// 开始不了(409):每种情况一句说法和下一步(未开启就地打开剖析设置;正忙写清是谁的、到哪了,可以去看它的进度)。 // =========================================================================== -/** 一次剖析的状态。放在页面上而不是弹窗里:关掉弹窗剖析照常进行,完成后还能再打开报告 */ -export type ProfileJob = - | { phase: 'running'; startedAt: number; tables: string[] | null; settings: CatalogProfileSettings } - /** filled:从报告去填了含义的码值列(fillKey)→ 保存后还剩几个含义待填写,报告里按它改写那一列 */ - | { phase: 'done'; report: CatalogProfileOut; ms: number; filled?: Record } - | { phase: 'blocked'; kind: ProfileBlock; message: string; tables: string[] | null } - | { phase: 'failed'; error: unknown; tables: string[] | null } +type Scope = 'selected' | 'filtered' | 'needed' | 'all' +type Filter = 'all' | 'findings' | 'skipped' | 'failed' +const FILTERS: readonly Filter[] = ['all', 'findings', 'skipped', 'failed'] -type Scope = 'selected' | 'filtered' | 'default' - -/** 报告里每类最多逐条列几项,其余合成一句 */ +/** 结果里每类最多逐条列几项,其余折起来 */ const SKIP_SHOWN = 6 const CODES_SHOWN = 12 export function ProfileDialog({ - sourceName, settings, rows, visible, selected, filtered, job, returnTo, onStart, onClose, onSettings, onGoSource, onOpenTable, onFillCodes, - onReset, + sourceId, sourceName, settings, rows, visible, selected, filtered, job, initial, initialScope = null, draftActive, stopping, filled, + returnTo = null, onStart, onCancel, onRetry, onShowActive, onTrack, onClose, onSettings, onGoSource, onOpenTable, onFillCodes, }: { + sourceId: string sourceName: string /** 这个数据源此刻的剖析设置(数据源列表里读的) */ settings: CatalogProfileSettings + /** 全部表(服务端的顺序) */ rows: CatalogTableRow[] + /** 当前筛选结果 */ visible: CatalogTableRow[] selected: string[] + /** 清单上有没有生效的搜索或筛选:有时才给「当前筛选结果」这一项 */ filtered: boolean - /** null:还没开始(选范围) */ - job: ProfileJob | null - /** 填完一列码值的含义、回到报告:焦点落在下一列还有待填写的「填写含义」上 */ + /** 页面跟着的剖析任务(进行中的、刚结束的) */ + job: CatalogProfileJob | null + /** 打开时显示什么:setup 选范围、开始;job 显示页面跟着的那个任务 */ + initial: 'setup' | 'job' + /** 从表清单的「剖析所选」打开:范围默认是已选的表 */ + initialScope?: 'selected' | null + /** 同一个数据源有起草任务在排队、进行中:剖析会排在它后面 */ + draftActive: boolean + /** 点了停止、服务端还没停下来 */ + stopping: boolean + /** 从结果去填了含义的码值列(fillKey)→ 保存后还剩几个含义待填写 */ + filled: Record + /** 填完一列码值的含义、回到结果:展开那张表,焦点落在下一列还有待填写的「填写含义」上 */ returnTo?: { table: string; column: string } | null - /** tables 为 null 时由服务端挑表 */ - onStart: (tables: string[] | null) => void + onStart: (body: ProfileStartBody) => Promise + onCancel: () => Promise + onRetry: (from: CatalogProfileJob) => Promise + /** 正忙(409)时改去跟着进行中的那一个 */ + onShowActive: () => Promise + /** 设置阶段点开了上一次剖析的结果(或者发现别处建的进行中的任务):页面改跟这个任务 */ + onTrack: (job: CatalogProfileJob) => void onClose: () => void /** 打开这个数据源的剖析设置 */ onSettings: () => void @@ -62,29 +92,156 @@ export function ProfileDialog({ onGoSource: () => void onOpenTable: (table: string) => void onFillCodes: (table: string, column: string) => void - /** 回到选范围 */ - onReset: () => void }) { - const live = useMemo(() => rows.filter((r) => r.in_schema).map((r) => r.table_name), [rows]) const scopes = useMemo(() => { - const liveSet = new Set(live) - const out: { key: Scope; label: string; hint?: string; tables: string[] | null }[] = [] - const sel = selected.filter((t) => liveSet.has(t)) - if (sel.length) out.push({ key: 'selected', label: PT.scopeSelected(sel.length), tables: sel }) - const vis = visible.filter((r) => r.in_schema).map((r) => r.table_name) - if (filtered && vis.length && vis.length !== live.length) out.push({ key: 'filtered', label: PT.scopeFiltered(vis.length), tables: vis }) - out.push({ key: 'default', label: PT.scopeDefault, hint: PT.scopeDefaultHint(PROFILE_DEFAULT_TABLES), tables: null }) + const out: { key: Scope; label: string; hint?: string; tables: string[]; leftOut: number; body: ProfileStartBody }[] = [] + const byName = new Map(rows.map((r) => [r.table_name, r])) + // 已选、筛选结果里剖析不了的表(视图、表结构里没有、没有可剖析的项)不交给服务端,写明有几张 + const explicit = (names: string[]) => { + const tables = names.filter((n) => { const r = byName.get(n); return !!r && isProfilable(r) }) + return { tables, leftOut: names.length - tables.length } + } + if (selected.length) { + const s = explicit(selected) + out.push({ key: 'selected', label: PT.scopeSelected(s.tables.length), ...s, body: { tables: s.tables } }) + } + if (filtered && visible.length && visible.length !== rows.length) { + const f = explicit(visible.map((r) => r.table_name)) + out.push({ key: 'filtered', label: PT.scopeFiltered(f.tables.length), ...f, body: { tables: f.tables } }) + } + const needed = profileScopeTables(rows, 'needed') + out.push({ key: 'needed', label: PT.scopeNeeded(needed.length), hint: needed.length ? PT.scopeNeededHint : PT.scopeNeededNone, + tables: needed, leftOut: 0, body: { scope: 'needed' } }) + const all = profileScopeTables(rows, 'all') + out.push({ key: 'all', label: PT.scopeAll(all.length), hint: PT.scopeAllHint, tables: all, leftOut: 0, body: { scope: 'all' } }) return out - }, [live, selected, visible, filtered]) - const [scope, setScope] = useState(() => scopes[0].key) + }, [rows, visible, selected, filtered]) + const [scope, setScope] = useState(() => { + if (initialScope === 'selected' && scopes.some((s) => s.key === 'selected')) return 'selected' + // 默认「需要剖析的表」;一张都没有时选「全部表」(选一个开始不了的范围没有意义) + return scopes.find((s) => s.key === 'needed')!.tables.length ? 'needed' : 'all' + }) const picked = scopes.find((s) => s.key === scope) ?? scopes[0] - const tooMany = (picked.tables?.length ?? 0) > PROFILE_MAX_TABLES - const running = job?.phase === 'running' - useTicker(1000, running) + const tooMany = !!picked.body.tables && picked.tables.length > PROFILE_MAX_TABLES + + const [mode, setMode] = useState<'setup' | 'job' | 'blocked'>(initial === 'job' && job ? 'job' : 'setup') + const [blocked, setBlocked] = useState<{ kind: ProfileBlock; message: string; active: CatalogProfileJob | null; body: ProfileStartBody } | null>(null) + const [starting, setStarting] = useState(false) + const [startError, setStartError] = useState(null) + /** 是 409(正忙)转过来显示的那一个 */ + const [busy, setBusy] = useState(false) + const [acting, setActing] = useState(false) + const [actionError, setActionError] = useState(null) + /** 上一次剖析(没做完的才提),设置阶段写一行、可以点开 */ + const [last, setLast] = useState(null) + const onTrackRef = useRef(onTrack) + onTrackRef.current = onTrack + + const inSetup = mode === 'setup' || (mode === 'job' && !job) + // 设置阶段:取上一次剖析。别处(另一个标签页、表详情)建的任务还在进行:直接显示它 + useEffect(() => { + if (!inSetup) return + let alive = true + api.dataCatalog.profileJobs.latest(sourceId).then(({ job: latest }) => { + if (!alive || !latest) return + if (isJobActive(latest)) { + onTrackRef.current(latest) + setBusy(true) + setMode('job') + } else if (latest.status !== 'succeeded') { + setLast(latest) + } + }, () => {}) + return () => { alive = false } + }, [inSetup, sourceId]) + + /** 开始不了:409 各有说法;正忙时顺手取进行中的那一个(写清是谁的、到哪了);别的错误留在设置里写明 */ + const fail = async (e: unknown, body: ProfileStartBody) => { + if (e instanceof ApiError && e.status === 409) { + const kind = profileBlockOf(e) + const active = isProfileBusy(e) ? await api.dataCatalog.profileJobs.active(sourceId).then((r) => r.job, () => null) : null + setBlocked({ kind, message: e.message, active, body }) + setMode('blocked') + } else { + setStartError(PT.startFailed(errorMessage(e))) + } + } + + const begin = async (body: ProfileStartBody = picked.body) => { + if (starting) return + setStarting(true) + setStartError(null) + try { + await onStart(body) + setBusy(false) + setBlocked(null) + setMode('job') + } catch (e) { + await fail(e, body) + } finally { + setStarting(false) + } + } + + const act = async (fn: () => Promise) => { + setActing(true) + setActionError(null) + try { + await fn() + } catch (e) { + setActionError(errorMessage(e)) + } finally { + setActing(false) + } + } + const stop = () => act(onCancel) + // 重试是新建一个任务:正忙(别处又建了一个)时和开始一样转到「开始不了」,写清是谁的 + const retry = (from: CatalogProfileJob) => act(async () => { + try { + await onRetry(from) + setBusy(false) + } catch (e) { + if (e instanceof ApiError && e.status === 409) await fail(e, { tables: retryableTasks(from).map((t) => t.table_name) }) + else throw e + } + }) + const viewActive = async () => { + const current = await onShowActive().catch(() => null) + if (current) { + setBusy(true) + setBlocked(null) + setMode('job') + } + } + const backToSetup = () => { + setBlocked(null) + setStartError(null) + setActionError(null) + setBusy(false) + setLast(null) + setMode('setup') + } + const openLast = (j: CatalogProfileJob) => { + onTrack(j) + setBusy(false) + setMode('job') + } + + // 因为没开启被拒的:就地开了(设置存好、这里拿到的设置变成开启)就回到选范围,可以直接开始 + useEffect(() => { + if (mode === 'blocked' && blocked?.kind === 'disabled' && settings.enabled) { + setBlocked(null) + setMode('setup') + } + }, [mode, blocked, settings.enabled]) + + const phase: 'setup' | 'blocked' | 'running' | 'done' = inSetup ? 'setup' : mode === 'blocked' ? 'blocked' : isJobActive(job) ? 'running' : 'done' + const canStart = settings.enabled && picked.tables.length > 0 && !tooMany && !starting + const retryable = job && phase === 'done' ? retryableTasks(job).length : 0 let body: ReactNode let footer: ReactNode - if (!job) { + if (phase === 'setup') { body = ( <>

{PT.intro}

@@ -99,17 +256,18 @@ export function ProfileDialog({ {PT.scope}
{scopes.map((s) => { - const over = (s.tables?.length ?? 0) > PROFILE_MAX_TABLES + const over = !!s.body.tables && s.tables.length > PROFILE_MAX_TABLES + const note = over ? PT.scopeTooMany(PROFILE_MAX_TABLES) : !s.tables.length && s.key !== 'needed' ? PT.scopeEmpty + : s.leftOut ? PT.scopeLeftOut(s.leftOut) : s.hint return ( @@ -117,103 +275,88 @@ export function ProfileDialog({ })}
- {settings.enabled && } -

- {PT.noCancel} + {settings.enabled && } +

+ {PT.readOnly}

+ {draftActive && settings.enabled && ( +

+ {PT.queueHint} +

+ )} + {last && ( +
+ + {PT.last(PT.jobTitle[last.status], formatTime(last.finished_at ?? last.created_at))} + +
+ )} + {startError && {startError}} ) footer = ( <> - ) - } else if (job.phase === 'running') { - const used = Date.now() - job.startedAt - const max = job.settings.max_total_s * 1000 - body = ( -
-
- - {PT.running} - - - {PT.elapsed(secondsText(used), secondsText(max))} - -
- {/* 不是进度:服务端不报进度,这条只量已用时长占总时长上限的多少,剖析最晚在上限处停下 */} -
-
-
- {PT.runningHint} -

{PT.leavePage}

-
- ) - footer = - } else if (job.phase === 'done') { - body = + } else if (phase === 'blocked' && blocked) { + body = void viewActive()} + onChooseAll={() => { setScope('all'); void begin({ scope: 'all' }) }} onRetry={() => void begin(blocked.body)} /> footer = ( <> - - - - ) - } else if (job.phase === 'blocked') { - const action = job.kind === 'disabled' - ? - : job.kind === 'busy' || job.kind === 'other' - ? - : - body = ( - -

{PT.blocked[job.kind]}

-

{job.message}

- {job.kind !== 'other' &&

{PT.blockedNext[job.kind]}

} -
{action}
-
- ) - footer = ( - <> - + ) - } else { + } else if (job) { + const running = phase === 'running' body = ( -
-

{PT.failedTitle}

- onStart(job.tables)} /> -
- ) - footer = ( <> - - + + {actionError && {actionError}} ) + footer = running + ? ( + <> + + + + ) + : ( + <> + + {retryable > 0 && ( + + )} + + + ) } return ( - -
{body}
+ +
{body}
) } -/** 确认前写明预算:按这个数据源的剖析设置,最多发几条查询、单条和总时长、抽样规模、整表统计的门槛 */ -function Budget({ settings: s, sourceName, onSettings }: { settings: CatalogProfileSettings; sourceName: string; onSettings: () => void }) { - const lines = [PT.budgetQueries(s.max_queries, s.query_timeout_s), PT.budgetTotal(s.max_total_s), PT.budgetSample(s.sample_size), +/** 确认前写明预算:按表的查询数和时长、单条时限、抽样规模、整表统计的门槛,再按这次的表数估一个上限 */ +function Budget({ settings: s, sourceName, tables, onSettings }: { + settings: CatalogProfileSettings; sourceName: string; tables: number; onSettings: () => void +}) { + const lines = [PT.budgetPerTable(s.table_max_queries, s.table_max_s), PT.budgetQuery(s.query_timeout_s), PT.budgetSample(s.sample_size), PT.budgetScan(s.max_scan_rows)] return ( -
-
+
+
{PT.budget} {PT.budgetFrom(sourceName)} @@ -228,159 +371,479 @@ function Budget({ settings: s, sourceName, onSettings }: { settings: CatalogProf ))} + {tables > 0 && ( +

+ {PT.budgetTotal(tables, tables * s.table_max_queries, spanText(tables * s.table_max_s))} +

+ )}
) } +/** 开始不了(409):是哪种情况、服务端的原话、下一步和对应的按钮 */ +function Blocked({ blocked: b, onSettings, onGoSource, onViewActive, onChooseAll, onRetry }: { + blocked: { kind: ProfileBlock; message: string; active: CatalogProfileJob | null } + onSettings: () => void + onGoSource: () => void + onViewActive: () => void + onChooseAll: () => void + onRetry: () => void +}) { + const action = b.kind === 'disabled' + ? + : b.kind === 'busy' + ? + : b.kind === 'nothing' + ? + : b.kind === 'other' + ? + : + return ( + +

{PT.blocked[b.kind]}

+

{b.message}

+ {b.kind === 'busy' && b.active && ( +

{PT.busyAt(b.active.created_by, b.active.done, b.active.total)}

+ )} +

{PT.blockedNext[b.kind]}

+
{action}
+
+ ) +} + // --------------------------------------------------------------------------- -// 报告 +// 任务:进度、汇总、逐表 // --------------------------------------------------------------------------- -function Report({ report: r, ms, filled, returnTo, onSettings, onOpenTable, onFillCodes }: { - report: CatalogProfileOut - ms: number +const hasFindings = (t: CatalogProfileTaskRow) => t.summary.relations_checked + t.summary.codes + t.summary.business_dates > 0 +const FILTER_TEST: Record boolean> = { + all: () => true, + findings: hasFindings, + skipped: (t) => t.skipped_count > 0 || t.status === 'skipped', + failed: (t) => t.status === 'failed', +} + +function JobView({ sourceId, job, running, stopping, busy, filled, returnTo, onSettings, onOpenTable, onFillCodes }: { + sourceId: string + job: CatalogProfileJob + running: boolean + stopping: boolean + busy: boolean filled: Record returnTo: { table: string; column: string } | null onSettings: () => void onOpenTable: (table: string) => void onFillCodes: (table: string, column: string) => void }) { - const changed = r.total.added + r.total.updated + r.total.removed - const stop = r.stopped - // 一项发现、一项跳过都没有的表合成一行,几十张表时报告不被空卡片撑长 - const quiet = r.tables.filter((t) => !t.error && !t.findings.length && !t.skipped.length && !t.date_ranges.length) - const shown = r.tables.filter((t) => !quiet.includes(t)) - // 填完一列回到报告:焦点交给它后面第一个还有含义待填写的「填写含义」,没有了就落回刚填的那一列。 - // 弹窗自己的初始焦点在下一帧,这里先聚焦,它看到焦点已在弹窗里就不再动 - const root = useRef(null) - useEffect(() => { - if (!returnTo) return - const buttons = [...(root.current?.querySelectorAll('[data-profile-fill-key]') ?? [])] - const at = buttons.findIndex((b) => b.dataset.profileFillKey === fillKey(returnTo.table, returnTo.column)) - const target = buttons.slice(at + 1).find((b) => Number(b.dataset.pending) > 0) ?? buttons[at] - target?.focus() - target?.scrollIntoView({ block: 'nearest' }) - }, [returnTo]) + const now = useTicker(1000, running) + const queued = job.status === 'queued' + const title = running ? (stopping ? PT.stopping : PT.jobTitle[job.status]) : PT.jobTitle[job.status] + const started = job.started_at ? Date.parse(job.started_at) : NaN + const ended = job.finished_at ? Date.parse(job.finished_at) : NaN + const elapsedMs = Number.isFinite(started) ? (running ? Math.max(now, Date.now()) : Number.isFinite(ended) ? ended : started) - started : null + const pct = job.total ? (job.done / job.total) * 100 : 0 + const bad = job.tasks.some((t) => PROFILE_RETRYABLE.has(t.status)) + const tone = job.status === 'failed' ? 'var(--err)' : running ? 'var(--accent)' : bad ? 'var(--warn)' : 'var(--st-done)' + const progress = PT.progress(job.done, job.total) + const badQueries = PT.queriesBad(job.timeouts, job.errors) + const notice = running ? job.notice?.trim() ?? '' : '' + const changed = job.added + job.updated + job.removed + const findings = PT.findings(job.summary) + const [filter, setFilter] = useState('all') + const radio = useRadioGroup(FILTERS, filter, setFilter) + const tasks = running ? job.tasks : job.tasks.filter(FILTER_TEST[filter]) + const counts = useMemo(() => Object.fromEntries(FILTERS.map((f) => [f, job.tasks.filter(FILTER_TEST[f]).length])) as Record, + [job.tasks]) + // 展开的表:填完码值回到结果时,那张表照旧展开 + const [open, setOpen] = useState>(() => { + const t = returnTo ? job.tasks.find((x) => x.table_name === returnTo.table) : null + return new Set(t ? [t.seq] : []) + }) + const toggle = (seq: number) => setOpen((cur) => { + const next = new Set(cur) + if (next.has(seq)) next.delete(seq) + else next.add(seq) + return next + }) return ( -
-
- {stop - ? - : } -
-

{stop ? PT.stoppedTitle(PROFILE_STOP_LABEL[stop] ?? stop) : PT.done}

-

- {PT.summary(r.queries_used, r.settings.max_queries, r.tables.length)} - · {secondsText(ms)} -

-

- {changed ? PT.changes(r.total.added, r.total.updated, r.total.removed) : {PT.noChange}} +

+ {busy && {PT.busyShown}} +
+ {running && } + {title} + + + {progress} + {!queued && · {PT.queriesSent(job.queries)}} + {badQueries && ({badQueries})} + {elapsedMs != null && · {PT.elapsed(secondsText(elapsedMs))}} + +
+
+
+
+ {queued && job.waiting_for === 'draft' && ( +

+ {PT.waitingDraft} +

+ )} + {notice && {notice}} + {job.created_by &&

{PT.createdBy(job.created_by)}

} + {running &&

{PT.hideHint}

} + {!running && ( +
+

{PT.counts(job.total, job.counts)}

+

{findings || PT.noFindings}

+

+ {changed ? PT.changes(job.added, job.updated, job.removed) : PT.noChange}

-
- {stop && ( - -

{PROFILE_STOP_NEXT[stop] ?? ''}

- {stop !== 'failed' && ( + )} + {job.stop_reason && ( + +

{PT.stopTitle[job.stop_reason]}

+ {job.error &&

{job.error}

} +

{PT.stopNext[job.stop_reason]}

+ {job.stop_reason === 'slow' && ( )}
)} - {r.note && {r.note}} - {/* 挑表途中确知是空表、跳过的表(服务端 empty_tables):一张都没剖析时 note 已经说了,这里只在剖析了别的表时列出 */} - {r.tables.length > 0 && (r.empty_tables?.length ?? 0) > 0 && ( -

- {UT.emptyTables(r.empty_tables!.length)}:{r.empty_tables!.join('、')} -

- )} - {shown.map((t) => )} - {quiet.length > 0 && ( -

- {PT.noFindings}:{quiet.map((t) => t.table_name).join('、')} -

+ {!job.stop_reason && job.status === 'failed' && job.error && ( + {job.error} )} + {!running && job.status === 'cancelled' &&

{PT.cancelledNote}

} + {job.status === 'interrupted' && {PT.interruptedNote}} +
+
+ {PT.tasks} + + {!running && job.tasks.length > 1 && ( +
+ {FILTERS.map((f) => ( + + ))} +
+ )} +
+
    + {tasks.map((t) => ( + toggle(t.seq)} filled={filled} + focusAfter={returnTo && returnTo.table === t.table_name ? returnTo.column : null} + onOpenTable={onOpenTable} onFillCodes={onFillCodes} /> + ))} + {!tasks.length &&
  • {PT.filterNone}
  • } +
+
) } -function sizeText(size: CatalogProfileSize | null): { text: string; hint?: string } | null { - if (!size) return null - if (size.rows != null) return { text: size.method === 'stats' ? PT.rowsStats(size.rows) : PT.rowsCount(size.rows), hint: PT.rowsHint[size.method] } - if (size.at_least != null) return { text: PT.rowsAtLeast(size.at_least), hint: PT.rowsHint.count } - return null +export const PROFILE_TASK_ICON: Record = { + pending: { Icon: CircleDashed, color: 'var(--text-faint)' }, + running: { Icon: null, color: 'var(--accent)' }, + ok: { Icon: CircleCheck, color: 'var(--st-done)' }, + partial: { Icon: TriangleAlert, color: 'var(--warn)' }, + failed: { Icon: CircleX, color: 'var(--err)' }, + skipped: { Icon: CircleMinus, color: 'var(--text-faint)' }, + cancelled: { Icon: CircleSlash, color: 'var(--text-faint)' }, } -function TableReport({ t, filled, onOpenTable, onFillCodes }: { - t: CatalogProfileTable +/** 各类检查的进度:「关系 3/4 · 码值 5/6 · 日期 1/1」,这类没有检查的不写 */ +export function checksText(t: Pick): string { + return (['relation', 'codes', 'date'] as const) + .map((k) => { + const [done, total] = t.checks_by_kind?.[k] ?? [0, 0] + return total > 0 ? PT.kindProgress(PT.checkKind[k], done, total) : '' + }) + .filter(Boolean).join(' · ') +} + +/** + * 一张表的结论:进行中写正在做哪一项;结束了写发现(核实了几条关系、取到几列码值……),没查成的写几项,整张表的原因照服务端 + * 的原话。jobError:任务整体的原因(止损、服务重启时服务端把后面的表也记上同一句),和它一样的不再逐表重复 + */ +export function taskNote(t: CatalogProfileTaskRow, jobError: string | null): string { + if (t.status === 'pending') return '' + if (t.status === 'running') return t.current ?? '' + const own = t.error && t.error !== jobError ? t.error : '' + if (t.status === 'skipped' || t.status === 'failed') return own + const parts = [PT.findings(t.summary, true), t.unfinished ? PT.unfinished(t.unfinished) : '', own] + if (t.status === 'ok' && !parts.some(Boolean)) return PT.noFindings + return parts.filter(Boolean).join(';') +} + +function TaskItem({ sourceId, job, task: t, open, onToggle, filled, focusAfter, onOpenTable, onFillCodes }: { + sourceId: string + job: CatalogProfileJob + task: CatalogProfileTaskRow + open: boolean + onToggle: () => void filled: Record + focusAfter: string | null onOpenTable: (table: string) => void onFillCodes: (table: string, column: string) => void }) { - const relations = t.findings.filter((f): f is CatalogProfileRelationFinding => f.kind === 'relation') - const codes = t.findings.filter((f): f is CatalogProfileCodesFinding => f.kind === 'codes') - const date = t.findings.find((f): f is CatalogProfileDateFinding => f.kind === 'business_date') - const size = sizeText(t.row_estimate) - const changed = t.added + t.updated + t.removed + const { Icon, color } = PROFILE_TASK_ICON[t.status] + const checks = checksText(t) + const size = t.status !== 'skipped' ? sizeText(t.row_estimate) : null + const usedUp = t.queries > 0 && t.queries >= job.settings.table_max_queries + const note = taskNote(t, job.error) + const started = t.status !== 'pending' + const panelId = `catalog-profile-task-${t.seq}` return ( -
-
-

{t.table_name}

- - {PT.tableQueries(t.queries)} - {size && · {size.text}} - {changed > 0 && · {PT.changes(t.added, t.updated, t.removed)}} +
  • + + {started && (checks || t.queries > 0 || size) && ( + + {checks && {checks}} + {t.queries > 0 && ( + + {usedUp ? PT.tableQueriesUsedUp(t.queries) : PT.tableQueries(t.queries)} + + )} + {t.elapsed_ms > 0 && {secondsText(t.elapsed_ms)}} + {size && {size.text}} + {t.skipped_count > 0 && t.status !== 'skipped' && {PT.skippedN(t.skipped_count)}} + )} -
  • -
    - {t.error && ( -

    - {t.error} -

    + {note && ( + + {note} + )} - {relations.length > 0 && ( - - {relations.map((f) => )} - - )} - {codes.length > 0 && ( - - {codes.map((f) => ( - onFillCodes(t.table_name, f.column)} /> - ))} - - )} - {date && ( - -
  • - - {PT.dateProposal(date.column)} - ,{PT.dateSpan(date.min.slice(0, 19), date.max.slice(0, 19))} + + {open && ( +
    + +
    + )} +
  • + ) +} + +/** + * 展开的一张表:按 seq 取完整结果(检查清单、发现、发过的查询)。还在剖析的表,每多结束一项、多发一条查询就重取一次 + */ +function TaskDetailLoader({ sourceId, jobId, task: t, filled, focusAfter, onOpenTable, onFillCodes }: { + sourceId: string + jobId: string + task: CatalogProfileTaskRow + filled: Record + focusAfter: string | null + onOpenTable: (table: string) => void + onFillCodes: (table: string, column: string) => void +}) { + const [detail, setDetail] = useState(null) + const [error, setError] = useState(null) + const [again, setAgain] = useState(0) + const sig = `${t.status}:${t.checks_done}:${t.queries}` + useEffect(() => { + let alive = true + api.dataCatalog.profileJobs.task(sourceId, jobId, t.seq).then(({ task }) => { + if (!alive) return + setDetail(task) + setError(null) + }, (e) => { if (alive) setError(errorMessage(e)) }) + return () => { alive = false } + }, [sourceId, jobId, t.seq, sig, again]) + if (!detail) { + return error + ? ( +

    + {PT.loadFailed(error)} + +

    + ) + :

    {PT.loading}

    + } + return onFillCodes(t.table_name, column)} /> +} + +// --------------------------------------------------------------------------- +// 一张表的剖析结果:逐表展开里、表详情的「数据剖析」一节共用 +// --------------------------------------------------------------------------- + +/** + * mode job:任务里展开的一行,写检查清单(每项的状态、原因、用了几条查询)、发现、发过的查询; + * mode detail:表详情里,写行数和来源、日期范围、发现、跳过的项、发过的查询(检查清单在任务里看)。 + * onFill 不给时码值不带「填写含义」 + */ +export function ProfileTaskBody({ detail: d, mode, filled = {}, focusAfter = null, onOpenTable, onFill }: { + detail: CatalogProfileTaskDetail + mode: 'job' | 'detail' + filled?: Record + focusAfter?: string | null + onOpenTable?: (table: string) => void + onFill?: (column: string) => void +}) { + const relations = d.findings.filter((f): f is CatalogProfileRelationFinding => f.kind === 'relation') + const codes = d.findings.filter((f): f is CatalogProfileCodesFinding => f.kind === 'codes') + const date = d.findings.find((f): f is CatalogProfileDateFinding => f.kind === 'business_date') + const size = sizeText(d.row_estimate) + // 检查清单里没有的跳过项(行数估算、整张表):任务里也要看得到 + const skipped = mode === 'detail' ? d.skipped : d.skipped.filter((s) => s.kind !== 'relation' && s.kind !== 'codes' && s.kind !== 'date') + const root = useRef(null) + // 填完一列回到结果:焦点交给它后面第一个还有含义待填写的「填写含义」,没有了就落回刚填的那一列。弹窗自己的初始焦点在 + // 下一帧,这里先聚焦,它看到焦点已在弹窗里就不再动 + useEffect(() => { + if (!focusAfter) return + const buttons = [...(root.current?.querySelectorAll('[data-profile-fill-key]') ?? [])] + const at = buttons.findIndex((b) => b.dataset.profileFillKey === fillKey(d.table_name, focusAfter)) + const target = buttons.slice(at + 1).find((b) => Number(b.dataset.pending) > 0) ?? buttons[at] + target?.focus() + target?.scrollIntoView({ block: 'nearest' }) + }, [focusAfter, d.table_name]) + return ( +
    + {(size || (mode === 'job' && onOpenTable && d.status !== 'skipped')) && ( +
    + {size && ( + + {PT.rowsTitle}:{size.text}{size.source && ({size.source})} + + )} + + {mode === 'job' && onOpenTable && ( + + )} +
    + )} + {d.error && ( +

    + {d.error} +

    + )} + {mode === 'job' && d.checks.length > 0 && } + {relations.length > 0 && ( + + {relations.map((f) => )} + + )} + {codes.length > 0 && ( + + {codes.map((f) => ( + onFill(f.column) : undefined} /> + ))} + + )} + {date && ( + +
  • + + {PT.dateProposal(date.column)} + ,{PT.dateSpan(date.min.slice(0, 19), date.max.slice(0, 19))} +
  • +
    + )} + {d.date_ranges.length > 0 && (!date || d.date_ranges.length > 1) && ( + + {d.date_ranges.map((r) => ( +
  • + {r.column}:{PT.dateSpan(r.min.slice(0, 19), r.max.slice(0, 19))}
  • -
    - )} - {!date && t.date_ranges.length > 0 && ( - - {t.date_ranges.map((d) => ( -
  • - {d.column}:{PT.dateSpan(d.min.slice(0, 19), d.max.slice(0, 19))} -
  • - ))} -
    - )} - {!t.error && !t.findings.length && !t.date_ranges.length && ( -

    {PT.noFindings}

    - )} - {t.skipped.length > 0 && } -
    -
    + ))} + + )} + {!d.findings.length && !d.error && (d.status === 'ok' || d.status === 'partial') && ( +

    {PT.noFindings}

    + )} + {skipped.length > 0 && } + {d.query_log.length > 0 && } +
    + ) +} + +const CHECK_ICON: Record = { + pending: { Icon: CircleDashed, color: 'var(--text-faint)' }, + running: { Icon: null, color: 'var(--accent)' }, + done: { Icon: CircleCheck, color: 'var(--st-done)' }, + skipped: { Icon: CircleMinus, color: 'var(--text-faint)' }, +} + +/** 检查清单:每项的状态、原因(没查成的用警告色:不是结论错了,旧结论保留)、用了几条查询 */ +function Checks({ checks }: { checks: CatalogProfileCheck[] }) { + const done = checks.filter((c) => c.state === 'done' || c.state === 'skipped').length + return ( +
    +
    {PT.checksTitle(done, checks.length)}
    +
      + {checks.map((c, i) => { + const { Icon, color } = CHECK_ICON[c.state] + const unfinished = c.state === 'skipped' && !!c.reason && UNFINISHED_REASONS.has(c.reason) + return ( +
    • + {Icon ? : } + + {PT.checkKind[c.kind] ?? c.kind} + {c.target} + · {PT.checkState[c.state] ?? c.state} + {c.reason && ( + + {PROFILE_SKIP_REASON_LABEL[c.reason] ?? c.reason} + + )} + {c.detail && {c.detail}} + + {c.queries > 0 && {PT.checkQueries(c.queries)}} +
    • + ) + })} +
    +
    + ) +} + +/** 发过的查询(字面量换成了 ?):折起来,点开看每一条的类别、目标、结果、用时、行数和 SQL */ +function QueryLog({ list }: { list: CatalogProfileQuery[] }) { + return ( +
    + {PT.queriesTitle(list.length)} +

    {PT.queriesHint}

    +
      + {list.map((q, i) => ( +
    1. +
      + {PT.queryKind[q.kind] ?? q.kind} + {q.target} + + + {PT.queryOutcome[q.outcome] ?? q.outcome} + + {PT.queryMs(q.ms)} + {q.rows != null && {PT.queryRows(q.rows)}} + {formatTime(q.at)} +
      +
      {q.sql}
      + {q.detail &&

      {q.detail}

      } +
    2. + ))} +
    +
    ) } @@ -397,7 +860,7 @@ type RelationOutcome = 'confirmed' | 'raised' | 'same' | 'lowered' | 'kept' /** * 一条关系这次剖析的结论,按实际变化说:原本就是已验证、这次照样核实的写「仍为已验证,覆盖率 x」,不能说「升为」—— - * 重新剖析、目录没变时报告头写「目录没有变化」,下面却说升了,自相矛盾。老服务端不给剖析前的状态,按升级说 + * 重新剖析、目录没变时汇总写「目录没有变化」,下面却说升了,自相矛盾。服务端不给剖析前的状态时按升级说 */ function relationOutcome(f: CatalogProfileRelationFinding): { kind: RelationOutcome; text: string; tone: string; icon: LucideIcon } { const reason = f.coverage < 0.95 ? PT.relationLowCoverage : PT.relationNotUnique @@ -448,11 +911,11 @@ function RelationLine({ f }: { f: CatalogProfileRelationFinding }) { function CodesLine({ f, fillKey: key, filled, onFill }: { f: CatalogProfileCodesFinding fillKey: string - /** 从报告去填过含义:保存后还剩几个待填写。没去填过为 undefined,按剖析时的说 */ + /** 从结果去填过含义:保存后还剩几个待填写。没去填过为 undefined,按剖析时的说 */ filled: number | undefined - onFill: () => void + onFill?: () => void }) { - // 几个含义待填写读服务端的 pending(别的来源写过的含义会沿用),不从 summary 的原话里认;老服务端不给时不说 + // 几个含义待填写读服务端的 pending(别的来源写过的含义会沿用),不从 summary 的原话里认 const pending = filled ?? f.pending ?? 0 const done = filled != null && filled === 0 return ( @@ -478,16 +941,18 @@ function CodesLine({ f, fillKey: key, filled, onFill }: { {UT.codesFilled} )} - + {onFill && ( + + )}
    ) } -/** 跳过的项:原因(短)+ 哪一项 + 服务端给的整句。多了折起来,每类停下的原因只列前几项 */ +/** 跳过的项:原因(短)+ 哪一项 + 服务端给的整句。多了折起来 */ function Skipped({ list }: { list: CatalogProfileSkip[] }) { const [all, setAll] = useState(false) const shown = all ? list : list.slice(0, SKIP_SHOWN) @@ -497,13 +962,14 @@ function Skipped({ list }: { list: CatalogProfileSkip[] }) {
      {shown.map((s, i) => (
    • - + {PROFILE_SKIP_REASON_LABEL[s.reason] ?? s.reason} {PROFILE_SKIP_KIND_LABEL[s.kind] ?? s.kind} {s.target} - {s.detail} + {s.detail}
    • ))} @@ -517,4 +983,3 @@ function Skipped({ list }: { list: CatalogProfileSkip[] }) {
    ) } - diff --git a/frontend/src/pages/catalog/ProfileSettings.tsx b/frontend/src/pages/catalog/ProfileSettings.tsx index 072b92f2..ddd9a3ec 100644 --- a/frontend/src/pages/catalog/ProfileSettings.tsx +++ b/frontend/src/pages/catalog/ProfileSettings.tsx @@ -14,6 +14,8 @@ import type { ProfileForm } from './profile' // 数据剖析的设置:开关、五项预算、风险说明。数据源编辑框里是其中一段;导入表格的源没有编辑框,数据目录页在 // 剖析被拒(未开启)时也要就地打开它,所以另有一个只改这一项的弹窗(options 的其余键原样带上)。 // 剖析会对业务库发查询,开关默认关着;开启后五项预算才展开,风险说明写的是此刻填的数。 +// 预算按表算:每张表最多查询数、每张表时长上限(用完只停这张表),另有单条查询时限、抽样键值数、整表统计行数上限; +// 没有整次剖析的总量上限(任务自己在连续失败、多数超时时停下)。 // =========================================================================== /** 保存被拒时的落点:某一项,或者整段(认不出是哪一项) */ @@ -57,7 +59,7 @@ export function ProfileSettingsSection({ idPrefix, form, onChange, serverError }

    - {PT.risk(s.max_queries, s.query_timeout_s, s.max_total_s)} + {PT.risk(s.table_max_queries, s.table_max_s, s.query_timeout_s)}

    )}
    diff --git a/frontend/src/pages/catalog/TableDetail.tsx b/frontend/src/pages/catalog/TableDetail.tsx index 0f0882c4..ebfbc768 100644 --- a/frontend/src/pages/catalog/TableDetail.tsx +++ b/frontend/src/pages/catalog/TableDetail.tsx @@ -1,12 +1,13 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react' import type { ReactNode } from 'react' import { - AlertTriangle, ArrowLeft, ChevronLeft, ChevronRight, GitBranch, Info, ListChecks, Lock, MessagesSquare, Pencil, Plus, RotateCw, Search, Sparkles, X, + AlertTriangle, ArrowLeft, ChevronLeft, ChevronRight, GitBranch, Info, ListChecks, Lock, MessagesSquare, Pencil, Plus, RotateCw, ScanSearch, Search, + Sparkles, X, } from 'lucide-react' import clsx from 'clsx' import { ApiError, api } from '../../api/client' import type { - CatalogBusinessDate, CatalogCardinality, CatalogDetail, CatalogItem, CatalogRelation, CatalogReviewAction, CatalogTableKind, + CatalogBusinessDate, CatalogCardinality, CatalogDetail, CatalogItem, CatalogProfileJob, CatalogRelation, CatalogReviewAction, CatalogTableKind, CatalogTableRow, } from '../../types' import { DeleteButton, EmptyState, ErrorState, Notice, Skeleton, Spinner, confirmDialog, toast } from '../../components/ui' @@ -15,7 +16,7 @@ import { formatDateTime, formatNumber, formatTime } from '../../lib/format' import { CATALOG_ASSISTANT_TEXT as AT, CATALOG_CARDINALITY_LABEL, CATALOG_DRAFT_TEXT as DT, CATALOG_IMPACT_TEXT, CATALOG_KIND_HINT, CATALOG_KIND_LABEL, CATALOG_REVIEW_TEXT as RT, CATALOG_TABLE_FIELD_HINT, CATALOG_TABLE_FIELD_LABEL, CATALOG_TEXT as CT, CATALOG_UI_TEXT as UT, - CATALOG_USAGE_TEXT, CODES_TEXT as KT, + CATALOG_USAGE_TEXT, CODES_TEXT as KT, PROFILE_TEXT as PT, } from '../../lib/terms' import { CatalogImpactList } from '../../components/CatalogImpact' import { @@ -26,8 +27,10 @@ import type { ReviewTarget } from './parts' import { ColumnTable, NARROW, Value, pendingCodes } from './ColumnTable' import type { ColumnModel } from './ColumnTable' import { CodesDialog } from './CodesDialog' -import { coverageText } from './profile' -import { lastBlankInfo, taskReason } from './draftJob' +import { coverageText, isProfilable, profileBlockOf } from './profile' +import { isJobActive, lastBlankInfo, taskReason } from './draftJob' +import { isProfileBusy } from './profileJob' +import { PROFILE_TASK_ICON, ProfileTaskBody } from './ProfileDialog' import { TableAssistant } from './TableAssistant' import { TableUsageList } from './TableUsage' import { @@ -42,6 +45,9 @@ import type { CodesItem, ColumnField, EditForm, RelationDraft, TableField, Table // // 写操作都带读到的版本。409 说明别人刚改过:不静默覆盖,横幅写明并给「重新载入」;编辑中的修改留着,重新载入前 // 先问一句。编辑中换表、离开页面走全站的离开前确认。 +// +// 「数据剖析」一节:这张表上次剖析的时间、发起人、所属任务、行数估算和来源、日期范围、发现、跳过的项、发过的查询; +// 「剖析这张表」建只含这张表的剖析任务(没开剖析时引到剖析设置;已有剖析任务时写明它到哪了、这张表在不在里面)。 // =========================================================================== const KINDS = Object.keys(CATALOG_KIND_LABEL) as CatalogTableKind[] @@ -62,7 +68,7 @@ type Conflict = 'edit' | 'review' export function TableDetail({ sourceId, table, rows, drafted, fillCodes, onFillDone, prev, next, onOpen, onBack, onDetail, draftBusy = false, onRedraft, onDraftGuidance, - profileEnabled = false, focusPath = null, usageOpen = false, + profileEnabled = false, focusPath = null, usageOpen = false, profile, }: { sourceId: string table: string @@ -99,6 +105,8 @@ export function TableDetail({ focusPath?: string | null /** 「用过这张表的运行」直接取(从审阅队列打开时) */ usageOpen?: boolean + /** 数据剖析:页面跟着的剖析任务、建只含这张表的任务、打开剖析设置和剖析对话框 */ + profile?: TableProfileProps }) { const [detail, setDetail] = useState(null) const [loadError, setLoadError] = useState(null) @@ -620,6 +628,11 @@ export function TableDetail({ focusPath={focusPath} />} + {profile && ( + r.table_name === table) ?? null} + enabled={profileEnabled} profile={profile} onFillCodes={detail.in_schema ? fillCodesOf : undefined} /> + )} +

    {CATALOG_IMPACT_TEXT.title}

    @@ -1037,3 +1050,151 @@ function RelationEditor({ drafts, initial, problems, rows, onChange }: {
    ) } + +// --------------------------------------------------------------------------- +// 数据剖析 +// --------------------------------------------------------------------------- + +export interface TableProfileProps { + /** 页面跟着的剖析任务(进行中的、刚结束的) */ + job: CatalogProfileJob | null + /** 建只含这张表的剖析任务;409、422 照原样抛 */ + onStart: (table: string) => Promise + /** 正忙(409)时取进行中的那一个,页面改跟着它 */ + onShowActive: () => Promise + /** 打开这个数据源的剖析设置 */ + onSettings: () => void + /** 打开剖析对话框,显示这个任务(页面跟着的那一个,或者按编号取来) */ + onOpenJob: (jobId: string | null) => void +} + +/** + * 表详情里的「数据剖析」一节。上次剖析的结论在服务端的 last_profile(GET /catalog/{table},任务还在进行就是进行中的状态), + * 页面跟着的任务里这张表在排队、剖析中时以任务为准(比载入详情时新);这张表剖析完,页面会重新载入详情 + */ +function ProfileSection({ table, detail, row, enabled, profile, onFillCodes }: { + table: string + detail: CatalogDetail + row: CatalogTableRow | null + enabled: boolean + profile: TableProfileProps + onFillCodes?: (column: string) => void +}) { + const [starting, setStarting] = useState(false) + /** 开始不了(409):服务端的原话、正在进行的那个任务里有没有这张表 */ + const [blocked, setBlocked] = useState<{ kind: string; message: string; where: string } | null>(null) + useEffect(() => { setBlocked(null) }, [table]) + const last = detail.last_profile ?? null + const job = profile.job + const live = job && isJobActive(job) ? job.tasks.find((t) => t.table_name === table) ?? null : null + const liveActive = live && (live.status === 'pending' || live.status === 'running') ? live : null + const isView = !!detail.structure?.is_view || !!row?.is_view + const profilable = detail.in_schema && !isView && (row ? isProfilable(row) : true) + const canStart = enabled && profilable && !liveActive && !starting + // 上次剖析所在的任务:正有别的剖析任务在进行时不去换(页面只跟着一个任务,换了就看不到进行中的那个了) + const canOpenJob = !!last && (!job || !isJobActive(job) || job.id === last.job_id) + + const start = async () => { + setStarting(true) + setBlocked(null) + try { + const next = await profile.onStart(table) + const text = next.status === 'queued' && next.waiting_for === 'draft' ? PT.profileThisQueued(table) : PT.profileThisStarted(table) + toast.ok(text, { action: { label: PT.viewProgress, onClick: () => profile.onOpenJob(next.id) } }) + } catch (e) { + if (e instanceof ApiError && e.status === 409) { + const kind = profileBlockOf(e) + let where = '' + if (isProfileBusy(e)) { + // 正忙:写清进行中的那个任务到哪了、这张表在不在里面、排第几 + const active = await profile.onShowActive().catch(() => null) + if (active) { + const at = active.tasks.findIndex((t) => t.table_name === table) + where = [PT.busyAt(active.created_by, active.done, active.total), at >= 0 ? PT.busyThisIn(at + 1, active.total) : PT.busyThisNotIn].join(';') + } + } + setBlocked({ kind, message: e.message, where }) + } else { + toast.error(e) + } + } finally { + setStarting(false) + } + } + + let status: ReactNode = null + if (liveActive) { + status = ( + +

    {liveActive.status === 'running' ? PT.detailRunning : PT.detailPending}

    + {liveActive.status === 'running' && liveActive.current &&

    {liveActive.current}

    } + +
    + ) + } + const lastAt = last?.finished_at ?? last?.started_at ?? null + const ended = last && last.status !== 'pending' && last.status !== 'running' + const LastIcon = last ? PROFILE_TASK_ICON[last.status].Icon : null + return ( +
    +
    +

    {PT.title}

    + {PT.detailHint} + + {profilable && ( + + )} +
    +
    + {!enabled && profilable && ( +

    + {PT.profileThisDisabled} + +

    + )} + {blocked && ( + +

    {PT.blocked[blocked.kind] ?? PT.blocked.other}

    +

    {blocked.message}

    + {blocked.where &&

    {blocked.where}

    } + {blocked.kind === 'busy' && ( + + )} + {blocked.kind === 'disabled' && ( + + )} +
    + )} + {status} + {!last && !liveActive && ( +

    + {!detail.in_schema ? PT.detailMissing : isView ? PT.detailView : !profilable ? PT.detailNoChecks : PT.detailNever} +

    + )} + {last && ended && ( + <> +
    + {LastIcon && } + {PT.taskStatus[last.status]} + {lastAt && {PT.detailLast(formatTime(lastAt))}} + {last.created_by && · {PT.detailBy(last.created_by)}} + {last.queries > 0 && · {PT.tableQueries(last.queries)}} + + {canOpenJob && ( + + )} +
    + {PT.findings(last.summary, true) && ( +

    {PT.findings(last.summary, true)}

    + )} + + + )} +
    +
    + ) +} diff --git a/frontend/src/pages/catalog/TableIndex.tsx b/frontend/src/pages/catalog/TableIndex.tsx index c9386d7e..a821e24c 100644 --- a/frontend/src/pages/catalog/TableIndex.tsx +++ b/frontend/src/pages/catalog/TableIndex.tsx @@ -2,8 +2,8 @@ import { memo, useCallback, useEffect, useRef } from 'react' import type { KeyboardEvent as ReactKeyboardEvent, MouseEvent as ReactMouseEvent } from 'react' import { AlertTriangle, History, Info, ScanSearch, Search, Sparkles, X } from 'lucide-react' import clsx from 'clsx' -import type { CatalogTableRow } from '../../types' -import { useRadioGroup } from '../../components/ui' +import type { CatalogProfileTaskStatus, CatalogTableRow } from '../../types' +import { useRadioGroup, useTicker } from '../../components/ui' import { formatNumber } from '../../lib/format' import { CATALOG_DRAFT_TEXT as DT, CATALOG_KIND_LABEL, CATALOG_TEXT as CT, CATALOG_UI_TEXT as UT, PROFILE_TEXT as PT, @@ -11,6 +11,7 @@ import { import { CountBadges } from './parts' import { LIST_FILTERS, LIST_SORTS, progressOf } from './model' import { lastBlankInfo, taskReason } from './draftJob' +import { profileCell } from './profile' import type { ListFilter, ListSort } from './model' // =========================================================================== @@ -19,11 +20,15 @@ import type { ListFilter, ListSort } from './model' // 下面淡淡地写「未填写中文名」——一屏几十行「未填写中文名」认不出是哪张表。上次起草失败、部分完成的表再多一行, // 写明原因(全文在悬停里,点开表还有「重新起草这张表」);上次起草有留空列(模型拿不准)的表另有一行淡色的轻提示, // 不是失败标记。键盘:↑↓ 在行之间移动,Home / End 到两头,Enter 打开;复选框按住 Shift 点选可以连选一段。 +// +// 开了数据剖析的源(或者剖析过的表)第二行多一格「剖析」:未剖析 / 3 天前剖析 / 剖析部分完成 / 剖析失败 / 剖析中 / 排队剖析, +// 需要剖析的(从没剖析过、上次没做完、之后出现了新的待核实关系)用提醒色,悬停写上次剖析的时间和原因。筛选组下面另有 +// 「需要剖析」开关,和审阅进度的筛选叠加(审阅进度的四项数字照旧按审阅算),口径同剖析范围里的「需要剖析的表」。 // =========================================================================== export function TableIndex({ rows, total, counts, query, onQuery, filter, onFilter, usedOnly = false, onUsedOnly, sort, onSort, selected, onSelected, active, onOpen, - onDraft, onProfile, + onDraft, onProfile, profileOn = false, profileLive, dueOnly = false, onDueOnly, dueTotal = 0, }: { /** 筛选、排序后的行 */ rows: CatalogTableRow[] @@ -46,6 +51,15 @@ export function TableIndex({ onDraft: () => void /** 剖析所选(数据剖析弹窗,范围默认是已选的表) */ onProfile?: () => void + /** 这个数据源开了数据剖析:每行写「剖析」一格,给「需要剖析」开关 */ + profileOn?: boolean + /** 页面正跟着的剖析任务里各表的状态(排队、剖析中),比清单新 */ + profileLive?: Map + /** 只看需要剖析的表 */ + dueOnly?: boolean + onDueOnly?: (v: boolean) => void + /** 需要剖析的表有几张(按搜索词算) */ + dueTotal?: number }) { const radio = useRadioGroup(LIST_FILTERS, filter, onFilter) const listRef = useRef(null) @@ -53,6 +67,8 @@ export function TableIndex({ const lastClicked = useRef(null) const picked = rows.filter((r) => selected.has(r.table_name)).length const all = rows.length > 0 && picked === rows.length + // 「3 天前剖析」「12 分钟前剖析」隔一分钟重算一次 + const now = useTicker(60_000, profileOn) useEffect(() => { if (allRef.current) allRef.current.indeterminate = picked > 0 && !all @@ -140,6 +156,17 @@ export function TableIndex({ ))}
    + {onDueOnly && (profileOn || dueOnly) && ( +
    + +
    + )} {usedOnly && (
    {rows.map((r, i) => ( + onToggle={toggle} onOpen={onOpen} onKey={onRowKey} profileOn={profileOn} live={profileLive?.get(r.table_name) ?? null} now={now} /> ))} @@ -199,7 +226,9 @@ export function TableIndex({ ) } -const Row = memo(function Row({ row: r, index, active, checked, onToggle, onOpen, onKey }: { +const PROFILE_TONE = { due: 'var(--st-waiting)', bad: 'var(--err)', active: 'var(--accent)', quiet: undefined } as const + +const Row = memo(function Row({ row: r, index, active, checked, onToggle, onOpen, onKey, profileOn, live, now }: { row: CatalogTableRow index: number active: boolean @@ -207,7 +236,12 @@ const Row = memo(function Row({ row: r, index, active, checked, onToggle, onOpen onToggle: (i: number, e: ReactMouseEvent) => void onOpen: (table: string) => void onKey: (i: number, e: ReactKeyboardEvent) => void + profileOn: boolean + live: CatalogProfileTaskStatus | null + now: number }) { + // 「剖析」一格:开了剖析的源每行都写;没开的源只写剖析过的表(以前开过,结论还在目录里) + const cell = profileOn || r.last_profile || live ? profileCell(r, live, now) : null const progress = progressOf(r.counts) // 上次起草没写全:失败、部分完成各一种说法,后面接原因 const last = r.last_draft && (r.last_draft.status === 'failed' || r.last_draft.status === 'partial') ? r.last_draft : null @@ -254,6 +288,11 @@ const Row = memo(function Row({ row: r, index, active, checked, onToggle, onOpen {r.is_view && · {CT.view}} {r.relations > 0 && · {CT.relationCount(r.relations)}} + {cell && ( + + {PT.title}:{cell.text} + + )} {CT.usage(r.usage)} {!r.in_schema && ( diff --git a/frontend/src/pages/catalog/draftJob.ts b/frontend/src/pages/catalog/draftJob.ts index 4ec9d547..cb856979 100644 --- a/frontend/src/pages/catalog/draftJob.ts +++ b/frontend/src/pages/catalog/draftJob.ts @@ -9,6 +9,9 @@ import type { CatalogActiveDraftJob, CatalogDraftJob, CatalogDraftTask, CatalogD // // 取回来的状态只往前走:已经结束的任务不会被一个晚到的「进行中」翻回去,已结束的表数不减少(停止接口的回答和 // 正在路上的那次取进度谁先到都一样)。 +// +// 同一个数据源的起草任务和剖析任务按建立顺序排队(服务端每个源一条队):起草建在剖析之后时是排队、waiting_for 为 +// profile,剖析结束后自动开始,界面照这样说(waitsForProfile)。 // =========================================================================== /** 进行中多久取一次进度 */ @@ -26,6 +29,10 @@ const DRAFT_FINISHED = 'catalog_draft_finished' export const isJobActive = (j: { status: string } | null | undefined): boolean => !!j && (j.status === 'queued' || j.status === 'running') +/** 起草任务排在同一个数据源的剖析任务后面(剖析结束后自动开始)。老服务端没有 waiting_for */ +export const waitsForProfile = (j: { status: string; waiting_for?: string | null } | null | undefined): boolean => + !!j && j.status === 'queued' && j.waiting_for === 'profile' + const TASK_ENDED = new Set(['ok', 'partial', 'failed', 'skipped', 'cancelled']) /** 「只重试失败的表」带上的:失败、部分完成、已停止 */ export const RETRYABLE = new Set(['failed', 'partial', 'cancelled']) diff --git a/frontend/src/pages/catalog/model.ts b/frontend/src/pages/catalog/model.ts index e2974a2e..7de1f733 100644 --- a/frontend/src/pages/catalog/model.ts +++ b/frontend/src/pages/catalog/model.ts @@ -2,6 +2,7 @@ import type { CatalogBusinessDate, CatalogColumnNotes, CatalogCounts, CatalogDetail, CatalogItem, CatalogNotes, CatalogRelation, CatalogSource, CatalogStatus, CatalogStructureColumn, CatalogTableRow, } from '../../types' +import { isProfileDue } from './profile' // =========================================================================== // 数据目录界面的纯函数:清单的筛选和排序、单表目录摊平成「项」、编辑表单和目录之间的来回转换、表单校验。 @@ -114,11 +115,13 @@ export const isUsed = (r: CatalogTableRow): boolean => r.in_schema && r.usage > /** * 筛选并排序。默认保持服务端的顺序(使用次数多的在前,次数相同按表结构里的顺序);表结构里已经没有的表 - * 不论怎么排都在最后。used:只看运行中查询过的表(从顶部摘要点进来时带上,和摘要的数字对得上) + * 不论怎么排都在最后。used:只看运行中查询过的表(从顶部摘要点进来时带上,和摘要的数字对得上); + * due:只看需要剖析的表(和剖析范围「需要剖析的表」同一个口径)。这两项和审阅进度的筛选叠加 */ -export function visibleRows(rows: CatalogTableRow[], q: string, filter: ListFilter, sort: ListSort, used = false): CatalogTableRow[] { +export function visibleRows(rows: CatalogTableRow[], q: string, filter: ListFilter, sort: ListSort, used = false, due = false): CatalogTableRow[] { const order = new Map(rows.map((r, i) => [r.table_name, i])) - const picked = rows.filter((r) => matchesQuery(r, q) && (!used || isUsed(r)) && (filter === 'all' || progressOf(r.counts) === filter)) + const picked = rows.filter((r) => matchesQuery(r, q) && (!used || isUsed(r)) && (!due || isProfileDue(r)) + && (filter === 'all' || progressOf(r.counts) === filter)) const idx = (r: CatalogTableRow) => order.get(r.table_name) ?? 0 return picked.sort((a, b) => { if (a.in_schema !== b.in_schema) return a.in_schema ? -1 : 1 @@ -144,22 +147,27 @@ export function usageSummary(rows: CatalogTableRow[]): { used: number; pending: return out } -/** 各筛选项下有几张表(筛选按钮上的数字,按搜索词和「只看运行中查询过的表」算) */ -export function filterCounts(rows: CatalogTableRow[], q: string, used = false): Record { +/** 各筛选项下有几张表(筛选按钮上的数字,按搜索词、「只看运行中查询过的表」「只看需要剖析的表」算) */ +export function filterCounts(rows: CatalogTableRow[], q: string, used = false, due = false): Record { const out: Record = { all: 0, pending: 0, done: 0, none: 0 } for (const r of rows) { - if (!matchesQuery(r, q) || (used && !isUsed(r))) continue + if (!matchesQuery(r, q) || (used && !isUsed(r)) || (due && !isProfileDue(r))) continue out.all++ out[progressOf(r.counts)]++ } return out } +/** 「需要剖析」那个开关上的数字:按搜索词和「只看运行中查询过的表」算,不管审阅进度筛到哪一项 */ +export function dueCount(rows: CatalogTableRow[], q: string, used = false): number { + return rows.filter((r) => matchesQuery(r, q) && (!used || isUsed(r)) && isProfileDue(r)).length +} + /** * 清单的筛选、搜索、排序记在地址栏里(刷新、前进后退、把链接发给同事都不丢):q 搜索词,filter 筛选,sort 排序, - * used=1 只看运行中查询过的表。默认值不写进地址;认不出的值按默认 + * used=1 只看运行中查询过的表,profile=due 只看需要剖析的表。默认值不写进地址;认不出的值按默认 */ -export interface ListParams { query: string; filter: ListFilter; sort: ListSort; used: boolean } +export interface ListParams { query: string; filter: ListFilter; sort: ListSort; used: boolean; due?: boolean } export function listParamsOf(p: URLSearchParams): ListParams { const filter = p.get('filter') as ListFilter | null @@ -169,6 +177,7 @@ export function listParamsOf(p: URLSearchParams): ListParams { filter: filter && LIST_FILTERS.includes(filter) ? filter : 'all', sort: sort && LIST_SORTS.includes(sort) ? sort : 'usage', used: p.get('used') === '1', + due: p.get('profile') === 'due', } } @@ -180,6 +189,7 @@ export function writeListParams(p: URLSearchParams, s: ListParams): URLSearchPar put('filter', s.filter === 'all' ? null : s.filter) put('sort', s.sort === 'usage' ? null : s.sort) put('used', s.used ? '1' : null) + put('profile', s.due ? 'due' : null) return next } diff --git a/frontend/src/pages/catalog/profile.ts b/frontend/src/pages/catalog/profile.ts index 236b7a99..2168fb5d 100644 --- a/frontend/src/pages/catalog/profile.ts +++ b/frontend/src/pages/catalog/profile.ts @@ -1,31 +1,38 @@ -import type { CatalogProfileNumberKey, CatalogProfileSettings } from '../../types' +import type { + CatalogLastProfile, CatalogProfileNumberKey, CatalogProfileSettings, CatalogProfileSize, CatalogProfileTaskStatus, CatalogTableRow, +} from '../../types' import { PROFILE_TEXT } from '../../lib/terms' +import { formatTime, parseServerTime } from '../../lib/format' // =========================================================================== // 数据剖析的设置:数据源 options.catalog_profile 和表单之间的来回转换、校验、保存被拒时认出是哪一项。 // 规则跟着服务端(backend/app/data/catalog_profile.py 的 ProfileSettings、profile_settings、 // profile_settings_problem)走:缺省即关闭;读的时候宽松(某一项不对就用缺省),写的时候严格(超出范围的拒收)。 +// +// 预算按表算(每张表最多查询数、每张表时长上限),没有整次的总量上限。以前的「查询次数上限」「总时长上限」按整次剖析算, +// 含义不同:服务端启动时把这两个旧键删掉、保存时摘掉,这里读的时候也不认它们(表单里没有、保存时不带)。 // =========================================================================== /** 数据源 options 里剖析设置的键 */ export const PROFILE_OPTION = 'catalog_profile' +/** 设置里的数值项,也是表单上的顺序:先是按表的两项预算,再是单条查询和抽样 */ export const PROFILE_NUMBER_KEYS: readonly CatalogProfileNumberKey[] = [ - 'max_queries', 'query_timeout_s', 'sample_size', 'max_scan_rows', 'max_total_s', + 'table_max_queries', 'table_max_s', 'query_timeout_s', 'sample_size', 'max_scan_rows', ] /** 缺省值(服务端 ProfileSettings 的默认) */ export const PROFILE_DEFAULTS: CatalogProfileSettings = { - enabled: false, max_queries: 60, query_timeout_s: 10, sample_size: 2000, max_scan_rows: 100_000, max_total_s: 120, + enabled: false, table_max_queries: 40, table_max_s: 60, query_timeout_s: 10, sample_size: 2000, max_scan_rows: 100_000, } /** 各数值项的范围(服务端 _NUMBER_FIELDS):下限、上限、是否必须是整数 */ export const PROFILE_LIMITS: Record = { - max_queries: { min: 1, max: 500, integer: true }, + table_max_queries: { min: 1, max: 200, integer: true }, + table_max_s: { min: 10, max: 600, integer: false }, query_timeout_s: { min: 1, max: 60, integer: false }, sample_size: { min: 10, max: 10_000, integer: true }, max_scan_rows: { min: 0, max: 10_000_000, integer: true }, - max_total_s: { min: 10, max: 600, integer: false }, } /** 一个数:数字和数字写成的文字都认(同服务端 _number)。不合规返回 null */ @@ -80,7 +87,8 @@ export const hasProfileOption = (options: Record | null | undef /** * 表单 → 要写进 options 的设置。原来没有(stored 为假)、现在也没开、一项都没填的不写(不给 options 平白多一个键); - * 其余写开关和填了的项。填的不是数时原样交上去,由服务端拒收并说明是哪一项(表单上已经先拦过一次) + * 其余写开关和填了的项。填的不是数时原样交上去,由服务端拒收并说明是哪一项(表单上已经先拦过一次)。 + * 旧的两个键(整次的查询次数上限、总时长上限)不在表单里,保存时不带——服务端也会摘掉 */ export function profileOptionOf(form: ProfileForm, stored: boolean): Record | undefined { const filled = PROFILE_NUMBER_KEYS.filter((k) => form.values[k].trim()) @@ -107,7 +115,7 @@ export function profileProblems(form: ProfileForm): Partial | null | undefined, value: Record | undefined): Record { + const { [PROFILE_OPTION]: _old, ...rest } = options ?? {} + return value ? { ...rest, [PROFILE_OPTION]: value } : rest +} + // --------------------------------------------------------------------------- -// 剖析:范围、报告里的读数、409 的分类 +// 剖析:范围、读数、409 的分类 // --------------------------------------------------------------------------- -/** 不指定表时服务端剖析几张(PROFILE_DEFAULT_TABLES) */ -export const PROFILE_DEFAULT_TABLES = 10 -/** 一次最多指定几张表(接口 tables 的 max_length) */ -export const PROFILE_MAX_TABLES = 200 +/** 一次最多指定几张表(接口 tables 的上限) */ +export const PROFILE_MAX_TABLES = 500 + +/** + * 能不能剖析这张表:在表结构里、不是视图、有可剖析的项(profile_checks > 0)。老服务端不给 profile_checks 时不知道, + * 按能剖析算(交给服务端,剖析不了的它会记跳过) + */ +export const isProfilable = (r: Pick): boolean => + r.in_schema && !r.is_view && (r.profile_checks == null || r.profile_checks > 0) + +/** 服务端「需要剖析」的口径:能剖析、并且 profile_due(从没剖析过、上次没做完、之后出现了新的待核实关系) */ +export const isProfileDue = (r: CatalogTableRow): boolean => isProfilable(r) && r.profile_due === true + +/** 范围的几张表:needed 需要剖析的、all 全部能剖析的(服务端挑表的顺序:有待核实关系的在前,再按使用次数) */ +export function profileScopeTables(rows: CatalogTableRow[], scope: 'needed' | 'all'): string[] { + return rows.filter(scope === 'needed' ? isProfileDue : isProfilable).map((r) => r.table_name) +} + +/** 因为运行原因没查成的项(计入 unfinished、使表成为部分完成);其余跳过的原因是按规则不查 */ +export const UNFINISHED_REASONS = new Set(['budget', 'deadline', 'failed', 'timeout', 'error', 'rejected', 'slow', 'cancelled']) + +const TASK_ENDED = new Set(['ok', 'partial', 'failed', 'skipped', 'cancelled']) +export const isProfileTaskEnded = (t: { status: CatalogProfileTaskStatus }): boolean => TASK_ENDED.has(t.status) +/** 「重试没做完的表」带上的:失败、部分完成、已停止(同服务端 retry) */ +export const PROFILE_RETRYABLE = new Set(['failed', 'partial', 'cancelled']) /** 覆盖率写成百分数:最多一位小数,不到 100% 的不进成 100%(同服务端 _percent) */ export function coverageText(ratio: number): string { @@ -151,22 +186,94 @@ export function coverageText(ratio: number): string { return text === '100' && ratio < 1 ? '99.9%' : `${text}%` } -/** 已用时长的读数:「不到 1 秒」「8 秒」「1 分 05 秒」 */ +/** 用时的读数:「不到 1 秒」「8 秒」「1 分 05 秒」「1 小时 02 分」 */ export function secondsText(ms: number): string { const s = Math.max(0, Math.floor(ms / 1000)) if (s < 1) return '不到 1 秒' - return s < 60 ? `${s} 秒` : `${Math.floor(s / 60)} 分 ${String(s % 60).padStart(2, '0')} 秒` + if (s < 60) return `${s} 秒` + if (s < 3600) return `${Math.floor(s / 60)} 分 ${String(s % 60).padStart(2, '0')} 秒` + return `${Math.floor(s / 3600)} 小时 ${String(Math.floor((s % 3600) / 60)).padStart(2, '0')} 分` } -/** 剖析接口回 409 的几种情况:按机读码认(服务端 api/coded.py,api/catalog.profile_catalog_tables 带上) */ -export type ProfileBlock = 'disabled' | 'inactive' | 'noSchema' | 'busy' | 'tampered' | 'other' +/** 预算估算里的时长:「50 秒」「40 分钟」「2 小时 10 分钟」(只是上限的估算,不写到秒) */ +export function spanText(secs: number): string { + const s = Math.max(0, Math.round(secs)) + if (s < 60) return `${s} 秒` + const m = Math.round(s / 60) + if (m < 60) return `${m} 分钟` + const h = Math.floor(m / 60) + return m % 60 ? `${h} 小时 ${m % 60} 分钟` : `${h} 小时` +} + +/** 多久以前:「刚刚」「12 分钟前」「3 小时前」「3 天前」,一个月以上写日期(表清单上「3 天前剖析」) */ +export function agoText(v: string | null | undefined, now = Date.now()): string { + const d = parseServerTime(v) + if (!d) return '' + const diff = now - d.getTime() + if (diff < 45_000) return '刚刚' + const m = Math.round(diff / 60_000) + if (m < 60) return `${Math.max(1, m)} 分钟前` + const h = Math.round(diff / 3_600_000) + if (h < 24) return `${h} 小时前` + const days = Math.floor(diff / 86_400_000) + return days <= 30 ? `${days} 天前` : formatTime(d) +} + +/** 行数估算写成人话,带来源(统计信息 / 数到上限);没有估算为 null */ +export function sizeText(size: CatalogProfileSize | null | undefined): { text: string; hint?: string; source?: string } | null { + if (!size) return null + if (size.rows != null) { + return { text: size.method === 'stats' ? PROFILE_TEXT.rowsStats(size.rows) : PROFILE_TEXT.rowsCount(size.rows), + hint: PROFILE_TEXT.rowsHint[size.method], source: PROFILE_TEXT.rowsSource[size.method] } + } + if (size.at_least != null) { + return { text: PROFILE_TEXT.rowsAtLeast(size.at_least), hint: PROFILE_TEXT.rowsHint.count, source: PROFILE_TEXT.rowsSource.count } + } + return { text: PROFILE_TEXT.rowsUnknown } +} + +/** + * 表清单上「剖析」那一列:这张表剖析到哪了。live 是页面正跟着的剖析任务里这张表的状态(排队、剖析中),比清单新; + * 没有就按清单上的最近一次剖析。tone:due 需要剖析(从没剖析过、没做完、有新关系),bad 失败,quiet 其余 + */ +export function profileCell(r: CatalogTableRow, live: CatalogProfileTaskStatus | null, now = Date.now()): + { text: string; title: string; tone: 'due' | 'bad' | 'active' | 'quiet' } | null { + const T = PROFILE_TEXT + if (live === 'pending' || live === 'running') return { text: T.listStatus[live], title: T.listStatus[live], tone: 'active' } + const last: CatalogLastProfile | null | undefined = r.last_profile + if (last === undefined && r.profile_checks === undefined) return null + if (last && (last.status === 'pending' || last.status === 'running')) { + return { text: T.listStatus[last.status], title: T.listStatus[last.status], tone: 'active' } + } + if (!last) { + if (!r.in_schema || r.is_view) return null + if (r.profile_checks === 0) return { text: T.listNoChecks, title: T.listNoChecksHint, tone: 'quiet' } + return { text: T.listNever, title: T.dueReason.never, tone: 'due' } + } + const at = last.finished_at ?? last.started_at + const when = at ? formatTime(at) : '' + const label = last.status === 'ok' ? T.listAgo(agoText(at, now)) : T.listStatus[last.status] ?? last.status + const title = [T.listHint(T.taskStatus[last.status] ?? last.status, when), last.error ?? '', + r.profile_due && r.profile_due_reason ? T.dueReason[r.profile_due_reason] : ''].filter(Boolean).join('。') + if (last.status === 'failed') return { text: label, title, tone: 'bad' } + if (r.profile_due && r.profile_due_reason === 'new_relations' && isProfilable(r)) return { text: T.listNewRelations, title, tone: 'due' } + return { text: label, title, tone: r.profile_due && isProfilable(r) ? 'due' : 'quiet' } +} + +/** 剖析接口回 409 的几种情况:按机读码认(服务端 api/coded.py) */ +export type ProfileBlock = 'disabled' | 'inactive' | 'noSchema' | 'busy' | 'tampered' | 'nothing' | 'other' + +/** 这个源已有排队、进行中的剖析任务;已经结束的任务不需要停止 */ +export const PROFILE_BUSY = 'profile_busy' +export const PROFILE_FINISHED = 'catalog_profile_finished' const PROFILE_BLOCK_CODES: Record = { profile_disabled: 'disabled', datasource_inactive: 'inactive', schema_missing: 'noSchema', - profile_busy: 'busy', + [PROFILE_BUSY]: 'busy', snapshot_tampered: 'tampered', + profile_nothing: 'nothing', } /** 没带机读码(老服务端)或认不出的按「无法开始剖析」说,给重试;不从原话里猜 */ @@ -174,11 +281,5 @@ export function profileBlockOf(e: { code?: string }): ProfileBlock { return (e.code && PROFILE_BLOCK_CODES[e.code]) || 'other' } -/** 设置里有没有剖析设置这一项(options 的其余键原样带上) */ -export function withProfileOption(options: Record | null | undefined, value: Record | undefined): Record { - const { [PROFILE_OPTION]: _old, ...rest } = options ?? {} - return value ? { ...rest, [PROFILE_OPTION]: value } : rest -} - -/** 剖析报告里一列码值的键(表名 + 列名):从报告去填含义、填完回到报告时按它对上是哪一列 */ +/** 剖析结果里一列码值的键(表名 + 列名):从结果去填含义、填完回到结果时按它对上是哪一列 */ export const fillKey = (table: string, column: string): string => `${table}\u0000${column}` diff --git a/frontend/src/pages/catalog/profileJob.ts b/frontend/src/pages/catalog/profileJob.ts new file mode 100644 index 00000000..23dd2ca7 --- /dev/null +++ b/frontend/src/pages/catalog/profileJob.ts @@ -0,0 +1,162 @@ +import { useCallback, useEffect, useRef, useState } from 'react' +import { ApiError, api } from '../../api/client' +import type { CatalogActiveProfileJob, CatalogProfileJob, CatalogProfileTaskRow, CatalogProfileTaskStatus } from '../../types' +import { PROFILE_BUSY, PROFILE_FINISHED, PROFILE_RETRYABLE, isProfileTaskEnded } from './profile' +import { isJobActive } from './draftJob' + +// =========================================================================== +// 剖析任务(/api/datasources/{id}/catalog/profile-jobs)在页面上的状态:建、停、重试,进行中每 1.5 秒取一次进度,结束就停。 +// 照起草任务(draftJob.ts 的 useDraftJob)的做法:放在页面上而不是对话框里,关掉对话框任务照常进行,页头的进度入口、 +// 结束时的提示都靠它;再打开对话框看到的还是同一个任务。 +// +// 取回来的状态只往前走:已经结束的任务不会被一个晚到的「进行中」翻回去,已结束的表数不减少(停止接口的回答和正在路上的 +// 那次取进度谁先到都一样)。 +// +// 建任务、重试撞上 409 时照原样抛给调用方:剖析的几种 409(未开启、停用、没有表结构、快照被改动、正忙、挑不出表)各有 +// 说法和下一步,由对话框、表详情去说;正忙时可以用 showActive 改去跟着进行中的那一个。 +// =========================================================================== + +/** 进行中多久取一次进度(同起草) */ +export const PROFILE_POLL_MS = 1500 + +export const retryableTasks = (j: CatalogProfileJob): CatalogProfileTaskRow[] => j.tasks.filter((t) => PROFILE_RETRYABLE.has(t.status)) + +export interface ProfileStartBody { tables?: string[]; scope?: 'needed' | 'all' } + +export function useProfileJob(sourceId: string, active: CatalogActiveProfileJob | null | undefined, on: { + /** 这一次取回来新结束的表(写进目录了,或者失败、停止了):重取清单,正在看的表在里面就重新载入 */ + onTablesEnded: (tables: string[]) => void + /** 看着它从排队、进行中变成结束(这一页上亲眼看到的,进页面时已经结束的不算) */ + onFinished: (job: CatalogProfileJob) => void +}) { + // 任务挂在数据源上:换了数据源,上一个源的任务不再显示(那边照常进行) + const [held, setHeld] = useState<{ src: string; job: CatalogProfileJob } | null>(null) + const job = held?.src === sourceId ? held.job : null + const jobRef = useRef(job) + jobRef.current = job + const sourceRef = useRef(sourceId) + sourceRef.current = sourceId + const onRef = useRef(on) + onRef.current = on + /** 点了停止、服务端还没停下来的任务 */ + const [stopping, setStopping] = useState(null) + /** 取进度失败(网络抖了一下)、或取回来的是旧状态时,原样再排一拍 */ + const [tick, setTick] = useState(0) + + /** 换上取回来的状态。不往回走;返回是否换上了 */ + const accept = useCallback((next: CatalogProfileJob, src: string): boolean => { + if (src !== sourceRef.current) return false + const cur = jobRef.current + const same = cur?.id === next.id + if (same && cur) { + if (!isJobActive(cur) && isJobActive(next)) return false + if (isJobActive(next) && next.done < cur.done) return false + } + // 新结束的表:同一个任务比上一次多结束的;头一回见到的进行中任务(别处建的、刷新页面前建的)已经结束的那些也算, + // 清单可能还没跟上。头一回见到的已结束任务(点开上一次的结果)不算,那些早就写进去了。跳过的表没写目录,不算 + const before = new Map((same && cur ? cur.tasks : []).map((t) => [t.table_name, t.status])) + const ended = same || isJobActive(next) + ? next.tasks + .filter((t) => isProfileTaskEnded(t) && t.status !== 'skipped' && !isProfileTaskEnded({ status: before.get(t.table_name) ?? 'pending' })) + .map((t) => t.table_name) + : [] + jobRef.current = next + setHeld({ src, job: next }) + if (ended.length) onRef.current.onTablesEnded(ended) + if (same && cur && isJobActive(cur) && !isJobActive(next)) onRef.current.onFinished(next) + return true + }, []) + + /** 换成另一个任务(刚建的、重试建的、正忙时进行中的那个、对话框和表详情里点开的上一次)。同一个任务照常只往前走 */ + const track = useCallback((next: CatalogProfileJob) => { + accept(next, sourceRef.current) + }, [accept]) + + // 进行中:隔 1.5 秒取一次,结束就停 + const jobId = job?.id + const jobActive = isJobActive(job) + useEffect(() => { + if (!jobId || !jobActive) return + const src = sourceId + let alive = true + const t = setTimeout(async () => { + try { + const { job: next } = await api.dataCatalog.profileJobs.get(src, jobId) + if (alive && !accept(next, src)) setTick((n) => n + 1) + } catch (e) { + if (!alive) return + // 任务没了(不属于这个数据源、库被换掉了):不再跟着它;别的失败(网络抖动、服务重启中)下一拍再取 + if (e instanceof ApiError && e.status === 404) setHeld((h) => (h?.job.id === jobId ? null : h)) + else setTick((n) => n + 1) + } + }, PROFILE_POLL_MS) + return () => { + alive = false + clearTimeout(t) + } + }, [sourceId, jobId, jobActive, job, tick, accept]) + + // 清单说有进行中的任务、这一页还不知道(别的标签页建的、刷新了页面、表详情里建的):取来跟着它 + const activeId = isJobActive(active) ? active?.id : undefined + useEffect(() => { + if (!activeId) return + const cur = jobRef.current + if (cur?.id === activeId) return + const src = sourceId + let alive = true + api.dataCatalog.profileJobs.get(src, activeId).then(({ job: next }) => { if (alive) accept(next, src) }, () => {}) + return () => { alive = false } + }, [sourceId, activeId, accept]) + + /** 建任务。409、422 等照原样抛给调用方 */ + const start = useCallback(async (body: ProfileStartBody): Promise => { + const src = sourceRef.current + const { job: next } = await api.dataCatalog.profileJobs.create(src, body) + accept(next, src) + return next + }, [accept]) + + /** 重试没做完的表:新任务只含失败、部分完成、已停止的表。409(正忙)、422(没有可重试的)照原样抛 */ + const retry = useCallback(async (from: CatalogProfileJob): Promise => { + const src = sourceRef.current + const { job: next } = await api.dataCatalog.profileJobs.retry(src, from.id) + accept(next, src) + return next + }, [accept]) + + /** 正忙(409 profile_busy)时改去跟着进行中的那一个;没有了返回 null */ + const showActive = useCallback(async (): Promise => { + const src = sourceRef.current + const { job: current } = await api.dataCatalog.profileJobs.active(src) + if (current) accept(current, src) + return current + }, [accept]) + + /** + * 停止。服务端停下正在执行的那条查询(由数据库按单条时限截止)之前任务还是进行中:这期间 stopping 为这个任务。 + * 已经结束了(409)就取一次最新的 + */ + const cancel = useCallback(async () => { + const cur = jobRef.current + if (!cur || !isJobActive(cur)) return + const src = sourceRef.current + setStopping(cur.id) + try { + const { job: next } = await api.dataCatalog.profileJobs.cancel(src, cur.id) + accept(next, src) + } catch (e) { + if (e instanceof ApiError && e.status === 409 && e.code === PROFILE_FINISHED) { + const { job: next } = await api.dataCatalog.profileJobs.get(src, cur.id) + accept(next, src) + return + } + setStopping(null) + throw e + } + }, [accept]) + + return { job, track, start, retry, showActive, cancel, stopping: !!job && stopping === job.id && isJobActive(job) } +} + +/** 正忙(409)的错误:这个源已有排队、进行中的剖析任务 */ +export const isProfileBusy = (e: unknown): boolean => e instanceof ApiError && e.status === 409 && e.code === PROFILE_BUSY diff --git a/frontend/src/types.ts b/frontend/src/types.ts index edd239da..78de787e 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -2839,6 +2839,17 @@ export interface CatalogTableRow { updated_by: string | null /** 这张表最近一次起草任务里的那一项(任务还在进行就是进行中的状态);没起草过为 null。老服务端没有这个键 */ last_draft?: CatalogLastDraft | null + /** 这张表最近一次剖析(任务还在进行就是进行中的状态);没剖析过为 null。老服务端没有这个键 */ + last_profile?: CatalogLastProfile | null + /** 按「需要剖析」的口径这张表要不要剖析(表结构里、不是视图、有可剖析的项,且满足 profile_due_reason 之一) */ + profile_due?: boolean + /** + * 为什么需要剖析:never 从没剖析过;unfinished 最近一次剖析没做完(部分完成、失败、已停止);new_relations 目录里有待核实的 + * 关系不在最近一次剖析的检查项里。不需要时为 null + */ + profile_due_reason?: CatalogProfileDueReason | null + /** 按当前目录和表结构这张表要做几项检查;0 表示没有可剖析的项(视图为 0) */ + profile_checks?: number } /** GET /datasources/{id}/catalog */ @@ -2857,6 +2868,8 @@ export interface CatalogList { guidance?: string | null /** 排队或进行中的起草任务(页面据此显示进度);没有为 null。老服务端没有这一项 */ active_job?: CatalogActiveDraftJob | null + /** 排队或进行中的剖析任务(页面据此显示进度);没有为 null。老服务端没有这一项 */ + active_profile_job?: CatalogActiveProfileJob | null } /** 表结构里的一列 */ @@ -2899,6 +2912,8 @@ export interface CatalogDetail { structure: CatalogStructure | null system_notes: boolean usage: number + /** 这张表最近一次剖析的完整结果(含检查清单、发现、发过的查询;任务还在进行就是进行中的状态);没剖析过为 null */ + last_profile?: CatalogProfileTaskDetail | null } /** 单项审阅:确认、驳回、撤销审阅(回到来源的初始状态;人工填写的项直接删掉) */ @@ -3027,6 +3042,11 @@ export interface CatalogDraftJob { notice?: string | null /** 任务整体出错的原因 */ error: string | null + /** + * 排在同一个数据源的剖析任务后面(起草和剖析同源串行,按建立顺序):剖析结束后自动开始。不在等时为 null; + * 老服务端没有这个键 + */ + waiting_for?: 'profile' | null tasks: CatalogDraftTask[] } @@ -3036,6 +3056,8 @@ export interface CatalogActiveDraftJob { status: CatalogDraftJobStatus total: number done: number + /** 排在剖析任务后面(同 CatalogDraftJob.waiting_for) */ + waiting_for?: 'profile' | null } /** 清单一行上这张表最近一次起草的结果 */ @@ -3303,26 +3325,68 @@ export interface CatalogImpact { } // =========================================================================== -// 数据剖析(后端 app/data/catalog_profile.py):对业务库发少量只读查询,核实推断的关联关系、取码值候选、 -// 提议业务日期。默认关闭,按数据源在 options.catalog_profile 里开启。 +// 数据剖析(后端 app/data/catalog_profile.py、catalog_profile_jobs.py):对业务库发少量只读查询,核实推断的关联关系、 +// 取码值候选、提议业务日期。默认关闭,按数据源在 options.catalog_profile 里开启。 +// 剖析是服务端的后台任务(/api/datasources/{id}/catalog/profile-jobs):逐表剖析,每张表有自己的查询次数和时长预算, +// 每张表一结束就写进目录;查询一条一条发,可以随时停止。和同一个数据源的起草任务按建立顺序排队,一个结束了另一个才开始。 +// 服务重启时还没结束的任务记 interrupted,不自动续跑(由人点「重试没做完的表」)。 // =========================================================================== /** 剖析设置里的数值项 */ -export type CatalogProfileNumberKey = 'max_queries' | 'query_timeout_s' | 'sample_size' | 'max_scan_rows' | 'max_total_s' +export type CatalogProfileNumberKey = 'table_max_queries' | 'table_max_s' | 'query_timeout_s' | 'sample_size' | 'max_scan_rows' -/** 一个数据源的剖析设置(服务端 ProfileSettings.to_dict) */ +/** 一个数据源的剖析设置(服务端 ProfileSettings.to_dict);任务里的 settings 是建任务时的快照 */ export interface CatalogProfileSettings { enabled: boolean - max_queries: number + /** 每张表最多发几条查询,用完只停这张表 */ + table_max_queries: number + /** 每张表最多剖析几秒,到时只停这张表 */ + table_max_s: number + /** 单条查询时限(不超过数据源自身的查询时限,也不超过这张表剩下的时长) */ query_timeout_s: number sample_size: number /** 0 表示一律不做整表统计 */ max_scan_rows: number - max_total_s: number } -/** 整次剖析中途停下的原因:查询次数用完、总时长用完、连续多条查询失败 */ -export type CatalogProfileStop = 'budget' | 'deadline' | 'failed' +/** 为什么需要剖析(GET /catalog 每行的 profile_due_reason) */ +export type CatalogProfileDueReason = 'never' | 'unfinished' | 'new_relations' + +/** 剖析的范围:selected 指定的表;needed 需要剖析的表;all 全部可剖析的表(不含视图) */ +export type CatalogProfileScope = 'selected' | 'needed' | 'all' + +/** + * 剖析任务的状态:排队、进行中、完成、部分完成(有失败、部分完成、已停止的表)、失败(任务整体出错)、已停止(使用者停止)、 + * 服务重启中断。和起草任务同一套 + */ +export type CatalogProfileJobStatus = CatalogDraftJobStatus + +/** + * 任务里一张表的状态。ok:每项检查都有结论;partial:有项因为运行原因没查成(unfinished);failed:写入一直冲突或整张表出错; + * skipped:视图、表结构里没有、空表、没有可剖析的项;cancelled:停止、止损、服务重启时还没开始、没做完 + */ +export type CatalogProfileTaskStatus = CatalogDraftTaskStatus + +/** 跳过一项的原因(CatalogProfileSkip.reason、检查项的 reason) */ +export type CatalogProfileSkipReason = 'budget' | 'deadline' | 'failed' | 'timeout' | 'error' | 'rejected' | 'slow' | 'cancelled' + | 'masked' | 'too_large' | 'view' | 'unsupported' | 'no_data' | 'missing' | 'high_cardinality' | 'nothing' + +/** 一类检查:关联关系、码值、日期列 */ +export type CatalogProfileCheckKind = 'relation' | 'codes' | 'date' + +/** 发现的合计(任务里是逐表合计) */ +export interface CatalogProfileSummary { + /** 有结论的关系条数 */ + relations_checked: number + /** 其中状态为已验证的 */ + relations_verified: number + /** 取到码值候选的列数 */ + codes: number + /** 这些码值里含义还空着的个数之和 */ + codes_pending: number + /** 提议了业务日期的表数(逐表时 0 或 1) */ + business_dates: number +} /** 一张表的行数:stats 数据库的统计信息(估算);count 数到上限为止;unknown 没能得到 */ export interface CatalogProfileSize { @@ -3386,43 +3450,146 @@ export interface CatalogProfileSkip { kind: string target: string path: string | null - /** budget / deadline / failed / timeout / error / rejected / masked / too_large / view / unsupported / no_data / missing / high_cardinality */ - reason: string + /** + * 运行原因没查成(计入 unfinished、使表成为部分完成):budget 这张表的查询次数用完、deadline 这张表的时长用完、failed 连续失败 + * 止损、timeout、error、rejected 未通过只读守卫、slow 超时比例止损、cancelled 停止或服务重启。 + * 按规则不查(不算没做完):masked、too_large、view、unsupported、no_data、missing、high_cardinality、nothing 没有可剖析的项 + */ + reason: CatalogProfileSkipReason | (string & {}) detail: string } -export interface CatalogProfileTable { +/** 任务里的一张表(按 seq) */ +export interface CatalogProfileTaskRow { + seq: number table_name: string + status: CatalogProfileTaskStatus + checks_total: number + /** 结束了的项(含跳过的) */ + checks_done: number + /** 每类检查 [结束了的, 一共];这类没有检查时是 [0, 0] */ + checks_by_kind: Record + /** 进行中时正在做哪一项的整句(如「正在核对 orders.customer_id → customers.id」);其余时候 null */ + current: string | null queries: number + elapsed_ms: number row_estimate: CatalogProfileSize | null - findings: CatalogProfileFinding[] - skipped: CatalogProfileSkip[] - /** 日期类列的取值范围 */ - date_ranges: { column: string; min: string; max: string }[] + summary: CatalogProfileSummary + /** 跳过的项数 */ + skipped_count: number + /** 其中因为运行原因没查成的项数 */ + unfinished: number added: number updated: number removed: number version: number - /** 这张表没剖析(表结构里没有)或结果没写进去(写入一直冲突) */ + /** 整张表的原因:表结构里没有、视图、空表、没有可剖析的项、写入一直冲突、停止 / 中断 / 止损 */ error: string | null + started_at: string | null + finished_at: string | null } -/** POST /datasources/{id}/catalog/profile */ -export interface CatalogProfileOut { - profiled_at: string - actor: string | null +/** 检查清单里的一项 */ +export interface CatalogProfileCheck { + kind: CatalogProfileCheckKind + /** 哪一项:关系写「customer_id → customers.id」,码值、日期写列名 */ + target: string + /** 审阅路径;没有为 null */ + path: string | null + state: 'pending' | 'running' | 'done' | 'skipped' + /** 跳过的原因(同 CatalogProfileSkip.reason);没跳过为 null */ + reason: CatalogProfileSkipReason | (string & {}) | null + /** 可以直接显示的整句(结论或跳过的原因);没有为 null */ + detail: string | null + queries: number +} + +/** 发过的一条查询。sql 的字面量都换成了 ?(IN 列表里抽到的键值不留在 AgentLab 的库里) */ +export interface CatalogProfileQuery { + at: string + kind: 'row_estimate' | CatalogProfileCheckKind + target: string + sql: string + ms: number + /** 返回的行数;没返回(超时、出错、守卫拒)时为 null */ + rows: number | null + outcome: 'ok' | 'timeout' | 'error' | 'rejected' + /** 超时、出错、守卫拒时的原因 */ + detail: string | null +} + +/** GET /profile-jobs/{id}/tasks/{seq}:一张表的完整结果;GET /catalog/{table} 的 last_profile 也是这个形状 */ +export interface CatalogProfileTaskDetail extends CatalogProfileTaskRow { + job_id: string + job_status: CatalogProfileJobStatus + created_by: string | null + checks: CatalogProfileCheck[] + findings: CatalogProfileFinding[] + skipped: CatalogProfileSkip[] + /** 日期类列的取值范围 */ + date_ranges: { column: string; min: string; max: string }[] + query_log: CatalogProfileQuery[] +} + +/** 一次剖析任务 */ +export interface CatalogProfileJob { + id: string + source_id: string + status: CatalogProfileJobStatus + scope: CatalogProfileScope + /** 建任务时的剖析设置快照 */ settings: CatalogProfileSettings - queries_used: number - stopped: CatalogProfileStop | null - tables: CatalogProfileTable[] - total: { added: number; updated: number; removed: number } - /** - * 不指定表时,挑表途中估过行数、确知是空表而跳过的表(老后端没有这个键;指定了表时是空列表)。 - * 估行数的查询算在 queries_used 里,不算在各表的 queries 里 - */ - empty_tables?: string[] - /** 一张表都没有剖析时的说明 */ - note: string | null + created_at: string + started_at: string | null + finished_at: string | null + /** 谁建的(X-Actor) */ + created_by: string | null + /** 重试建的任务记着原任务的 id */ + retry_of: string | null + /** 排在同一个数据源的起草任务后面:起草结束后自动开始。不在等时为 null */ + waiting_for: 'draft' | null + total: number + /** 已结束的表数 */ + done: number + counts: Record + /** 整个任务发出的查询数、其中超时的、失败的 */ + queries: number + timeouts: number + errors: number + /** 任务级止损停下的原因:failed 连续 3 条查询失败;slow 最近 10 条里 7 条超时。没停为 null */ + stop_reason: 'failed' | 'slow' | null + /** 最近查询多数超时时的提醒(一句中文);好转后服务端清掉 */ + notice: string | null + /** 任务整体出错或止损的整句原因 */ + error: string | null + added: number + updated: number + removed: number + summary: CatalogProfileSummary + tasks: CatalogProfileTaskRow[] +} + +/** 清单一行上这张表最近一次剖析 */ +export interface CatalogLastProfile { + job_id: string + seq: number + status: CatalogProfileTaskStatus + started_at: string | null + finished_at: string | null + summary: CatalogProfileSummary + skipped_count: number + unfinished: number + error: string | null + row_estimate: CatalogProfileSize | null +} + +/** 目录清单顶上的进行中剖析任务(GET /catalog 的 active_profile_job) */ +export interface CatalogActiveProfileJob { + id: string + status: CatalogProfileJobStatus + total: number + done: number + waiting_for: 'draft' | null } // =========================================================================== From e7ab6a14c5b4fc831b8b67dafe2dd27f5b76cded Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Thu, 15 Oct 2026 18:33:46 +0800 Subject: [PATCH 2/4] =?UTF-8?q?=E6=95=B0=E6=8D=AE=E5=89=96=E6=9E=90?= =?UTF-8?q?=E7=9A=84=E6=B5=8F=E8=A7=88=E5=99=A8=E6=A3=80=E6=9F=A5=EF=BC=9A?= =?UTF-8?q?=E6=96=B0=E5=BB=BA=20check-catalog-profile=EF=BC=8Ccheck-catalo?= =?UTF-8?q?g=20=E9=87=8C=E5=90=8C=E6=AD=A5=E6=8E=A5=E5=8F=A3=E9=82=A3?= =?UTF-8?q?=E5=A5=97=E5=89=96=E6=9E=90=E6=A3=80=E6=9F=A5=E5=88=A0=E6=8E=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit check-catalog-profile 照 check-catalog 的写法:数据源、目录、剖析任务、起草任务都用 page.route 伪造(虚构的网店业务库, 表名 orders、customers、visits 这类),任务怎么往前走由检查自己推,没配的写请求一律 503,最后核对真库没变。查的是: 范围与个数(按清单行上的 profile_due、profile_checks、视图算)、按表写的预算和这次最多约几条查询、只读一条一条发可随时 停止的说明、进行中逐表行和各类检查进度、页头进度入口和提醒、停止、展开逐表的检查清单和发过的查询、结束后的汇总筛选和 重试、止损和服务重启、409 六种情况加老服务端没带机读码、和起草互相排队的说法、表清单的「剖析」一列和「需要剖析」筛选、 表详情的一节和「剖析这张表」(含正忙时说清这张表在不在那个任务里)、设置两项新字段和被拒时报错落到那一项、窄屏。 同步接口那一节里顺带查的码值弹窗(光标落在空着的码值上、回车保存只换那一列、填完回到结果接着下一列)一并搬过来。 check-catalog 删掉旧的剖析设置、剖析、剖析 409 三节,假服务端的同步剖析接口换成剖析任务(取进行中、最近一次答空, 建任务一律 503);其余各节不动。check-all 的 ORDER、ESTIMATE 登记 catalog-profile。 界面顺带改三处(截图里看出来的):结束后的进度写做完了几张(同起草,停止、失败、跳过的不算);没开始就停下的表不写 各类检查的 0/N;表详情一节里码值候选不在目录里时不给「填写含义」(码值弹窗改的是目录里那一项)。 --- frontend/src/lib/terms.ts | 2 + frontend/src/pages/catalog/ProfileDialog.tsx | 31 +- frontend/src/pages/catalog/TableDetail.tsx | 3 +- scripts/check-all.mjs | 3 +- scripts/check-catalog-profile.mjs | 1439 ++++++++++++++++++ scripts/check-catalog.mjs | 492 +----- 6 files changed, 1470 insertions(+), 500 deletions(-) create mode 100644 scripts/check-catalog-profile.mjs diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index 89763c3a..e15408e5 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -2891,6 +2891,8 @@ export const PROFILE_TEXT = { // 停止时正在执行的那条查询由数据库按单条时限截止,不是立刻停 stopping: '正在停止:等正在执行的那条查询结束(不超过单条查询时限),已做完的项照常写进目录…', progress: (done: number, total: number) => `${formatNumber(done)} / ${formatNumber(total)} 张表`, + /** 结束以后:做完了几张(停止、失败、跳过的不算) */ + progressDone: (ok: number, total: number) => `完成 ${formatNumber(ok)} / ${formatNumber(total)} 张表`, queriesSent: (n: number) => `已发 ${formatNumber(n)} 条查询`, queriesBad: (timeouts: number, errors: number) => [ timeouts ? `超时 ${formatNumber(timeouts)} 条` : '', errors ? `失败 ${formatNumber(errors)} 条` : '', diff --git a/frontend/src/pages/catalog/ProfileDialog.tsx b/frontend/src/pages/catalog/ProfileDialog.tsx index c3ad94e1..1b7ee42e 100644 --- a/frontend/src/pages/catalog/ProfileDialog.tsx +++ b/frontend/src/pages/catalog/ProfileDialog.tsx @@ -125,7 +125,10 @@ export function ProfileDialog({ const tooMany = !!picked.body.tables && picked.tables.length > PROFILE_MAX_TABLES const [mode, setMode] = useState<'setup' | 'job' | 'blocked'>(initial === 'job' && job ? 'job' : 'setup') - const [blocked, setBlocked] = useState<{ kind: ProfileBlock; message: string; active: CatalogProfileJob | null; body: ProfileStartBody } | null>(null) + /** 开始不了(409):哪种情况、服务端原话、正忙时进行中的那一个、当时的范围和剖析设置(就地开启后据此回到选范围) */ + const [blocked, setBlocked] = useState<{ + kind: ProfileBlock; message: string; active: CatalogProfileJob | null; body: ProfileStartBody; settings: CatalogProfileSettings + } | null>(null) const [starting, setStarting] = useState(false) const [startError, setStartError] = useState(null) /** 是 409(正忙)转过来显示的那一个 */ @@ -160,7 +163,7 @@ export function ProfileDialog({ if (e instanceof ApiError && e.status === 409) { const kind = profileBlockOf(e) const active = isProfileBusy(e) ? await api.dataCatalog.profileJobs.active(sourceId).then((r) => r.job, () => null) : null - setBlocked({ kind, message: e.message, active, body }) + setBlocked({ kind, message: e.message, active, body, settings }) setMode('blocked') } else { setStartError(PT.startFailed(errorMessage(e))) @@ -227,13 +230,13 @@ export function ProfileDialog({ setMode('job') } - // 因为没开启被拒的:就地开了(设置存好、这里拿到的设置变成开启)就回到选范围,可以直接开始 + // 因为没开启被拒的:就地开了(被拒之后设置存过、这里拿到的是新的一份,并且开着)就回到选范围,可以直接开始 useEffect(() => { - if (mode === 'blocked' && blocked?.kind === 'disabled' && settings.enabled) { + if (mode === 'blocked' && blocked?.kind === 'disabled' && settings !== blocked.settings && settings.enabled) { setBlocked(null) setMode('setup') } - }, [mode, blocked, settings.enabled]) + }, [mode, blocked, settings]) const phase: 'setup' | 'blocked' | 'running' | 'done' = inSetup ? 'setup' : mode === 'blocked' ? 'blocked' : isJobActive(job) ? 'running' : 'done' const canStart = settings.enabled && picked.tables.length > 0 && !tooMany && !starting @@ -441,10 +444,13 @@ function JobView({ sourceId, job, running, stopping, busy, filled, returnTo, onS const started = job.started_at ? Date.parse(job.started_at) : NaN const ended = job.finished_at ? Date.parse(job.finished_at) : NaN const elapsedMs = Number.isFinite(started) ? (running ? Math.max(now, Date.now()) : Number.isFinite(ended) ? ended : started) - started : null - const pct = job.total ? (job.done / job.total) * 100 : 0 + // 进行中按已结束的表数走(真进度);结束以后写做完了几张(停止、失败、跳过的不算「完成」),同起草 + const ok = job.counts.ok ?? job.tasks.filter((t) => t.status === 'ok').length + const value = running ? job.done : ok + const pct = job.total ? (value / job.total) * 100 : 0 const bad = job.tasks.some((t) => PROFILE_RETRYABLE.has(t.status)) const tone = job.status === 'failed' ? 'var(--err)' : running ? 'var(--accent)' : bad ? 'var(--warn)' : 'var(--st-done)' - const progress = PT.progress(job.done, job.total) + const progress = running ? PT.progress(job.done, job.total) : PT.progressDone(ok, job.total) const badQueries = PT.queriesBad(job.timeouts, job.errors) const notice = running ? job.notice?.trim() ?? '' : '' const changed = job.added + job.updated + job.removed @@ -480,7 +486,7 @@ function JobView({ sourceId, job, running, stopping, busy, filled, returnTo, onS
    + aria-valuemin={0} aria-valuemax={job.total} aria-valuenow={value}>
    {queued && job.waiting_for === 'draft' && ( @@ -595,7 +601,8 @@ function TaskItem({ sourceId, job, task: t, open, onToggle, filled, focusAfter, const size = t.status !== 'skipped' ? sizeText(t.row_estimate) : null const usedUp = t.queries > 0 && t.queries >= job.settings.table_max_queries const note = taskNote(t, job.error) - const started = t.status !== 'pending' + // 没开始就停下的表(停止、止损、服务重启时还在排队)不写各类检查的 0/N:它一项都没做 + const started = !!t.started_at || t.queries > 0 const panelId = `catalog-profile-task-${t.seq}` return (
  • @@ -688,13 +695,15 @@ function TaskDetailLoader({ sourceId, jobId, task: t, filled, focusAfter, onOpen * mode detail:表详情里,写行数和来源、日期范围、发现、跳过的项、发过的查询(检查清单在任务里看)。 * onFill 不给时码值不带「填写含义」 */ -export function ProfileTaskBody({ detail: d, mode, filled = {}, focusAfter = null, onOpenTable, onFill }: { +export function ProfileTaskBody({ detail: d, mode, filled = {}, focusAfter = null, onOpenTable, onFill, canFill }: { detail: CatalogProfileTaskDetail mode: 'job' | 'detail' filled?: Record focusAfter?: string | null onOpenTable?: (table: string) => void onFill?: (column: string) => void + /** 这一列能不能填含义(码值候选还在目录里);不给就是都能填 */ + canFill?: (column: string) => boolean }) { const relations = d.findings.filter((f): f is CatalogProfileRelationFinding => f.kind === 'relation') const codes = d.findings.filter((f): f is CatalogProfileCodesFinding => f.kind === 'codes') @@ -746,7 +755,7 @@ export function ProfileTaskBody({ detail: d, mode, filled = {}, focusAfter = nul {codes.map((f) => ( onFill(f.column) : undefined} /> + onFill={onFill && (!canFill || canFill(f.column)) ? () => onFill(f.column) : undefined} /> ))} )} diff --git a/frontend/src/pages/catalog/TableDetail.tsx b/frontend/src/pages/catalog/TableDetail.tsx index ebfbc768..e6f0d69a 100644 --- a/frontend/src/pages/catalog/TableDetail.tsx +++ b/frontend/src/pages/catalog/TableDetail.tsx @@ -1191,7 +1191,8 @@ function ProfileSection({ table, detail, row, enabled, profile, onFillCodes }: { {PT.findings(last.summary, true) && (

    {PT.findings(last.summary, true)}

    )} - + {/* 码值候选还在目录里(没被改掉、删掉)才给「填写含义」:码值弹窗改的是目录里的那一项 */} + !!detail.notes.columns?.[col]?.codes} /> )}
  • diff --git a/scripts/check-all.mjs b/scripts/check-all.mjs index 81505e44..48befcc9 100644 --- a/scripts/check-all.mjs +++ b/scripts/check-all.mjs @@ -82,6 +82,7 @@ const ORDER = [ 'versions', // 上传表格的版本:回滚、清单、移除一期、清除原件 'catalog', // 数据目录:表清单、起草、单项审阅、批量确认、编辑即确认、409 'catalog-loop', // 数据目录闭环:表助手、审阅队列、导出导入、防错事实的说明 + 'catalog-profile', // 数据剖析:逐表的后台任务、表清单的剖析一列、表详情的一节、剖析设置 // 复核套件与导出 'schedules', // 定时任务 'compare', // 运行对比 @@ -109,7 +110,7 @@ const ORDER = [ const ESTIMATE = { challenge: 30, stream: 173, chat: 160, studio: 149, manage: 145, runs: 112, publish: 92, shell: 87, - evidence: 91, versions: 60, catalog: 150, 'catalog-loop': 60, 'ui-kit': 70, ui: 61, guards: 8, 'canvas-layout': 2, copy: 2, tokens: 1, trace: 1, decode: 1, + evidence: 91, versions: 60, catalog: 150, 'catalog-loop': 60, 'catalog-profile': 90, 'ui-kit': 70, ui: 61, guards: 8, 'canvas-layout': 2, copy: 2, tokens: 1, trace: 1, decode: 1, // 占位,换成真检查时按实测改 schedules: 60, compare: 45, verify: 71, inquiry: 60, bundle: 16, whatif: 33, 'report-export': 30, // 应用平台:还没实测,按同类页面估(全是伪造接口、不连真数据) diff --git a/scripts/check-catalog-profile.mjs b/scripts/check-catalog-profile.mjs new file mode 100644 index 00000000..3dadac4f --- /dev/null +++ b/scripts/check-catalog-profile.mjs @@ -0,0 +1,1439 @@ +// 数据剖析(数据目录页的「数据剖析」、表清单的「剖析」一列、表详情的「数据剖析」一节、数据源的剖析设置)的交互回归。 +// +// 剖析会对业务库发查询,是服务端逐表进行的后台任务(/catalog/profile-jobs)。这里查的是正常路径的截图看不出来的那些: +// 「开始前的范围和个数是不是按清单算的」「预算是不是按表写的、估了这次最多几条查询」「明说只读、一条一条发、可以停」 +// 「同源有起草任务时是不是写排队」「进行中每张表到哪了(各类检查几项、发了几条查询、正在做哪一项)」「停得下来、停下来 +// 写清做完的保留了什么」「展开一张表看得到检查清单和发过的查询(字面量换成了 ?)」「结束后能按有发现 / 有跳过 / 失败筛、 +// 只重试没做完的表」「止损、服务重启的原因照实写」「各种 409 各有下一步」「表清单的剖析一列和需要剖析的筛选」「表详情的一节 +// 和剖析这张表」「设置的两项新字段、被拒时报错落到那一项」「起草排在剖析后面时怎么说」。 +// +// 写法同 check-catalog:数据源列表、目录清单、单表目录、剖析任务、起草任务都用 page.route 伪造(虚构的网店业务库,表名 +// orders、customers、visits……都是通用的假名字),任务怎么往前走由检查自己推(settle、finish、stopJob),页面每 1.5 秒 +// 取一次,看到的就是推到的那一步。没配的写请求一律拦成 503,不写库、更不会发到业务库;最后核对真库里的数据源和目录 +// 数量没有变化。 +// 跑之前前后端都得起着(./scripts/dev.sh),默认连 5273 / 8000;对别的实例跑时带上地址: +// AGENTLAB_WEB=http://localhost:<前端端口> AGENTLAB_API=http://localhost:<后端端口>/api node scripts/check-catalog-profile.mjs +// 只跑其中几节:CHECK_ONLY=设置,409 node scripts/check-catalog-profile.mjs(按节名包含匹配)。 +// 截图:CHECK_SHOTS=/某个目录 时把关键状态存下来;CHECK_THEME=light 换浅色跑一遍。 +import { chromium } from '../frontend/node_modules/playwright-core/index.mjs' + +const WEB = process.env.AGENTLAB_WEB ?? 'http://localhost:5273' +const API = process.env.AGENTLAB_API ?? 'http://localhost:8000/api' +const SHOTS = process.env.CHECK_SHOTS ?? '' +const THEME = process.env.CHECK_THEME === 'light' ? 'light' : 'dark' +// 不用 playwright install:系统 Chrome 就够了 +const CHROME = process.env.CHROME_PATH ?? '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome' +/** 本机署名:写请求带的 X-Actor */ +const ACTOR = '检查脚本' +/** 夹具里另一个人建的剖析任务 */ +const OTHER = '检查脚本乙' + +let failed = 0 +const check = (name, cond, detail = '') => { + console.log(` ${cond ? '✓' : '✗'} ${name}${detail ? ` — ${detail}` : ''}`) + if (!cond) failed++ +} + +try { + const r = await fetch(`${API}/datasources/kinds`) + if (!r.ok) throw new Error(`HTTP ${r.status}`) +} catch (e) { + console.error(`✗ 连不上后端(${API})——先起沙箱后端\n ${e.message}`) + process.exit(1) +} + +// --------------------------------------------------------------------------- +// 真库的数量:开头记一次,最后再数一次,必须一样(检查本身不许写库) +// --------------------------------------------------------------------------- +async function realCounts() { + const sources = await (await fetch(`${API}/datasources`)).json() + let entries = 0 + let versions = 0 + for (const s of sources) { + const r = await fetch(`${API}/datasources/${s.id}/catalog`) + if (!r.ok) continue + const d = await r.json() + for (const t of d.tables ?? []) { + if (t.version > 0) entries++ + versions += t.version + } + } + return { sources: sources.length, entries, versions } +} +const before = await realCounts() + +const browser = await chromium.launch({ executablePath: CHROME }) +const opened = new Set() + +/** + * 开一页:handlers 按「METHOD 路径正则」匹配(GET 也可以配),没配的 GET 放行到后端,没配的写请求一律 + * 拦成 503。所有原生对话框都算失败 + */ +async function open(path, { handlers = [], viewport = { width: 1440, height: 900 } } = {}) { + const ctx = await browser.newContext({ viewport, colorScheme: THEME, timezoneId: 'Asia/Shanghai' }) + opened.add(ctx) + ctx.setDefaultTimeout(6000) + ctx.setDefaultNavigationTimeout(30000) + await ctx.addInitScript(([t, actor]) => { + try { + localStorage.setItem('agentlab.theme', t) + localStorage.removeItem('agentlab.health') + localStorage.setItem('agentlab_actor', actor) + } catch { /* noop */ } + }, [THEME, ACTOR]) + const page = await ctx.newPage() + const sent = [] + const errors = [] + const natives = [] + page.on('pageerror', (e) => errors.push(e.message)) + page.on('dialog', (d) => { natives.push(`${d.type()}: ${d.message().slice(0, 40)}`); void d.dismiss() }) + await page.route((u) => new URL(u).pathname.startsWith('/api/'), async (route) => { + const r = route.request() + const url = new URL(r.url()) + const key = `${r.method()} ${decodeURIComponent(url.pathname.replace(/^\/api/, ''))}` + for (const [pattern, handle] of handlers) { + if (pattern.test(key)) { + let body = null + try { body = r.postDataJSON() } catch { body = r.postData() } + sent.push({ key, body, actor: r.headers()['x-actor'] ?? null }) + return handle(route, { body, url, key }) + } + } + if (r.method() === 'GET') return route.continue() + sent.push({ key, body: null, blocked: true }) + return route.fulfill({ status: 503, body: '' }) + }) + await page.goto(`${WEB}${path}`, { waitUntil: 'networkidle' }) + await page.evaluate((t) => document.documentElement.setAttribute('data-theme', t), THEME) + await page.waitForTimeout(400) + // 关之前先撤掉拦截、忽略还在路上的那几条:满负荷并行时,取真实返回的拦截可能在页面关掉之后才回来 + return { page, sent, errors, natives, close: async () => { + opened.delete(ctx) + await page.unrouteAll({ behavior: 'ignoreErrors' }).catch(() => {}) + return ctx.close() + } } +} +const goto = async (page, path) => { + await page.goto(`${WEB}${path}`, { waitUntil: 'networkidle' }) + await page.evaluate((t) => document.documentElement.setAttribute('data-theme', t), THEME) + await page.waitForTimeout(300) +} +const json = (data, status = 200) => (route) => route.fulfill({ status, contentType: 'application/json', body: JSON.stringify(data) }) +const until = async (fn, ms = 6000, step = 120) => { + const end = Date.now() + ms + let v = await fn() + while (!v && Date.now() < end) { await new Promise((r) => setTimeout(r, step)); v = await fn() } + return v +} +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)) +const shot = async (page, name) => { if (SHOTS) await page.screenshot({ path: `${SHOTS}/${name}.png` }) } +const writes = (sent, re) => sent.filter((s) => re.test(s.key) && !s.key.startsWith('GET ')) +const count = (sent, re) => sent.filter((s) => re.test(s.key)).length +const lastBody = (sent, re) => sent.filter((s) => re.test(s.key)).at(-1)?.body ?? null + +const ONLY = process.env.CHECK_ONLY?.split(',').map((x) => x.trim()).filter(Boolean) +async function section(name, fn) { + if (ONLY?.length && !ONLY.some((k) => name.includes(k))) return + console.log(`\n=== ${name} ===`) + try { + await fn() + } catch (e) { + check(`${name} 中途出错`, false, String(e?.message ?? e).split('\n')[0]) + } finally { + for (const c of opened) await c.close().catch(() => {}) + opened.clear() + } +} + +// --------------------------------------------------------------------------- +// 夹具:虚构的网店业务库(订单、客户、访问记录、商品、退款、一张空的日志表、一个视图、一张没有可剖析项的配置表、一张只剩 +// 目录的旧表)、一个没开剖析的库(剖析设置里还留着旧的两个键)、一个导入表格的源。名字、数字全是假的 +// --------------------------------------------------------------------------- +const isoNow = () => new Date().toISOString() +const ago = (days, hours = 0) => new Date(Date.now() - days * 86_400_000 - hours * 3_600_000).toISOString() +const T0 = ago(3) +const clone = (x) => JSON.parse(JSON.stringify(x)) +const item = (value, source = 'llm', status = 'proposed') => ({ value, source, status, updated_at: T0 }) +const col = (name, type = 'INTEGER', extra = {}) => ({ name, type, pk: false, not_null: false, comment: null, ...extra }) +const pk = (name = 'id') => col(name, 'INTEGER', { pk: true, not_null: true }) + +const S1 = 'check-prof-shop' +const S2 = 'check-prof-off' +const S3 = 'check-prof-upload' +const NAME1 = 'zzprofshop' +const NAME2 = 'zzprofoff' +const NAME3 = 'zzprofupload' + +/** + * 一张表。plan:按当前目录和表结构这张表要做的检查(关系写「列 → 表.列」,码值、日期写列名),profile_checks 就是它的项数; + * newRelations:剖析之后目录里又出现了待核实的关系(profile_due_reason 为 new_relations) + */ +const table = (name, usage, columns, plan, extra = {}) => ({ + name, usage, columns, plan: { relation: [], codes: [], date: [], ...plan }, notes: extra.notes ?? null, version: extra.notes ? 2 : 0, + is_view: false, missing: false, newRelations: false, updated_at: extra.notes ? T0 : null, updated_by: extra.notes ? '审核甲' : null, ...extra, +}) +const checksOf = (t) => (t.missing || t.is_view ? 0 : t.plan.relation.length + t.plan.codes.length + t.plan.date.length) + +const ORDERS_PLAN = { + relation: ['customer_id → customers.id', 'product_id → products.id', 'store_id → stores.id', 'coupon_id → coupons.id'], + codes: ['status', 'channel', 'pay_type', 'region', 'level', 'source'], + date: ['created_at'], +} +function shopTables() { + return [ + table('orders', 40, [pk(), col('order_no', 'TEXT'), col('customer_id'), col('product_id'), col('store_id'), col('coupon_id'), col('status'), + col('channel', 'TEXT'), col('pay_type'), col('region', 'TEXT'), col('level'), col('source', 'TEXT'), col('created_at', 'TEXT'), col('amount', 'REAL')], + ORDERS_PLAN, { notes: { + label: item('订单'), + columns: { + status: { codes: item({ 1: '已下单', 2: '', 9: '' }, 'profile') }, + channel: { codes: item({ web: '', app: '' }, 'profile') }, + }, + } }), + table('customers', 30, [pk(), col('customer_no', 'TEXT'), col('level'), col('joined_at', 'TEXT')], + { relation: [], codes: ['level'], date: ['joined_at'] }), + table('visits', 22, [pk(), col('customer_id'), col('page', 'TEXT'), col('device'), col('visited_at', 'TEXT')], + { relation: ['customer_id → customers.id'], codes: ['device', 'page'], date: ['visited_at'] }), + table('products', 15, [pk(), col('category_id'), col('state'), col('listed_at', 'TEXT')], + { relation: ['category_id → categories.id'], codes: ['state'], date: ['listed_at'] }, { newRelations: true }), + table('refunds', 10, [pk(), col('order_id'), col('reason')], { relation: ['order_id → orders.id'], codes: ['reason'] }), + table('tmp_log', 0, [pk(), col('logged_at', 'TEXT')], { date: ['logged_at'] }), + table('order_summary', 5, [col('day', 'TEXT'), col('total', 'REAL')], {}, { is_view: true }), + table('settings_kv', 1, [col('k', 'TEXT'), col('v', 'TEXT')], {}), + table('legacy_coupons', 0, null, {}, { missing: true, notes: { label: item('旧版优惠券', 'human', 'confirmed') } }), + ] +} + +const blank = { + kind: 'sqlite', host: null, port: null, username: null, readonly: true, description: '', enabled: true, + password_masked: '', has_password: false, cached_schema: '', schema_error: '', available_schemas: [], + last_checked_at: null, last_check_ok: null, last_latency_ms: null, last_error: null, current_snapshot: null, +} +const manual = (id, name, count, options) => ({ + ...blank, id, name, origin: 'manual', database: `/tmp/check/${name}.db`, table_count: count, tools: [`db_query__${name}`], + schema_synced_at: T0, options, +}) +/** 开着剖析的库:每张表最多 40 条、60 秒,单条 10 秒(缺省) */ +const PROFILE_ON = { enabled: true } +const sources = () => [ + manual(S1, NAME1, 8, { catalog_profile: PROFILE_ON }), + // 没开剖析的库:剖析设置里还留着旧的两个键(整次的查询次数上限、总时长上限),界面不认、保存时不带 + manual(S2, NAME2, 2, { catalog_profile: { enabled: false, max_queries: 60, max_total_s: 120 } }), + { + ...blank, id: S3, name: NAME3, origin: 'upload', table_count: 1, tools: [`db_query__${NAME3}`], schema_synced_at: T0, + database: `/tmp/check/uploads/${S3}.db`, import_mode: 'simple', current_recipe: null, open_staging: null, options: { mask_columns: ['联系电话'] }, + current_snapshot: { id: 'snap-check-prof', created_at: T0, file_name: '客流日报.xlsx', raw_state: 'kept' }, + }, +] +const NAMES = [NAME1, NAME2, NAME3] + +const DEFAULTS = { enabled: false, table_max_queries: 40, table_max_s: 60, query_timeout_s: 10, sample_size: 2000, max_scan_rows: 100000 } +const settingsOf = (state, src) => { + const raw = state.sources.find((s) => s.id === src)?.options?.catalog_profile ?? {} + const out = { ...DEFAULTS, enabled: raw.enabled === true } + for (const k of Object.keys(DEFAULTS)) if (k !== 'enabled' && typeof raw[k] === 'number') out[k] = raw[k] + return out +} + +// --------------------------------------------------------------------------- +// 假服务端的剖析任务:照接口约定 C 的形状,逐表行由检查推进的状态算出来 +// --------------------------------------------------------------------------- +const JOB_ACTIVE = new Set(['queued', 'running']) +const TASK_END = new Set(['ok', 'partial', 'failed', 'skipped', 'cancelled']) +const RETRYABLE = new Set(['failed', 'partial', 'cancelled']) +const UNFINISHED = new Set(['budget', 'deadline', 'failed', 'timeout', 'error', 'rejected', 'slow', 'cancelled']) +const KINDS = ['relation', 'codes', 'date'] + +function newProfileJob(state, src, { tables, scope, retry_of = null, created_by = ACTOR, at = null }) { + const id = `check-pjob-${++state.pjobSeq}` + const tasks = tables.map((name, i) => { + const t = state.tables[src].find((x) => x.name === name) + const why = !t || t.missing ? `表结构里没有 ${name},请先重新探查结构` : t.is_view ? '视图不剖析' : !checksOf(t) ? '没有可剖析的项' : null + const checks = why ? [] : KINDS.flatMap((kind) => t.plan[kind].map((target) => ({ + kind, target, path: kind === 'relation' ? `relations.${target}` : kind === 'codes' ? `columns.${target}.codes` : 'business_date', + state: 'pending', reason: null, detail: null, queries: 0, + }))) + return { + seq: i + 1, table_name: name, status: why ? 'skipped' : 'pending', checks, current: null, queries: 0, elapsed_ms: 0, row_estimate: null, + findings: [], skipped: [], date_ranges: [], query_log: [], added: 0, updated: 0, removed: 0, version: t?.version ?? 0, error: why, + started_at: null, finished_at: why ? (at ?? isoNow()) : null, + } + }) + const job = { + id, source_id: src, status: 'queued', scope, settings: settingsOf(state, src), created_at: at ?? isoNow(), started_at: null, finished_at: null, + created_by, retry_of, waiting_for: activeDraft(state, src) ? 'draft' : null, stop_reason: null, notice: null, error: null, tasks, + } + state.pjobs.push(job) + return job +} + +const zero = () => ({ relations_checked: 0, relations_verified: 0, codes: 0, codes_pending: 0, business_dates: 0 }) +function summaryOf(findings) { + const s = zero() + for (const f of findings) { + if (f.kind === 'relation') { s.relations_checked++; if (f.status === 'verified') s.relations_verified++ } + if (f.kind === 'codes') { s.codes++; s.codes_pending += f.pending ?? 0 } + if (f.kind === 'business_date') s.business_dates++ + } + return s +} +function taskRow(t) { + const byKind = Object.fromEntries(KINDS.map((k) => { + const list = t.checks.filter((c) => c.kind === k) + return [k, [list.filter((c) => c.state === 'done' || c.state === 'skipped').length, list.length]] + })) + const { checks, findings, skipped, date_ranges, query_log, ...rest } = t + return { + ...rest, checks_total: checks.length, checks_done: checks.filter((c) => c.state === 'done' || c.state === 'skipped').length, + checks_by_kind: byKind, summary: summaryOf(findings), skipped_count: skipped.length, + unfinished: skipped.filter((s) => UNFINISHED.has(s.reason)).length, + } +} +const taskDetail = (job, t) => clone({ ...taskRow(t), job_id: job.id, job_status: job.status, created_by: job.created_by, checks: t.checks, + findings: t.findings, skipped: t.skipped, date_ranges: t.date_ranges, query_log: t.query_log }) +function jobOut(job) { + const tasks = job.tasks.map(taskRow) + const counts = { pending: 0, running: 0, ok: 0, partial: 0, failed: 0, skipped: 0, cancelled: 0 } + for (const t of tasks) counts[t.status]++ + const sum = (k) => tasks.reduce((n, t) => n + t[k], 0) + const summary = zero() + for (const t of tasks) for (const k of Object.keys(summary)) summary[k] += t.summary[k] + const log = job.tasks.flatMap((t) => t.query_log) + return clone({ + ...Object.fromEntries(Object.entries(job).filter(([k]) => k !== 'tasks' && k !== 'stopAsked')), + total: tasks.length, done: tasks.filter((t) => TASK_END.has(t.status)).length, counts, + queries: sum('queries'), timeouts: log.filter((q) => q.outcome === 'timeout').length, errors: log.filter((q) => q.outcome === 'error').length, + added: sum('added'), updated: sum('updated'), removed: sum('removed'), summary, tasks, + }) +} + +/** + * 推进一张表。o.status 默认 ok(每项都有结论);running 时 o.done 写各类做完了几项,下一项是「进行中」。 + * o.skip:[[类, 第几项, 原因, 整句]],跳过的项也记进 skipped。o.rest:结束时剩下没做的项一律记这个原因(停止、止损) + */ +function settle(state, job, name, o = {}) { + const task = job.tasks.find((t) => t.table_name === name) + if (job.status === 'queued') Object.assign(job, { status: 'running', started_at: isoNow(), waiting_for: null }) + const status = o.status ?? 'ok' + task.started_at ??= isoNow() + for (const c of task.checks) if (c.state === 'running') c.state = 'pending' + for (const k of KINDS) { + const list = task.checks.filter((c) => c.kind === k) + const n = status === 'ok' ? list.length : o.done?.[k] ?? list.filter((c) => c.state !== 'pending').length + list.forEach((c, i) => { + if (i < n && c.state === 'pending') Object.assign(c, { state: 'done', queries: k === 'relation' ? 3 : 1, detail: c.detail ?? `${c.target}:有结论` }) + }) + } + for (const [k, i, reason, detail] of o.skip ?? []) { + const c = task.checks.filter((x) => x.kind === k)[i] + Object.assign(c, { state: 'skipped', reason, detail }) + task.skipped.push({ kind: k, target: c.target, path: c.path, reason, detail }) + } + if (o.rest) { + for (const c of task.checks) { + if (c.state !== 'pending') continue + Object.assign(c, { state: 'skipped', reason: o.rest[0], detail: o.rest[1] }) + task.skipped.push({ kind: c.kind, target: c.target, path: c.path, reason: o.rest[0], detail: o.rest[1] }) + } + } + if (status === 'running') { const next = task.checks.find((c) => c.state === 'pending'); if (next) next.state = 'running' } + const { done: _d, skip: _s, rest: _r, status: _st, ...fields } = o + Object.assign(task, fields, { status }) + if (status !== 'running') task.current = null + if (TASK_END.has(status)) { + task.finished_at = isoNow() + const t = state.tables[job.source_id].find((x) => x.name === name) + if (t && (status === 'ok' || status === 'partial')) t.newRelations = false + if (t && task.added + task.updated > 0) { + t.version += 1 + t.updated_at = isoNow() + t.updated_by = ACTOR + t.notes ??= {} + task.version = t.version + } + } +} +/** 收尾:有失败、部分完成、停止的表就是 partial */ +function finish(job, status) { + const bad = job.tasks.some((t) => RETRYABLE.has(t.status)) + Object.assign(job, { status: status ?? (bad ? 'partial' : 'succeeded'), finished_at: isoNow(), notice: null }) + job.started_at ??= job.finished_at + delete job.stopAsked +} +/** 停下(使用者停止、止损、服务重启):正在做的表写入已做完的项、其余项记原因;没开始的表记已停止,原因同一句 */ +function stopJob(job, { status = 'cancelled', reason = 'cancelled', current = null, rest = null, error = null } = {}) { + for (const t of job.tasks) { + if (t.status === 'running') { + const left = t.checks.filter((c) => c.state === 'pending' || c.state === 'running') + for (const c of left) { + Object.assign(c, { state: 'skipped', reason, detail: current ?? '已停止,这一项没有查' }) + t.skipped.push({ kind: c.kind, target: c.target, path: c.path, reason, detail: c.detail }) + } + Object.assign(t, { status: 'cancelled', current: null, finished_at: isoNow(), error: rest ?? error }) + } else if (t.status === 'pending') { + Object.assign(t, { status: 'cancelled', finished_at: isoNow(), error: error ?? '已停止,这张表没有剖析' }) + } + } + if (error) job.error = error + finish(job, status) +} + +const activeProfile = (state, src) => state.pjobs.find((j) => j.source_id === src && JOB_ACTIVE.has(j.status)) ?? null +const pjobOf = (state, key) => { + const [, , src, , , id] = key.split(' ')[1].split('/') + return state.pjobs.find((j) => j.id === id && j.source_id === src) ?? null +} +/** 一张表最近一次剖析(清单行上的 last_profile、单表目录的 last_profile) */ +function lastTaskOf(state, src, name) { + for (const job of [...state.pjobs].reverse()) { + if (job.source_id !== src) continue + const t = job.tasks.find((x) => x.table_name === name) + if (t) return { job, t } + } + return null +} +function dueOf(state, src, t) { + if (t.missing || t.is_view || !checksOf(t)) return null + const last = lastTaskOf(state, src, t.name) + if (!last) return 'never' + if (last.t.status === 'pending' || last.t.status === 'running') return null + if (RETRYABLE.has(last.t.status)) return 'unfinished' + return t.newRelations ? 'new_relations' : null +} +/** 范围 needed / all:服务端的口径(有待核实关系的在前,这里按使用次数,同清单顺序) */ +const scopeTables = (state, src, scope) => state.tables[src].filter((t) => !t.missing).sort((a, b) => b.usage - a.usage) + .filter((t) => (scope === 'all' ? checksOf(t) > 0 && !t.is_view : dueOf(state, src, t) != null)).map((t) => t.name) + +// --------------------------------------------------------------------------- +// 假服务端的起草任务:只做排队要用的部分(建、取、停),起草本身不推 +// --------------------------------------------------------------------------- +function newDraftJob(state, src, tables) { + const id = `check-djob-${++state.djobSeq}` + const job = { + id, source_id: src, status: 'queued', use_model: false, model: null, guidance: null, note: null, created_at: isoNow(), started_at: null, + finished_at: null, created_by: ACTOR, usage: { calls: 0, input_tokens: 0, output_tokens: 0 }, error: null, notice: null, + waiting_for: activeProfile(state, src) ? 'profile' : null, + tasks: tables.map((name) => ({ table_name: name, status: 'pending', columns: 3, parts_total: null, parts_done: 0, added: 0, updated: 0, removed: 0, + version: 0, model_items: null, error: null, model_error: null, started_at: null, finished_at: null })), + } + state.djobs.push(job) + return job +} +function draftOut(job) { + const counts = { pending: 0, running: 0, ok: 0, partial: 0, failed: 0, skipped: 0, cancelled: 0 } + for (const t of job.tasks) counts[t.status]++ + return clone({ ...job, total: job.tasks.length, done: job.tasks.filter((t) => TASK_END.has(t.status)).length, counts, added: 0, updated: 0, removed: 0 }) +} +const activeDraft = (state, src) => state.djobs.find((j) => j.source_id === src && JOB_ACTIVE.has(j.status)) ?? null + +// --------------------------------------------------------------------------- +// 假服务端:清单、单表、剖析任务、起草任务、保存数据源 +// --------------------------------------------------------------------------- +const freshState = () => { + const state = { sources: sources(), tables: { [S1]: shopTables(), [S2]: [ + table('orders', 3, [pk(), col('status')], { codes: ['status'] }), + table('order_summary', 1, [col('day', 'TEXT')], {}, { is_view: true }), + ], [S3]: [table('日客流', 2, [col('统计日期', 'DATE'), col('入园人数')], { date: ['统计日期'] })] }, + pjobs: [], pjobSeq: 0, djobs: [], djobSeq: 0 } + // 三天前别人做的一次剖析(部分完成):客户、商品做完了,访问记录没做完,退款写入一直冲突 + const old = newProfileJob(state, S1, { tables: ['customers', 'visits', 'products', 'refunds'], scope: 'needed', created_by: OTHER, at: T0 }) + settle(state, old, 'customers', { + queries: 4, elapsed_ms: 2300, row_estimate: { rows: 12000, method: 'stats', at_least: null }, + findings: [{ kind: 'codes', path: 'columns.level.codes', column: 'level', values: [{ value: '1', rows: 9000 }, { value: '2', rows: 3000 }], rows: 12000, + status: 'proposed', pending: 2, summary: '2 个取值' }], + date_ranges: [{ column: 'joined_at', min: '2021-03-01 08:00:00', max: '2026-10-01 21:30:00' }], + skip: [], query_log: [ + { at: T0, kind: 'row_estimate', target: 'customers', sql: 'SELECT COUNT(*) FROM (SELECT 1 FROM customers LIMIT ?) t', ms: 12, rows: 1, outcome: 'ok', detail: null }, + { at: T0, kind: 'codes', target: 'level', sql: 'SELECT level, COUNT(*) FROM customers GROUP BY level ORDER BY COUNT(*) DESC LIMIT ?', ms: 40, rows: 2, outcome: 'ok', detail: null }, + { at: T0, kind: 'date', target: 'joined_at', sql: 'SELECT MIN(joined_at), MAX(joined_at) FROM customers', ms: 25, rows: 1, outcome: 'ok', detail: null }, + ], + }) + // 码值候选之外的一项按规则不查(遮罩),不算没做完 + old.tasks[0].skipped.push({ kind: 'codes', target: 'phone', path: 'columns.phone.codes', reason: 'masked', detail: 'customers.phone 在数据源设置中被遮罩,不取值' }) + settle(state, old, 'visits', { status: 'partial', done: { relation: 0, codes: 1, date: 1 }, queries: 40, elapsed_ms: 31000, + skip: [['relation', 0, 'budget', '这张表的查询次数用完(40 条),这一项没有查,保留原来的结论'], ['codes', 1, 'budget', '这张表的查询次数用完(40 条),这一项没有查,保留原来的结论']] }) + settle(state, old, 'products', { queries: 6, elapsed_ms: 3100 }) + settle(state, old, 'refunds', { status: 'failed', queries: 3, elapsed_ms: 900, error: '写入一直冲突:剖析期间这张表的数据目录被反复修改,结论没有写进去' }) + for (const t of old.tasks) { t.started_at = T0; t.finished_at = T0 } + Object.assign(old, { started_at: T0 }) + finish(old) + old.finished_at = T0 + // 那次剖析之后,目录里又出现了商品的待核实关系 + state.tables[S1].find((t) => t.name === 'products').newRelations = true + return state +} + +const parts = (key) => key.split(' ')[1].split('/') // ['', 'datasources', id, 'catalog', ...] +function rowOf(state, src, t) { + const last = lastTaskOf(state, src, t.name) + const lp = last ? (({ seq, status, started_at, finished_at, summary, skipped_count, unfinished, error, row_estimate }) => + ({ job_id: last.job.id, seq, status, started_at, finished_at, summary, skipped_count, unfinished, error, row_estimate }))(taskRow(last.t)) : null + const counts = { proposed: 0, verified: 0, confirmed: 0, rejected: 0 } + return { + table_name: t.name, qualified: t.name, is_view: t.is_view, in_schema: !t.missing, label: t.notes?.label?.value ?? null, + label_status: t.notes?.label?.status ?? null, kind: null, counts: t.notes ? { ...counts, proposed: 1 } : counts, relations: 0, usage: t.usage, + version: t.version, updated_at: t.updated_at, updated_by: t.updated_by, last_draft: null, + last_profile: lp, profile_due: dueOf(state, src, t) != null, profile_due_reason: dueOf(state, src, t), profile_checks: checksOf(t), + } +} +function listOf(state, src) { + const all = state.tables[src] + const rows = all.filter((t) => !t.missing).sort((a, b) => b.usage - a.usage).map((t) => rowOf(state, src, t)) + rows.push(...all.filter((t) => t.missing).map((t) => rowOf(state, src, t))) + const pa = activeProfile(state, src) + const da = activeDraft(state, src) + return { + tables: rows, system_notes: src === S3, schema_note: null, schema_truncated: false, schema_total: rows.length, guidance: null, + active_job: da ? (({ id, status, total, done, waiting_for }) => ({ id, status, total, done, waiting_for }))(draftOut(da)) : null, + active_profile_job: pa ? (({ id, status, total, done, waiting_for }) => ({ id, status, total, done, waiting_for }))(jobOut(pa)) : null, + } +} +function detailOf(state, src, t) { + const last = lastTaskOf(state, src, t.name) + return { + table_name: t.name, in_schema: !t.missing, notes: t.notes ?? {}, version: t.version, updated_at: t.updated_at, updated_by: t.updated_by, + structure: t.missing ? null : { qualified: t.name, is_view: t.is_view, comment: null, columns: t.columns, + primary_key: t.columns.filter((c) => c.pk).map((c) => c.name), foreign_keys: [], unique: [] }, + system_notes: src === S3, usage: t.usage, last_profile: last ? taskDetail(last.job, last.t) : null, + } +} +const busyOf = (state, job) => { + const out = jobOut(job) + return { detail: `数据源「${state.sources.find((s) => s.id === job.source_id)?.name}」正在进行数据剖析(已完成 ${out.done} / ${out.total} 张表),` + + '请等这次剖析结束(或先停止它)再开始新的剖析', code: 'profile_busy' } +} +const NO_PJOB = json({ detail: '剖析任务不存在,可能属于别的数据源,或者已随数据源删除' }, 404) + +function handlers(state, replies = {}) { + const reply = (k, fallback) => (route, ctx) => { + const q = replies[k] + const next = Array.isArray(q) ? q.shift() : q + return (next ?? fallback)(route, ctx) + } + const P = 'check-prof-[a-z]+' + return [ + [/^GET \/datasources$/, (route) => json(state.sources)(route)], + [new RegExp(`^GET /datasources/${P}/schema$`), (route, { key }) => + json({ tables: state.tables[parts(key)[2]].filter((t) => !t.missing).map((t) => t.name), summary: '', synced_at: T0 })(route)], + [new RegExp(`^GET /datasources/${P}/catalog$`), (route, { key }) => json(listOf(state, parts(key)[2]))(route)], + // ---- 剖析任务 + [new RegExp(`^POST /datasources/${P}/catalog/profile-jobs$`), reply('create', (route, { key, body }) => { + const src = parts(key)[2] + const s = state.sources.find((x) => x.id === src) + if (!settingsOf(state, src).enabled) { + return json({ detail: `数据源「${s.name}」未开启数据剖析。剖析会对业务库发查询,请先在数据源设置中开启数据剖析`, code: 'profile_disabled' }, 409)(route) + } + const active = activeProfile(state, src) + if (active) return json(busyOf(state, active), 409)(route) + if ((body?.tables?.length ?? 0) > 500) return json({ detail: '一次最多剖析 500 张表' }, 422)(route) + const tables = body?.tables?.length ? [...new Set(body.tables)] : scopeTables(state, src, body?.scope ?? 'needed') + if (!tables.length) { + return json({ detail: '按「需要剖析的表」挑出 0 张表:表都剖析过、上次都做完了,也没有新的待核实关联关系。可以改选「全部表」,或指定要剖析的表', + code: 'profile_nothing' }, 409)(route) + } + const job = newProfileJob(state, src, { tables, scope: body?.tables?.length ? 'selected' : body?.scope ?? 'needed' }) + return json({ job: jobOut(job) }, 202)(route) + })], + [new RegExp(`^GET /datasources/${P}/catalog/profile-jobs/active$`), (route, { key }) => { + const job = activeProfile(state, parts(key)[2]) + return json({ job: job && jobOut(job) })(route) + }], + [new RegExp(`^GET /datasources/${P}/catalog/profile-jobs/latest$`), (route, { key }) => { + const job = state.pjobs.filter((j) => j.source_id === parts(key)[2]).at(-1) ?? null + return json({ job: job && jobOut(job) })(route) + }], + [new RegExp(`^GET /datasources/${P}/catalog/profile-jobs/[^/]+/tasks/\\d+$`), (route, { key }) => { + const job = pjobOf(state, key) + const t = job?.tasks.find((x) => x.seq === Number(parts(key).at(-1))) + return t ? json({ task: taskDetail(job, t) })(route) : json({ detail: '任务里没有这张表' }, 404)(route) + }], + // 停止:排队的直接停;进行中的记下来,等检查用 stopJob 推到底(这期间页面应当写「正在停止」) + [new RegExp(`^POST /datasources/${P}/catalog/profile-jobs/[^/]+/cancel$`), reply('cancel', (route, { key }) => { + const job = pjobOf(state, key) + if (!job) return NO_PJOB(route) + if (!JOB_ACTIVE.has(job.status)) return json({ detail: '这次剖析已经结束了,不需要停止', code: 'catalog_profile_finished' }, 409)(route) + if (job.status === 'queued') stopJob(job) + else job.stopAsked = true + return json({ job: jobOut(job) })(route) + })], + [new RegExp(`^POST /datasources/${P}/catalog/profile-jobs/[^/]+/retry$`), reply('retry', (route, { key }) => { + const old = pjobOf(state, key) + if (!old) return NO_PJOB(route) + const active = activeProfile(state, old.source_id) + if (active) return json(busyOf(state, active), 409)(route) + const tables = old.tasks.filter((t) => RETRYABLE.has(t.status)).map((t) => t.table_name) + if (!tables.length) return json({ detail: '这次剖析里没有失败、部分完成或停止了的表,不需要重试' }, 422)(route) + const job = newProfileJob(state, old.source_id, { tables, scope: 'selected', retry_of: old.id }) + return json({ job: jobOut(job) }, 202)(route) + })], + [new RegExp(`^GET /datasources/${P}/catalog/profile-jobs/[^/]+$`), (route, { key }) => { + const job = pjobOf(state, key) + return job ? json({ job: jobOut(job) })(route) : NO_PJOB(route) + }], + // 旧的同步剖析接口:前端不该再用它,调了就答 410,下面的检查会数它 + [new RegExp(`^POST /datasources/${P}/catalog/profile$`), json({ detail: '检查脚本:前端不该再调用同步剖析接口' }, 410)], + // ---- 起草任务(排队那一节用) + [new RegExp(`^POST /datasources/${P}/catalog/draft-jobs$`), (route, { key, body }) => { + const src = parts(key)[2] + if (activeDraft(state, src)) return json({ detail: '正在起草', code: 'catalog_draft_busy' }, 409)(route) + const job = newDraftJob(state, src, body?.tables?.length ? body.tables : state.tables[src].filter((t) => !t.missing).map((t) => t.name).slice(0, 20)) + return json({ job: draftOut(job) }, 202)(route) + }], + [new RegExp(`^GET /datasources/${P}/catalog/draft-jobs/active$`), (route, { key }) => { + const job = activeDraft(state, parts(key)[2]) + return json({ job: job && draftOut(job) })(route) + }], + [new RegExp(`^GET /datasources/${P}/catalog/draft-jobs/latest$`), (route, { key }) => { + const job = state.djobs.filter((j) => j.source_id === parts(key)[2]).at(-1) ?? null + return json({ job: job && draftOut(job) })(route) + }], + [new RegExp(`^GET /datasources/${P}/catalog/draft-jobs/[^/]+$`), (route, { key }) => { + const job = state.djobs.find((j) => j.id === parts(key).at(-1)) + return job ? json({ job: draftOut(job) })(route) : json({ detail: '起草任务不存在' }, 404)(route) + }], + [/^GET \/settings$/, async (route) => { + const real = await route.fetch().catch(() => null) + if (!real) return route.abort().catch(() => {}) + const body = await real.json().catch(() => ({})) + return route.fulfill({ response: real, json: { ...body, catalog_draft: { provider: null, model: null } } }).catch(() => {}) + }], + [/^GET \/copilot\/model$/, json({ configured: true, provider: null, model: null, effective_provider: 'check-gateway', effective_model: 'check-model', error: null })], + // ---- 单表、保存数据源 + [new RegExp(`^GET /datasources/${P}/catalog/[^/]+/impact$`), (route, { key }) => json({ table: parts(key)[4], templates: [] })(route)], + [new RegExp(`^PATCH /datasources/${P}$`), reply('patch', (route, { key, body }) => { + const id = parts(key)[2] + state.sources = state.sources.map((s) => (s.id === id ? { ...s, ...body, password: undefined } : s)) + return json(state.sources.find((s) => s.id === id))(route) + })], + [new RegExp(`^GET /datasources/${P}/catalog/[^/]+$`), (route, { key }) => { + const src = parts(key)[2] + const t = state.tables[src]?.find((x) => x.name === parts(key)[4]) + return t ? json(detailOf(state, src, t))(route) : json({ detail: `数据源中没有表 ${parts(key)[4]}` }, 404)(route) + }], + [new RegExp(`^PUT /datasources/${P}/catalog/[^/]+$`), (route, { key, body }) => { + const src = parts(key)[2] + const t = state.tables[src].find((x) => x.name === parts(key)[4]) + if (body.if_version !== t.version) return json({ detail: `表「${t.name}」的数据目录刚被修改过,请重新载入后再提交` }, 409)(route) + Object.assign(t, { notes: body.notes, version: t.version + 1, updated_at: isoNow(), updated_by: ACTOR }) + return json(detailOf(state, src, t))(route) + }], + ] +} + +// --------------------------------------------------------------------------- +// 页面上的定位 +// --------------------------------------------------------------------------- +const box = (page) => page.locator('[data-profile-dialog]') +/** 剖析对话框在哪一步:setup / blocked / running / done;没开着为 null */ +const phase = (page) => box(page).getAttribute('data-profile-dialog', { timeout: 500 }).catch(() => null) +/** 某一处的文字(空白压成一个空格);没有这一处为空串 */ +const textOf = async (loc) => (await loc.first().innerText({ timeout: 800 }).catch(() => '')).replace(/\s+/g, ' ').trim() +const inBox = (page, sel) => textOf(box(page).locator(sel)) +const taskLine = (page, name) => box(page).locator(`[data-profile-task="${name}"]`) +const head = (page) => page.locator('[data-catalog-profile]') +/** 表清单一行的「剖析」一格(去掉给读屏的「数据剖析:」);没有这一格为空串 */ +const rowProfile = async (page, name) => (await textOf(page.locator(`[data-catalog-row="${name}"] [data-row-profile]`))).replace(/^数据剖析:\s*/, '') +const toasts = (page) => page.locator('[data-toast]').allInnerTexts().then((t) => t.join(' | ')).catch(() => '') +async function openDialog(page) { + await head(page).click() + await box(page).waitFor() +} +// 界面不该露的机读码:状态、跳过原因、检查类别、查询结果的英文值 +const CODE_RE = /\b(?:queued|running|succeeded|partial|failed|cancelled|interrupted|pending|skipped|budget|deadline|timeout|rejected|masked|too_large|slow|nothing|row_estimate|relation|codes|verified|proposed|unfinished|new_relations|never)\b/ +async function cleanCopy(page, where) { + let t = await box(page).innerText().catch(() => '') + // 表名、列名、SQL 是库里的标识,照原样显示,先去掉再查 + t = t.replace(/SELECT[\s\S]*?(?=\n|$)/g, '').replace(/[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z0-9_]+)?/g, (w) => (CODE_RE.test(w) && !/_/.test(w) ? w : '')) + const code = t.match(CODE_RE)?.[0] + check(`${where}:不露机读码`, !code, code ?? '') +} + +// --------------------------------------------------------------------------- + +try { + const probe = await open('/data') + const overlay = await probe.page.locator('vite-error-overlay').count() + await probe.close() + if (overlay) { + console.error('✗ 页面上是 vite 的报错层——有文件正在改,等一会儿再跑') + process.exit(1) + } +} catch (e) { + console.error(`✗ 打不开前端(${WEB})——先起沙箱前端\n ${e.message}`) + process.exit(1) +} + +await section('剖析 · 设置:每张表的两项预算、被拒时报错落到那一项', async () => { + const state = freshState() + const replies = { patch: [] } + const { page, sent, errors, natives, close } = await open('/data/databases', { handlers: handlers(state, replies) }) + // 没开剖析的库(设置里还留着旧的两个键):编辑框里「数据剖析」默认关闭,预算收着 + await page.locator(`[data-source="${NAME2}"]`).getByRole('button', { name: '编辑', exact: true }).click() + const sec = page.locator('[data-profile-settings]') + await sec.waitFor() + check('编辑框里有「数据剖析」,关着、预算收着', (await sec.getAttribute('data-profile-settings')) === 'off' + && await sec.locator('[data-profile-field]').count() === 0) + await sec.locator('[data-profile-enabled]').check() + const labels = await sec.locator('label.label').allInnerTexts() + check('开启后五项预算展开:每张表最多查询数、每张表时长上限在前,没有整次的查询次数上限、总时长上限', + labels.join('、') === '每张表最多查询数、每张表时长上限、单条查询时限、抽样键值数、整表统计行数上限', labels.join('、')) + const units = await sec.locator('[data-profile-field] span').allInnerTexts() + check('……每项带单位', units.join(',') === '条,秒,秒,个,行', units.join(',')) + const hints = await sec.locator('[id$="-hint"]').allInnerTexts() + check('……写明范围和默认值,说清用完只停这张表', hints.some((h) => h.startsWith('范围 1–200 条,默认 40') && h.includes('用完只停这张表')) + && hints.some((h) => h.startsWith('范围 10–600 秒,默认 60') && h.includes('只停这张表')), hints.join(' | ')) + check('……旧的两个键不回显成输入框的值', await page.locator('#ds-profile-table_max_queries').inputValue() === '' + && await page.locator('#ds-profile-table_max_s').inputValue() === '') + const risk = () => sec.locator('[data-profile-risk]').innerText().catch(() => '') + check('……风险说明按表写:一条一条地发、可随时停止,每张表最多 40 条、60 秒,单条 10 秒', + (await risk()).includes('一条一条地发,可随时停止:每张表最多 40 条、60 秒,单条不超过 10 秒'), await risk()) + await page.locator('#ds-profile-table_max_queries').fill('30') + check('……风险说明跟着填的数走', (await risk()).includes('每张表最多 30 条'), await risk()) + await page.locator('#ds-profile-table_max_queries').fill('500') + const err = await page.locator('#ds-profile-table_max_queries-error').innerText().catch(() => '') + check('填错(500 条,超过 200):就地写明范围,保存按钮禁用', err === '请填写 1 到 200 之间的整数' && await page.locator('[data-source-save]').isDisabled(), err) + check('……输入框标出错', (await page.locator('#ds-profile-table_max_queries').getAttribute('aria-invalid')) === 'true') + await page.locator('#ds-profile-table_max_queries').fill('30') + await page.locator('#ds-profile-table_max_s').fill('90') + // 服务端按字段拒收(422):带机读码和出错的那一项,界面按它落到那一格,不从原话里认 + const REJECT = '数据剖析的「每张表时长上限(秒)」需要填写 10 到 600 之间的数;当前为「90」' + replies.patch.push(json({ detail: REJECT, code: 'profile_settings_invalid', field: 'table_max_s' }, 422)) + await page.locator('[data-source-save]').click() + check('保存被拒(422,field 是每张表时长上限):服务端原话写在那一项下面', await until(async () => + (await page.locator('#ds-profile-table_max_s-error').innerText().catch(() => '')) === REJECT), + await page.locator('#ds-profile-table_max_s-error').innerText().catch(() => '')) + check('……光标落在那一项上,框标出错', await until(async () => page.evaluate(() => document.activeElement?.id === 'ds-profile-table_max_s')) + && (await page.locator('#ds-profile-table_max_s').getAttribute('aria-invalid')) === 'true') + replies.patch.push(json({ detail: '数据剖析的「每张表最多查询数」需要填写 1 到 200 之间的整数;当前为「30」', code: 'profile_settings_invalid', + field: 'table_max_queries' }, 422)) + await page.locator('#ds-profile-table_max_s').fill('95') + await page.locator('[data-source-save]').click() + check('……field 是每张表最多查询数:落在那一格', await until(async () => + (await page.locator('#ds-profile-table_max_queries').getAttribute('aria-invalid')) === 'true' + && page.evaluate(() => document.activeElement?.id === 'ds-profile-table_max_queries'))) + if (SHOTS) await sleep(350) + await shot(page, 'profile-settings-422') + await page.locator('#ds-profile-table_max_queries').fill('30') + await page.locator('#ds-profile-table_max_s').fill('90') + await page.locator('[data-source-save]').click() + await until(async () => (await page.locator('[data-profile-settings]').count()) === 0) + const saved = writes(sent, /^PATCH /).at(-1)?.body + check('保存:只写开关和填了的新项(数字),旧的两个键不带', JSON.stringify(saved?.options?.catalog_profile) + === JSON.stringify({ enabled: true, table_max_queries: 30, table_max_s: 90 }), JSON.stringify(saved?.options)) + check('……卡片上标「数据剖析已开启」,悬停按表写预算', await until(async () => (await page.locator(`[data-source="${NAME2}"] [data-profile-on]`).count()) === 1) + && (await page.locator(`[data-source="${NAME2}"] [data-profile-on]`).getAttribute('title')) === '数据剖析已开启:每张表最多 30 条只读查询、90 秒,单条不超过 10 秒', + await page.locator(`[data-source="${NAME2}"] [data-profile-on]`).getAttribute('title').catch(() => '')) + await page.locator(`[data-source="${NAME2}"]`).getByRole('button', { name: '编辑', exact: true }).click() + await page.locator('[data-profile-settings="on"]').waitFor() + check('再打开编辑框:开关和填的数回显', await page.locator('#ds-profile-table_max_queries').inputValue() === '30' + && await page.locator('#ds-profile-table_max_s').inputValue() === '90') + await page.getByRole('button', { name: '取消', exact: true }).click() + + // 导入表格的源没有编辑框:卡片上单独设置,options 的其余键原样带上 + await goto(page, '/data/tables') + await page.locator(`[data-source="${NAME3}"] [data-edit-profile]`).click() + const dialog = page.locator('[data-profile-settings-dialog]') + await dialog.waitFor() + check('导入表格的源:卡片上「设置数据剖析」打开只改这一项的弹窗,写明对之后新建的剖析生效', + (await textOf(page.locator('[role="dialog"]').last())).includes('对之后新建的剖析生效')) + await dialog.locator('[data-profile-enabled]').check() + await page.locator(`#profile-${S3}-table_max_s`).fill('120') + await page.locator('[data-profile-save]').click() + await until(async () => (await dialog.count()) === 0) + const body = writes(sent, new RegExp(`^PATCH /datasources/${S3}$`)).at(-1)?.body + check('……保存只换剖析设置,遮罩列原样带上', JSON.stringify(body) + === JSON.stringify({ options: { mask_columns: ['联系电话'], catalog_profile: { enabled: true, table_max_s: 120 } } }), JSON.stringify(body)) + check('写请求只有保存数据源', writes(sent, /./).every((s) => /^PATCH \/datasources\/check-prof-/.test(s.key)), + JSON.stringify(writes(sent, /./).map((s) => s.key))) + check('没有原生对话框', natives.length === 0, natives.join(' | ')) + check('没有运行时报错', errors.length === 0, errors[0] ?? '') + await close() +}) + +await section('剖析 · 开始前:范围、个数与预算', async () => { + const state = freshState() + const { page, sent, errors, natives, close } = await open(`/data/catalog/${S1}`, { handlers: handlers(state) }) + await page.locator('[data-catalog-row]').first().waitFor() + check('页头写「数据剖析」', (await head(page).getAttribute('aria-label')) === '数据剖析' && (await head(page).getAttribute('data-catalog-profile')) === '') + await openDialog(page) + const scopes = await box(page).locator('[data-profile-scope]').evaluateAll((els) => els.map((e) => `${e.getAttribute('data-profile-scope')}:${e.getAttribute('data-count')}`)) + // 需要剖析:从没剖析过的订单、空日志表;上次没做完的访问记录、退款;剖析后有新关系的商品。全部:再加上客户,不含视图、 + // 没有可剖析项的配置表、只剩目录的旧表 + check('没选表、没筛选:范围是「需要剖析的 5 张表」(默认)和「全部 6 张表」', scopes.join(',') === 'needed:5,all:6' + && await box(page).locator('[data-profile-scope="needed"] input').isChecked(), scopes.join(',')) + check('……写明需要剖析的口径、全部不含视图', (await inBox(page, '[data-profile-scope="needed"]')).includes('从没剖析过、上次没做完,或剖析之后又出现了待核实关联关系的表') + && (await inBox(page, '[data-profile-scope="all"]')).includes('不含视图')) + const budget = await inBox(page, '[data-profile-budget]') + check('预算按表写:每张表最多 40 条、60 秒,单条 10 秒,抽样、整表统计门槛,按哪个源的设置', + ['每张表最多 40 条查询、60 秒,用完只停这张表', '单条查询不超过 10 秒', '抽样 2,000 个键值', '行数超过 100,000 的表不做整表统计', `按「${NAME1}」的数据剖析设置`] + .every((x) => budget.includes(x)), budget) + check('……按这次的表数估上限:「这次 5 张表,最多约 200 条查询、5 分钟」', (await inBox(page, '[data-profile-budget-total]')) === '这次 5 张表,最多约 200 条查询、5 分钟', + await inBox(page, '[data-profile-budget-total]')) + await box(page).locator('[data-profile-scope="all"]').click() + check('……换成全部 6 张:估算跟着变', (await inBox(page, '[data-profile-budget-total]')) === '这次 6 张表,最多约 240 条查询、6 分钟') + check('醒目一句:只读查询、一条一条地发、可随时停止', (await inBox(page, '[data-profile-readonly]')).includes('只读查询,一条一条地发,可随时停止')) + check('……不再说「无法中途停止」', !(await textOf(box(page))).includes('无法中途停止')) + check('……没有起草任务:按钮写「开始剖析」', (await textOf(page.locator('[data-profile-start]'))).includes('开始剖析') + && (await page.locator('[data-profile-start]').getAttribute('data-profile-start')) === 'start') + check('上一次剖析没做完:设置里写一行,可以点开', (await inBox(page, '[data-profile-last]')).includes('剖析完成,部分表没做完') + && await box(page).locator('[data-profile-last-open]').count() === 1, await inBox(page, '[data-profile-last]')) + check('还没开始:没有写请求', writes(sent, /./).length === 0) + if (SHOTS) await sleep(350) + await shot(page, 'profile-setup') + await cleanCopy(page, '开始前') + await page.keyboard.press('Escape') + await until(async () => (await box(page).count()) === 0) + + // 已选三张(其中一张是视图):剖析不了的不交给服务端,写明另有几张不剖析 + for (const n of ['orders', 'customers', 'order_summary']) await page.locator(`[data-catalog-select="${n}"]`).click() + await page.locator('[data-catalog-profile-selected]').click() + await box(page).waitFor() + check('剖析所选:范围默认是「已选的 2 张表」,写明另有 1 张不剖析', await box(page).locator('[data-profile-scope="selected"] input').isChecked() + && (await inBox(page, '[data-profile-scope="selected"]')).includes('已选的 2 张表') + && (await inBox(page, '[data-profile-scope="selected"]')).includes('另有 1 张不剖析'), await inBox(page, '[data-profile-scope="selected"]')) + await page.keyboard.press('Escape') + await page.locator('[data-catalog-clear]').click() + // 搜索「order」:当前筛选里的订单、退款(id 列里有 order)……按清单的筛选结果算 + await page.locator('[data-catalog-search]').fill('order') + await openDialog(page) + check('有筛选:多一项「当前筛选结果中的 N 张表」', await box(page).locator('[data-profile-scope="filtered"]').count() === 1 + && (await inBox(page, '[data-profile-scope="filtered"]')).startsWith('当前筛选结果中的 1 张表'), await inBox(page, '[data-profile-scope="filtered"]')) + check('没有调用旧的同步剖析接口', count(sent, /catalog\/profile$/) === 0) + check('没有原生对话框', natives.length === 0, natives.join(' | ')) + check('没有运行时报错', errors.length === 0, errors[0] ?? '') + await close() +}) + +await section('剖析 · 进行中:逐表进度、后台继续与停止', async () => { + const state = freshState() + const { page, sent, errors, close } = await open(`/data/catalog/${S1}`, { handlers: handlers(state) }) + await page.locator('[data-catalog-row]').first().waitFor() + await openDialog(page) + await page.locator('[data-profile-start]').click() + check('开始:进入进行中,建任务的请求带「需要剖析」范围和本机署名', await until(async () => (await phase(page)) === 'running') + && JSON.stringify(lastBody(sent, /profile-jobs$/)) === JSON.stringify({ scope: 'needed' }) + && decodeURIComponent(writes(sent, /profile-jobs$/).at(-1)?.actor ?? '') === ACTOR, JSON.stringify(lastBody(sent, /profile-jobs$/))) + const job = state.pjobs.at(-1) + check('……刚建好还在排队:标题写「排队中」,逐表 5 行都写「排队」', (await inBox(page, '[data-profile-title]')).startsWith('排队中') + && (await box(page).locator('[data-profile-task]').count()) === 5 && (await inBox(page, '[data-profile-task="orders"] [data-profile-task-status]')) === '排队') + // 订单表做到一半:关系 3/4、码值 5/6、日期 1/1,发了 23 条查询,正在核对第 4 条关系 + settle(state, job, 'orders', { status: 'running', done: { relation: 3, codes: 5, date: 1 }, queries: 23, elapsed_ms: 18000, + current: '正在核对 orders.coupon_id → coupons.id' }) + check('进行中:逐表一行写各类检查的进度「关系 3/4 · 码值 5/6 · 日期 1/1」', await until(async () => + (await inBox(page, '[data-profile-task="orders"] [data-profile-task-checks]')) === '关系 3/4 · 码值 5/6 · 日期 1/1'), + await inBox(page, '[data-profile-task="orders"]')) + check('……查询数、用时、正在做哪一项', (await inBox(page, '[data-profile-task="orders"] [data-profile-task-queries]')) === '23 条查询' + && (await inBox(page, '[data-profile-task="orders"] [data-profile-task-elapsed]')) === '18 秒' + && (await inBox(page, '[data-profile-task="orders"] [data-profile-task-note]')) === '正在核对 orders.coupon_id → coupons.id' + && (await inBox(page, '[data-profile-task="orders"] [data-profile-task-status]')) === '剖析中') + check('……头部:几张 / 共几张、已发几条查询、用时', (await inBox(page, '[data-profile-progress]')) === '0 / 5 张表' + && (await inBox(page, '[data-profile-queries]')).includes('已发 23 条查询') && (await inBox(page, '[data-profile-elapsed]')).startsWith('· 用时'), + await textOf(box(page).locator('[role="status"]'))) + check('……标题写「正在剖析…」', (await inBox(page, '[data-profile-title]')) === '正在剖析…') + settle(state, job, 'orders', { + queries: 31, elapsed_ms: 26000, added: 3, updated: 2, row_estimate: { rows: 250000, method: 'stats', at_least: null }, + findings: [ + { kind: 'relation', path: 'relations.customer_id → customers.id', target: 'customer_id → customers.id', columns: ['customer_id'], to_table: 'customers', + to_columns: ['id'], status: 'verified', confirmed: false, previous_status: 'proposed', coverage: 0.991, cardinality: 'many_to_one', cardinality_checked: true, + sample: 2000, matched: 1982, summary: '' }, + { kind: 'codes', path: 'columns.status.codes', column: 'status', values: [{ value: '1', rows: 200000 }, { value: '2', rows: 40000 }, { value: '9', rows: 10000 }], + rows: 250000, status: 'proposed', pending: 2, summary: '' }, + { kind: 'codes', path: 'columns.channel.codes', column: 'channel', values: [{ value: 'web', rows: 150000 }, { value: 'app', rows: 100000 }], + rows: 250000, status: 'proposed', pending: 2, summary: '' }, + ], + query_log: [ + { at: isoNow(), kind: 'row_estimate', target: 'orders', sql: 'SELECT reltuples FROM pg_class WHERE relname = ?', ms: 8, rows: 1, outcome: 'ok', detail: null }, + { at: isoNow(), kind: 'relation', target: 'customer_id → customers.id', sql: 'SELECT DISTINCT customer_id FROM orders WHERE customer_id IS NOT NULL LIMIT ?', + ms: 120, rows: 2000, outcome: 'ok', detail: null }, + { at: isoNow(), kind: 'relation', target: 'customer_id → customers.id', sql: 'SELECT id FROM customers WHERE id IN (?, ?, ?)', ms: 95, rows: 1982, outcome: 'ok', detail: null }, + { at: isoNow(), kind: 'relation', target: 'coupon_id → coupons.id', sql: 'SELECT DISTINCT coupon_id FROM orders WHERE coupon_id IS NOT NULL LIMIT ?', + ms: 10000, rows: null, outcome: 'timeout', detail: '超过单条查询时限(10 秒),数据库已中断这条查询' }, + ], + }) + const lists0 = count(sent, new RegExp(`^GET /datasources/${S1}/catalog$`)) + check('一张表剖析完:「1 / 5 张表」,进度条读屏可读', await until(async () => (await inBox(page, '[data-profile-progress]')) === '1 / 5 张表') + && (await box(page).locator('[role="progressbar"]').getAttribute('aria-valuenow')) === '1' + && (await box(page).locator('[role="progressbar"]').getAttribute('aria-valuemax')) === '5') + check('……这张表写结论:核实了几条关系、取到几列码值候选(几个含义待填写)', (await inBox(page, '[data-profile-task="orders"] [data-profile-task-note]')) + === '核实 1 条关系,取到 2 列码值候选(4 个含义待填写)', await inBox(page, '[data-profile-task="orders"] [data-profile-task-note]')) + check('……写行数估算', (await inBox(page, '[data-profile-task="orders"]')).includes('约 250,000 行')) + check('……头部写出超时的查询', (await inBox(page, '[data-profile-queries-bad]')).includes('超时 1 条'), await inBox(page, '[data-profile-queries-bad]')) + check('写完的表马上并入目录:清单重取', await until(async () => count(sent, new RegExp(`^GET /datasources/${S1}/catalog$`)) > lists0)) + if (SHOTS) await sleep(350) + await shot(page, 'profile-running') + await cleanCopy(page, '进行中') + + // 关掉对话框:任务不停,页头换成进度入口;表清单的「剖析」一格写排队、剖析中 + settle(state, job, 'visits', { status: 'running', done: { relation: 0, codes: 1, date: 0 }, queries: 3, elapsed_ms: 1500, current: '正在取 visits.page 的码值候选' }) + await until(async () => (await inBox(page, '[data-profile-task="visits"] [data-profile-task-status]')) === '剖析中') + await page.locator('[data-profile-hide]').click() + check('「在后台继续」:对话框关上,没有发停止', await until(async () => (await box(page).count()) === 0) && count(sent, /\/cancel$/) === 0) + check('……页头换成「剖析 1/5」,读屏念得出', await until(async () => (await head(page).getAttribute('data-catalog-profile')) === 'running') + && (await head(page).getAttribute('aria-label')) === '剖析 1/5' && (await textOf(head(page))) === '剖析 1/5', await textOf(head(page))) + check('……表清单:剖析完的写「刚刚剖析」,正在剖析的写「剖析中」,后面的写「排队剖析」', await until(async () => + (await rowProfile(page, 'orders')) === '刚刚剖析' && (await rowProfile(page, 'visits')) === '剖析中' && (await rowProfile(page, 'products')) === '排队剖析'), + [await rowProfile(page, 'orders'), await rowProfile(page, 'visits'), await rowProfile(page, 'products')].join(' / ')) + // 最近查询多数超时:服务端给提醒,关着对话框时页头的入口带提醒色,悬停写全文 + const NOTICE = '最近 6 条查询有 4 条超时,数据库响应变慢了;剖析会继续,可以停止,换在业务不忙的时候再剖析' + job.notice = NOTICE + check('多数超时的提醒:页头入口带提醒标记,悬停写全文', await until(async () => (await page.locator('[data-catalog-profile-notice]').count()) === 1) + && (await head(page).getAttribute('title')).includes(NOTICE)) + await shot(page, 'profile-header') + await head(page).click() + check('点页头:回到对话框,直接是进行中的任务,提醒写在进度条下面', await until(async () => (await phase(page)) === 'running') + && (await inBox(page, '[data-profile-notice]')) === NOTICE && await box(page).locator('[data-profile-scope]').count() === 0) + + // 停止:调停止接口;正在执行的那条查询由数据库按单条时限截止,这期间写「正在停止」 + await page.locator('[data-profile-stop]').click() + check('点「停止」:调停止接口,写「正在停止:等正在执行的那条查询结束…」,停止按钮不能再点', await until(async () => + (await inBox(page, '[data-profile-title]')).startsWith('正在停止:等正在执行的那条查询结束')) + && count(sent, new RegExp(`^POST .*profile-jobs/${job.id}/cancel$`)) === 1 && await page.locator('[data-profile-stop]').isDisabled(), + await inBox(page, '[data-profile-title]')) + stopJob(job, { rest: '已停止:做完的 1 项已写进目录,其余 3 项保留原来的结论' }) + check('……停下来:标题写「已停止」,写明做完的已写进目录、没做的保留原来的结论', await until(async () => (await phase(page)) === 'done') + && (await inBox(page, '[data-profile-title]')) === '已停止' && (await inBox(page, '[data-profile-cancelled]')).includes('没做的项保留原来的结论'), + `${await inBox(page, '[data-profile-title]')} · ${await inBox(page, '[data-profile-cancelled]')}`) + check('……正在剖析的那张写已停止和服务端的原因;后面的表都记已停止', (await inBox(page, '[data-profile-task="visits"] [data-profile-task-status]')) === '已停止' + && (await inBox(page, '[data-profile-task="visits"] [data-profile-task-note]')).includes('其余 3 项保留原来的结论') + && (await inBox(page, '[data-profile-task="tmp_log"] [data-profile-task-status]')) === '已停止' + && await taskLine(page, 'tmp_log').locator('[data-profile-task-checks]').count() === 0) + check('……汇总照实写:5 张表,完成 1,已停止 4', (await inBox(page, '[data-profile-counts]')) === '5 张表:完成 1,已停止 4', await inBox(page, '[data-profile-counts]')) + check('……可以重试没做完的 4 张', (await textOf(page.locator('[data-profile-retry]'))).includes('重试没做完的表(4 张)')) + check('……页头回到「数据剖析」', await until(async () => (await head(page).getAttribute('data-catalog-profile')) === '')) + const polls = () => count(sent, new RegExp(`^GET .*profile-jobs/${job.id}$`)) + const n = polls() + await sleep(3200) + check('结束以后不再取进度', polls() === n, `${n} → ${polls()}`) + check('写请求只有建任务和停止', writes(sent, /./).every((s) => /profile-jobs(\/[^/]+\/cancel)?$/.test(s.key)), JSON.stringify(writes(sent, /./).map((s) => s.key))) + check('没有运行时报错', errors.length === 0, errors[0] ?? '') + await close() +}) + +await section('剖析 · 结束后:汇总、筛选、展开与重试', async () => { + const state = freshState() + const replies = { retry: [] } + const { page, sent, errors, natives, close } = await open(`/data/catalog/${S1}`, { handlers: handlers(state, replies) }) + await page.locator('[data-catalog-row]').first().waitFor() + await openDialog(page) + await box(page).locator('[data-profile-scope="all"]').click() + await page.locator('[data-profile-start]').click() + await until(async () => (await phase(page)) === 'running') + check('全部表:请求带「全部」范围', JSON.stringify(lastBody(sent, /profile-jobs$/)) === JSON.stringify({ scope: 'all' })) + const job = state.pjobs.at(-1) + settle(state, job, 'orders', { queries: 31, elapsed_ms: 26000, added: 2, updated: 1, findings: [ + { kind: 'relation', path: 'relations.customer_id → customers.id', target: 'customer_id → customers.id', columns: ['customer_id'], to_table: 'customers', + to_columns: ['id'], status: 'verified', confirmed: false, previous_status: 'proposed', coverage: 0.991, cardinality: 'many_to_one', cardinality_checked: true, + sample: 2000, matched: 1982, summary: '' }, + { kind: 'relation', path: 'relations.coupon_id → coupons.id', target: 'coupon_id → coupons.id', columns: ['coupon_id'], to_table: 'coupons', to_columns: ['id'], + status: 'proposed', confirmed: false, previous_status: 'proposed', coverage: 0.4, cardinality: 'many_to_one', cardinality_checked: false, sample: 500, matched: 200, summary: '' }, + { kind: 'codes', path: 'columns.status.codes', column: 'status', values: [{ value: '1', rows: 9 }, { value: '2', rows: 3 }, { value: '9', rows: 1 }], rows: 13, + status: 'proposed', pending: 2, summary: '' }, + { kind: 'codes', path: 'columns.channel.codes', column: 'channel', values: [{ value: 'web', rows: 8 }, { value: 'app', rows: 5 }], rows: 13, + status: 'proposed', pending: 2, summary: '' }, + { kind: 'business_date', path: 'business_date', column: 'created_at', min: '2024-01-01 00:00:00', max: '2026-10-14 23:59:00', status: 'proposed', summary: '' }, + ] }) + settle(state, job, 'customers', { queries: 4, elapsed_ms: 2000 }) + // 访问记录:这张表的 40 条查询用完了,2 项没查成(旧结论保留) + settle(state, job, 'visits', { status: 'partial', done: { relation: 0, codes: 1, date: 1 }, queries: 40, elapsed_ms: 33000, + skip: [['relation', 0, 'budget', '这张表的查询次数用完(40 条),这一项没有查,保留原来的结论'], ['codes', 1, 'timeout', '超过单条查询时限(10 秒),这一项没有查成']], + query_log: [ + { at: isoNow(), kind: 'row_estimate', target: 'visits', sql: 'SELECT COUNT(*) FROM (SELECT 1 FROM visits LIMIT ?) t', ms: 30, rows: 1, outcome: 'ok', detail: null }, + { at: isoNow(), kind: 'codes', target: 'page', sql: 'SELECT page, COUNT(*) FROM visits GROUP BY page ORDER BY COUNT(*) DESC LIMIT ?', ms: 10000, rows: null, + outcome: 'timeout', detail: '超过单条查询时限(10 秒),数据库已中断这条查询' }, + { at: isoNow(), kind: 'date', target: 'visited_at', sql: "SELECT MIN(visited_at), MAX(visited_at) FROM visits WHERE visited_at > ?", ms: 60, rows: 1, outcome: 'ok', detail: null }, + ] }) + settle(state, job, 'products', { queries: 6, elapsed_ms: 3000 }) + settle(state, job, 'refunds', { status: 'failed', queries: 3, elapsed_ms: 800, error: '写入一直冲突:剖析期间这张表的数据目录被反复修改,结论没有写进去' }) + settle(state, job, 'tmp_log', { status: 'skipped', queries: 1, elapsed_ms: 100, error: '空表,没有可剖析的数据' }) + finish(job) + check('结束:标题写「剖析完成,部分表没做完」', await until(async () => (await phase(page)) === 'done') + && (await inBox(page, '[data-profile-title]')) === '剖析完成,部分表没做完', await inBox(page, '[data-profile-title]')) + check('……进度写做完了几张「完成 3 / 6 张表」(跳过、失败、部分完成的不算),进度条同', (await inBox(page, '[data-profile-progress]')) === '完成 3 / 6 张表' + && (await box(page).locator('[role="progressbar"]').getAttribute('aria-valuenow')) === '3', await inBox(page, '[data-profile-progress]')) + check('……汇总:6 张表:完成 3,部分完成 1,跳过 1,失败 1', (await inBox(page, '[data-profile-counts]')) === '6 张表:完成 3,部分完成 1,跳过 1,失败 1', + await inBox(page, '[data-profile-counts]')) + check('……发现的合计:核实了几条、几条仍为推断、几列码值候选(几个含义待填写)、几张表的业务日期', (await inBox(page, '[data-profile-findings-summary]')) + === '核实 1 条关系,1 条关系仍为推断,取到 2 列码值候选(4 个含义待填写),提议 1 张表的业务日期', await inBox(page, '[data-profile-findings-summary]')) + check('……目录的变化', (await inBox(page, '[data-profile-changes]')) === '目录新增 2 项、更新 1 项') + check('……查询次数用完的表写「40 条查询(已用完)」,没查成的项数', (await inBox(page, '[data-profile-task="visits"] [data-profile-task-queries]')) === '40 条查询(已用完)' + && (await inBox(page, '[data-profile-task="visits"] [data-profile-task-note]')).includes('2 项没查成'), await inBox(page, '[data-profile-task="visits"]')) + check('……空表、写入冲突照服务端的原话', (await inBox(page, '[data-profile-task="tmp_log"] [data-profile-task-note]')) === '空表,没有可剖析的数据' + && (await inBox(page, '[data-profile-task="refunds"] [data-profile-task-note]')).startsWith('写入一直冲突')) + // 筛选:有发现 / 有跳过 / 失败 + const chips = await box(page).locator('[data-profile-filter-key]').evaluateAll((els) => els.map((e) => e.innerText.replace(/\s+/g, ''))) + check('筛选带数量:全部 6、有发现 1、有跳过 2、失败 1', chips.join('|') === '全部6|有发现1|有跳过2|失败1', chips.join('|')) + const shown = () => box(page).locator('[data-profile-task]').evaluateAll((els) => els.map((e) => e.getAttribute('data-profile-task')).join(',')) + await box(page).locator('[data-profile-filter-key="failed"]').click() + check('「失败」:只有写入冲突的退款', (await shown()) === 'refunds', await shown()) + await box(page).locator('[data-profile-filter-key="skipped"]').click() + check('「有跳过」:访问记录(有项没查成)、空日志表', (await shown()) === 'visits,tmp_log', await shown()) + await box(page).locator('[data-profile-filter-key="findings"]').click() + check('「有发现」:订单', (await shown()) === 'orders', await shown()) + await box(page).locator('[data-profile-filter-key="all"]').click() + + // 展开一张表:检查清单(每项状态、原因、几条查询)、发过的查询(字面量换成了 ?) + await taskLine(page, 'visits').locator('[data-profile-task-toggle]').click() + const visits = box(page).locator('[data-profile-task-detail="visits"]') + check('点一行展开:按 seq 取这张表的完整结果', await until(async () => (await visits.locator('[data-profile-checks]').count()) === 1) + && count(sent, new RegExp(`^GET .*profile-jobs/${job.id}/tasks/3$`)) >= 1 + && (await taskLine(page, 'visits').locator('[data-profile-task-toggle]').getAttribute('aria-expanded')) === 'true') + const checksList = await visits.locator('[data-profile-check]').evaluateAll((els) => els.map((e) => `${e.getAttribute('data-profile-check')}:${e.getAttribute('data-state')}`)) + check('……检查清单 4 项:关系没查成、码值一项完成一项没查成、日期完成', checksList.join(',') === 'relation:skipped,codes:done,codes:skipped,date:done', checksList.join(',')) + check('……没查成的写原因(中文)和服务端的整句', (await textOf(visits.locator('[data-profile-check="relation"]'))).includes('本表查询次数用完') + && (await textOf(visits.locator('[data-profile-check="relation"]'))).includes('保留原来的结论') + && (await textOf(visits.locator('[data-profile-check="codes"][data-state="skipped"]'))).includes('查询超时')) + check('……检查清单的标题写几项结束了', (await textOf(visits.locator('[data-profile-checks] > div').first())) === '检查清单(4 / 4 项)') + const log = visits.locator('[data-profile-queries-log]') + check('……发过的查询折着:「看发过的查询(3 条)」', (await textOf(log.locator('summary'))) === '看发过的查询(3 条)') + await log.locator('summary').click() + const sqls = await log.locator('[data-profile-query-sql]').allInnerTexts() + check('……展开:每条的 SQL 照服务端给的写(字面量都是 ?),写明为什么', sqls.length === 3 && sqls.every((q) => q.includes('?')) + && (await textOf(log)).includes('SQL 里的取值都换成了「?」'), sqls.join(' | ')) + check('……结果写中文:成功、超时(带原因),写用时和行数', (await textOf(log.locator('[data-profile-query][data-outcome="timeout"]'))).includes('超时') + && (await textOf(log.locator('[data-profile-query][data-outcome="timeout"]'))).includes('数据库已中断这条查询') + && (await textOf(log.locator('[data-profile-query]').first())).includes('30 毫秒')) + if (SHOTS) await sleep(350) + await shot(page, 'profile-expanded') + // 有发现的表:关系的结论、码值的「填写含义」 + await taskLine(page, 'orders').locator('[data-profile-task-toggle]').click() + const orders = box(page).locator('[data-profile-task-detail="orders"]') + await until(async () => (await orders.locator('[data-profile-finding]').count()) > 0) + check('展开有发现的表:关系的结论「升为已验证」、覆盖率不足的「保持推断」', (await textOf(orders.locator('[data-profile-outcome="raised"]'))) === '升为已验证' + && (await textOf(orders.locator('[data-profile-outcome="kept"]'))).startsWith('保持推断:覆盖率不足')) + check('……码值候选给「填写含义」,提议的业务日期', await orders.locator('[data-profile-fill="status"]').count() === 1 + && (await textOf(orders.locator('[data-profile-finding="business_date"]'))).includes('提议按 created_at 作为业务日期')) + await cleanCopy(page, '结束后的结果') + // 填写含义:对话框收起,打开那张表、弹出那一列的码值;取消后回到结果,那张表照旧展开,焦点落在下一列的「填写含义」上 + await orders.locator('[data-profile-fill="status"]').click() + const codes = page.locator('[data-codes-dialog="status"]') + check('点「填写含义」:对话框收起,弹出这一列的码值', await until(async () => (await codes.count()) === 1) && (await box(page).count()) === 0) + check('……每个码值一个输入框,空着的写「含义待填写」,光标在第一个空着的上面', await codes.locator('input[data-code]').count() === 3 + && (await codes.locator('input[data-code="2"]').getAttribute('placeholder')) === '含义待填写' + && await until(async () => page.evaluate(() => document.activeElement?.getAttribute('data-code') === '2'))) + check('……没填时保存禁用', await page.locator('[data-codes-save]').isDisabled()) + await page.keyboard.press('Escape') + check('……取消:回到结果,那张表照旧展开,焦点在下一列(channel)的「填写含义」上', await until(async () => (await phase(page)) === 'done') + && await until(async () => (await box(page).locator('[data-profile-task-detail="orders"] [data-profile-fill]').count()) === 2) + && await until(async () => page.evaluate(() => document.activeElement?.getAttribute('data-profile-fill') === 'channel'), 2000), + await page.evaluate(() => document.activeElement?.outerHTML?.slice(0, 120) ?? '')) + + // 接着填下一列:填完回车保存(整份提交,只换这一列码值的含义,带读到的版本),回到结果,这一列写「含义已填写」 + await page.keyboard.press('Enter') + const channel = page.locator('[data-codes-dialog="channel"]') + check('……回车接着填下一列', await until(async () => (await channel.count()) === 1)) + const v0 = state.tables[S1].find((t) => t.name === 'orders').version + await channel.locator('input[data-code="web"]').fill('网站') + await channel.locator('input[data-code="app"]').fill('应用') + await page.keyboard.press('Enter') + const put = () => lastBody(sent, /^PUT .*catalog\/orders$/) + check('回车保存:整份提交,只换这一列码值的含义,带读到的版本', await until(async () => (await channel.count()) === 0 && !!put()) + && JSON.stringify(put()?.notes?.columns?.channel?.codes?.value) === JSON.stringify({ web: '网站', app: '应用' }) && put()?.if_version === v0, + JSON.stringify(put()?.notes?.columns?.channel)) + const channelLine = box(page).locator('[data-profile-task-detail="orders"] [data-profile-finding="codes"][data-column="channel"]') + check('……回到结果:这一列写「含义已填写」,按钮改成「修改含义」,前一列照旧待填写', await until(async () => (await phase(page)) === 'done') + && await until(async () => (await textOf(channelLine.locator('[data-codes-filled]'))).includes('含义已填写')) + && (await textOf(channelLine.locator('[data-profile-fill]'))) === '修改含义' && await channelLine.locator('[data-codes-pending]').count() === 0 + && (await textOf(box(page).locator('[data-profile-task-detail="orders"] [data-column="status"] [data-codes-pending]'))) === '2 个含义待填写', + await textOf(channelLine)) + + // 重试没做完的表:别处又建了一个(正忙)时写清是谁的;之后只重试访问记录、退款 + replies.retry.push(json({ detail: `数据源「${NAME1}」正在进行数据剖析(已完成 0 / 3 张表),请等这次剖析结束(或先停止它)再开始新的剖析`, code: 'profile_busy' }, 409)) + await page.locator('[data-profile-retry]').click() + check('重试撞上正忙(409):转到「开始不了」,写明正忙和下一步', await until(async () => (await phase(page)) === 'blocked') + && (await inBox(page, '[data-profile-blocked="busy"]')).includes('这个数据源已有剖析任务'), await inBox(page, '[data-profile-blocked]')) + await page.locator('[data-profile-back]').click() + await page.locator('[data-profile-last-open]').click() + await until(async () => (await phase(page)) === 'done') + check('……「返回」再点开上一次:还是这次的结果', (await inBox(page, '[data-profile-counts]')).startsWith('6 张表')) + await page.locator('[data-profile-retry]').click() + check('重试:只含部分完成和失败的 2 张,进入新任务的进行中', await until(async () => (await phase(page)) === 'running') + && JSON.stringify(state.pjobs.at(-1).tasks.map((t) => t.table_name)) === JSON.stringify(['visits', 'refunds']) + && state.pjobs.at(-1).retry_of === job.id && (await box(page).locator('[data-profile-task]').count()) === 2) + const again = state.pjobs.at(-1) + // 关着对话框时结束:右下角提示,带「查看报告」 + await page.locator('[data-profile-hide]').click() + settle(state, again, 'visits', { queries: 12, elapsed_ms: 9000 }) + settle(state, again, 'refunds', { queries: 3, elapsed_ms: 900 }) + finish(again) + const toastBox = page.locator('[data-toast]').filter({ hasText: '数据剖析完成' }) + check('关着对话框时结束:提示「数据剖析完成」,带「查看报告」', await until(async () => (await toastBox.count()) > 0), await toasts(page)) + await toastBox.getByRole('button', { name: '查看报告' }).click() + check('……点「查看报告」打开这一次的结果', await until(async () => (await phase(page)) === 'done') && (await inBox(page, '[data-profile-title]')) === '剖析完成') + check('……都做完了:不再给重试', await page.locator('[data-profile-retry]').count() === 0) + await page.locator('[data-profile-close]').click() + await openDialog(page) + check('结果看过了,再点页头就是重新选范围', (await phase(page)) === 'setup') + check('写请求只有建任务、重试和保存码值的含义', writes(sent, /./).every((s) => /profile-jobs(\/[^/]+\/retry)?$|^PUT .*catalog\/orders$/.test(s.key)), + JSON.stringify(writes(sent, /./).map((s) => s.key))) + check('没有原生对话框', natives.length === 0, natives.join(' | ')) + check('没有运行时报错', errors.length === 0, errors[0] ?? '') + await close() +}) + +await section('剖析 · 止损与服务重启', async () => { + const state = freshState() + const { page, errors, close } = await open(`/data/catalog/${S1}`, { handlers: handlers(state) }) + await page.locator('[data-catalog-row]').first().waitFor() + await openDialog(page) + await page.locator('[data-profile-start]').click() + await until(async () => (await phase(page)) === 'running') + const job = state.pjobs.at(-1) + settle(state, job, 'orders', { queries: 31, elapsed_ms: 26000 }) + settle(state, job, 'visits', { status: 'running', done: { relation: 1 }, queries: 9, elapsed_ms: 70000 }) + // 最近 10 条查询里 7 条超时:任务停下,后面的表记已停止、原因同一句 + const SLOW = '数据库响应慢:最近 10 条查询 7 条超时,已停止,避免给业务库加压' + job.stop_reason = 'slow' + stopJob(job, { status: 'partial', reason: 'slow', current: '数据库响应慢,这一项没有查', error: SLOW, rest: SLOW }) + check('止损(多数超时):写「数据库响应慢,剖析已停止」和服务端的整句、下一步,给「修改设置」', await until(async () => (await phase(page)) === 'done') + && (await inBox(page, '[data-profile-stopped="slow"]')).includes('数据库响应慢,剖析已停止') + && (await inBox(page, '[data-profile-stopped="slow"]')).includes(SLOW) + && (await inBox(page, '[data-profile-stopped="slow"]')).includes('业务库不忙的时候重试没做完的表') + && await box(page).locator('[data-profile-stopped] [data-profile-settings-open]').count() === 1, await inBox(page, '[data-profile-stopped]')) + check('……同一句原因不逐表重复', await box(page).locator('[data-profile-task-note]').filter({ hasText: SLOW }).count() === 0) + check('……后面的表记已停止,可以重试', (await inBox(page, '[data-profile-task="products"] [data-profile-task-status]')) === '已停止' + && (await textOf(page.locator('[data-profile-retry]'))).includes('(4 张)')) + await shot(page, 'profile-slow') + await page.locator('[data-profile-close]').click() + + // 连续失败:写出第一行报错,下一步是先测试连接 + await openDialog(page) + await page.locator('[data-profile-start]').click() + await until(async () => (await phase(page)) === 'running') + const job2 = state.pjobs.at(-1) + const FAIL = '连续 3 条查询失败,已停止:could not connect to server: Connection refused' + job2.stop_reason = 'failed' + settle(state, job2, job2.tasks[0].table_name, { status: 'running', queries: 3 }) + stopJob(job2, { status: 'partial', reason: 'failed', error: FAIL, rest: FAIL }) + check('止损(连续失败):写「连续多条查询失败」、服务端的第一行报错、先测试连接', await until(async () => (await phase(page)) === 'done') + && (await inBox(page, '[data-profile-stopped="failed"]')).includes('连续多条查询失败,剖析已停止') + && (await inBox(page, '[data-profile-stop-error]')) === FAIL && (await inBox(page, '[data-profile-stopped="failed"]')).includes('测试连接')) + await page.locator('[data-profile-close]').click() + + // 服务重启:任务标中断,不自动续跑 + await openDialog(page) + await page.locator('[data-profile-start]').click() + await until(async () => (await phase(page)) === 'running') + const job3 = state.pjobs.at(-1) + settle(state, job3, job3.tasks[0].table_name, { queries: 5 }) + stopJob(job3, { status: 'interrupted', error: '服务重启,剖析中断' }) + check('服务重启:标题写「服务重启,剖析中断」,写明不会自动续上、可以重试没做完的表', await until(async () => (await phase(page)) === 'done') + && (await inBox(page, '[data-profile-title]')) === '服务重启,剖析中断' && (await inBox(page, '[data-profile-interrupted]')).includes('不会自动续上')) + check('……和任务相同的原因不逐表重复', await box(page).locator('[data-profile-task-note]').filter({ hasText: '服务重启' }).count() === 0) + check('没有运行时报错', errors.length === 0, errors[0] ?? '') + await close() +}) + +await section('剖析 · 开始不了(409)', async () => { + const state = freshState() + const replies = { create: [], patch: [] } + // 这个源在数据源列表里没开剖析(服务端也会说没开) + state.sources = state.sources.map((s) => (s.id === S1 ? { ...s, options: {} } : s)) + const { page, sent, errors, natives, close } = await open(`/data/catalog/${S1}`, { handlers: handlers(state, replies) }) + await page.locator('[data-catalog-row]').first().waitFor() + await openDialog(page) + check('没开启:写明需要先开启,「开始剖析」禁用,没有预算', (await inBox(page, '[data-profile-disabled]')).includes('未开启数据剖析') + && await page.locator('[data-profile-start]').isDisabled() && await box(page).locator('[data-profile-budget]').count() === 0) + await box(page).locator('[data-profile-disabled] [data-profile-enable]').click() + const settings = page.locator('[data-profile-settings-dialog]') + await settings.waitFor() + await settings.locator('[data-profile-enabled]').check() + await page.locator(`#profile-${S1}-table_max_queries`).fill('25') + await page.locator('[data-profile-save]').click() + check('就地开启:保存后回到剖析,预算按刚存的设置', await until(async () => (await settings.count()) === 0) + && await until(async () => (await inBox(page, '[data-profile-budget]')).includes('每张表最多 25 条查询')) + && !(await page.locator('[data-profile-start]').isDisabled())) + check('……保存的是这个源的剖析设置', JSON.stringify(writes(sent, /^PATCH /).at(-1)?.body) === JSON.stringify({ options: { catalog_profile: { enabled: true, table_max_queries: 25 } } }), + JSON.stringify(writes(sent, /^PATCH /).at(-1)?.body)) + + // 服务端的 409 都带机读码,界面按它分情况,不从原话里认 + const cases = [ + ['disabled', '数据源「zzprofshop」未开启数据剖析。剖析会对业务库发查询,请先在数据源设置中开启数据剖析', '未开启数据剖析', 'data-profile-enable', 'profile_disabled'], + ['inactive', '数据源「zzprofshop」已停用,无法进行数据剖析。请先在数据源设置中启用', '数据源已停用', 'data-profile-go-source', 'datasource_inactive'], + ['noSchema', '尚未探查结构,无法进行数据剖析。请先在数据源卡片上点「探查结构」', '还没有表结构', 'data-profile-go-source', 'schema_missing'], + ['tampered', '数据文件与登记的版本不一致,可能被修改过,已拒绝查询', '数据文件核对未通过', 'data-profile-go-source', 'snapshot_tampered'], + ['nothing', '按「需要剖析的表」挑出 0 张表:表都剖析过、上次都做完了,也没有新的待核实关联关系。可以改选「全部表」,或指定要剖析的表', '没有可剖析的表', + 'data-profile-choose-all', 'profile_nothing'], + ] + for (const [kind, detail, title, action, code] of cases) { + replies.create.push(json({ detail, code }, 409)) + await page.locator('[data-profile-start]').click() + const blocked = box(page).locator(`[data-profile-blocked="${kind}"]`) + const ok = await until(async () => (await blocked.count()) === 1) + const text = ok ? await textOf(blocked) : '' + check(`409(${title}):写明是哪种情况、服务端原话和下一步,给对应的按钮`, ok && text.includes(title) && text.includes(detail) + && await blocked.locator(`[${action}]`).count() === 1, text.slice(0, 160)) + if (kind === 'nothing') { + // 改选全部表:这回建得起来 + await blocked.locator('[data-profile-choose-all]').click() + check('……「改选全部表」:按全部表再建一次,进入进行中', await until(async () => (await phase(page)) === 'running') + && JSON.stringify(lastBody(sent, /profile-jobs$/)) === JSON.stringify({ scope: 'all' })) + stopJob(state.pjobs.at(-1)) + await until(async () => (await phase(page)) === 'done') + await page.locator('[data-profile-again]').click() + await until(async () => (await phase(page)) === 'setup') + continue + } + if (kind === 'disabled') { + // 服务端说没开(列表里的设置过时了):同样就地开启 + await blocked.locator('[data-profile-enable]').click() + await settings.waitFor() + await page.locator(`#profile-${S1}-table_max_queries`).fill('30') + await page.locator('[data-profile-save]').click() + check('……就地开启后回到选范围', await until(async () => (await phase(page)) === 'setup')) + continue + } + await page.locator('[data-profile-back]').click() + await until(async () => (await phase(page)) === 'setup') + } + // 没带机读码的 409(老服务端):不从原话里猜,按「无法开始剖析」说,给重试 + replies.create.push(json({ detail: '数据源正在进行数据剖析,请等待完成后再试' }, 409)) + await page.locator('[data-profile-start]').click() + check('409 没带机读码:不从原话里猜,按「无法开始剖析」说,给重试', await until(async () => (await box(page).locator('[data-profile-blocked="other"]').count()) === 1) + && await box(page).locator('[data-profile-retry-start]').count() === 1) + await box(page).locator('[data-profile-retry-start]').click() + check('……「重试」同一个范围再发一次,这次建起来', await until(async () => (await phase(page)) === 'running')) + stopJob(state.pjobs.at(-1)) + await until(async () => (await phase(page)) === 'done') + const latestN = count(sent, /profile-jobs\/latest$/) + await page.locator('[data-profile-again]').click() + await until(async () => (await phase(page)) === 'setup') + // 正忙:别处(另一个标签页)在这一页打开以后建了一个,写清是谁的、到哪了,可以去看它的进度。先等设置阶段把上一次剖析 + // 取回来,免得它晚到、取回的已经是别处那个进行中的任务(那样会直接转去显示它,走不到 409) + await until(async () => count(sent, /profile-jobs\/latest$/) > latestN) + await sleep(300) + const other = newProfileJob(state, S1, { tables: ['orders', 'customers', 'visits'], scope: 'selected', created_by: OTHER }) + settle(state, other, 'orders', { queries: 20 }) + await page.locator('[data-profile-start]').click() + const busy = box(page).locator('[data-profile-blocked="busy"]') + check('409(正忙):写服务端原话、谁发起的、已完成几张,下一步可以查看它的进度', await until(async () => (await busy.count()) === 1) + && (await textOf(busy)).includes('已完成 1 / 3 张表') && (await inBox(page, '[data-profile-busy-at]')) === `由「${OTHER}」发起,已完成 1 / 3 张表` + && await busy.locator('[data-profile-view-active]').count() === 1, await textOf(busy)) + if (SHOTS) await sleep(350) + await shot(page, 'profile-blocked') + await busy.locator('[data-profile-view-active]').click() + check('……「查看它的进度」:跟着进行中的那一个,写明是已有的任务', await until(async () => (await phase(page)) === 'running') + && (await inBox(page, '[data-profile-busy]')).includes('已有剖析任务在进行') && (await inBox(page, '[data-profile-progress]')) === '1 / 3 张表' + && (await inBox(page, '[data-profile-created-by]')) === `由「${OTHER}」发起`) + stopJob(other) + await until(async () => (await phase(page)) === 'done') + await page.locator('[data-profile-again]').click() + await until(async () => (await phase(page)) === 'setup') + // 不是 409 的失败:留在设置里写明,能再点开始 + replies.create.push(json({ detail: '服务端处理剖析时出错,请稍后重试' }, 500)) + await page.locator('[data-profile-start]').click() + check('其余失败:留在设置里写「未能开始剖析」和原因', await until(async () => (await inBox(page, '[data-profile-start-error]')).startsWith('未能开始剖析')) + && (await phase(page)) === 'setup' && !(await page.locator('[data-profile-start]').isDisabled())) + // 去数据源:停用、没有表结构、快照被改动时 + replies.create.push(json({ detail: cases[1][1], code: cases[1][4] }, 409)) + await page.locator('[data-profile-start]').click() + await box(page).locator('[data-profile-go-source]').click() + await page.waitForURL('**/data/databases', { timeout: 5000 }).catch(() => {}) + check('「前往数据源」回到数据源卡片', new URL(page.url()).pathname === '/data/databases', page.url()) + check('写请求只有建任务、停止和保存剖析设置', writes(sent, /./).every((s) => /profile-jobs(\/[^/]+\/cancel)?$|^PATCH \/datasources\/check-prof-shop$/.test(s.key)), + JSON.stringify(writes(sent, /./).map((s) => s.key))) + check('没有原生对话框', natives.length === 0, natives.join(' | ')) + check('没有运行时报错', errors.length === 0, errors[0] ?? '') + await close() +}) + +await section('剖析 · 和起草排队', async () => { + const state = freshState() + // 起草任务在进行:剖析排在它后面 + const draft = newDraftJob(state, S1, ['orders', 'customers', 'visits']) + Object.assign(draft, { status: 'running', started_at: isoNow() }) + Object.assign(draft.tasks[0], { status: 'ok', finished_at: isoNow() }) + const { page, sent, errors, close } = await open(`/data/catalog/${S1}`, { handlers: handlers(state) }) + await page.locator('[data-catalog-row]').first().waitFor() + check('页头的起草入口写进度「起草 1/3」', await until(async () => (await page.locator('[data-catalog-draft-head="running"]').getAttribute('aria-label')) === '起草 1/3')) + await openDialog(page) + check('同源有起草任务:按钮写「排队剖析」,下面写起草任务结束后自动开始', (await textOf(page.locator('[data-profile-start]'))).includes('排队剖析') + && (await page.locator('[data-profile-start]').getAttribute('data-profile-start')) === 'queue' + && (await inBox(page, '[data-profile-queue]')).includes('起草任务结束后自动开始')) + await page.locator('[data-profile-start]').click() + await until(async () => (await phase(page)) === 'running') + const job = state.pjobs.at(-1) + check('……建出来是排队,写「排在起草任务后面,起草结束后自动开始」', job.waiting_for === 'draft' && (await inBox(page, '[data-profile-title]')).startsWith('排队中') + && (await inBox(page, '[data-profile-waiting="draft"]')) === '排在起草任务后面,起草结束后自动开始') + await page.locator('[data-profile-hide]').click() + check('……页头写「剖析 排队中」,悬停写排在起草任务后面', await until(async () => (await head(page).getAttribute('data-catalog-profile')) === 'queued') + && (await textOf(head(page))) === '剖析 排队中' && (await head(page).getAttribute('title')).includes('排在起草任务后面')) + await shot(page, 'profile-queued') + // 起草结束、剖析开始 + for (const t of draft.tasks) Object.assign(t, { status: 'ok', finished_at: isoNow() }) + Object.assign(draft, { status: 'succeeded', finished_at: isoNow() }) + settle(state, job, job.tasks[0].table_name, { status: 'running', queries: 2 }) + await goto(page, `/data/catalog/${S1}`) + check('起草结束后剖析开始:页头写「剖析 0/5」', await until(async () => (await head(page).getAttribute('aria-label')) === `剖析 0/${job.tasks.length}`), + await head(page).getAttribute('aria-label').catch(() => '')) + + // 反过来:剖析在进行时起草,排在剖析后面 + await page.locator('[data-catalog-draft]').first().click() + const draftBox = page.locator('[data-draft-dialog]') + await draftBox.waitFor() + check('起草设置:写明有剖析任务在进行,起草会排在它后面', (await textOf(draftBox.locator('[data-draft-queue]'))).includes('起草会排在它后面,剖析结束后自动开始')) + await page.locator('[data-draft-start]').click() + check('……建出来的起草任务排在剖析后面:写「排在剖析任务后面,剖析结束后自动开始」', await until(async () => + (await textOf(draftBox.locator('[data-draft-waiting="profile"]'))) === '排在剖析任务后面,剖析结束后自动开始')) + await page.locator('[data-draft-hide]').click() + check('……目录页上方的起草进度也这么写', await until(async () => (await textOf(page.locator('[data-catalog-draft-progress]'))).includes('起草任务排在剖析任务后面,剖析结束后自动开始')), + await textOf(page.locator('[data-catalog-draft-progress]'))) + check('……页头的起草入口写「起草 排队中」,悬停写排在剖析任务后面', (await textOf(page.locator('[data-catalog-draft-head="queued"]'))) === '起草 排队中' + && (await page.locator('[data-catalog-draft-head="queued"]').getAttribute('title')).includes('排在剖析任务后面')) + stopJob(job) + check('写请求只有建剖析任务、建起草任务', writes(sent, /./).every((s) => /(profile|draft)-jobs$/.test(s.key)), JSON.stringify(writes(sent, /./).map((s) => s.key))) + check('没有运行时报错', errors.length === 0, errors[0] ?? '') + await close() +}) + +await section('剖析 · 表清单的「剖析」一列与筛选', async () => { + const state = freshState() + const { page, sent, errors, close } = await open(`/data/catalog/${S1}`, { handlers: handlers(state) }) + await page.locator('[data-catalog-row]').first().waitFor() + const cell = (n) => rowProfile(page, n) + const tone = (n) => page.locator(`[data-catalog-row="${n}"] [data-row-profile]`).getAttribute('data-row-profile').catch(() => null) + check('从没剖析过:「未剖析」,用提醒色', (await cell('orders')) === '未剖析' && (await tone('orders')) === 'due', await cell('orders')) + check('剖析过、做完了:「3 天前剖析」', (await cell('customers')) === '3 天前剖析' && (await tone('customers')) === 'quiet', await cell('customers')) + check('上次没做完:「剖析部分完成」,用提醒色', (await cell('visits')) === '剖析部分完成' && (await tone('visits')) === 'due') + check('剖析之后出现了新的待核实关系:「有新关系待剖析」', (await cell('products')) === '有新关系待剖析' && (await tone('products')) === 'due') + check('上次失败:「剖析失败」,用错误色', (await cell('refunds')) === '剖析失败' && (await tone('refunds')) === 'bad') + check('没有可剖析的项:「无可剖析项」;视图、只剩目录的表不写', (await cell('settings_kv')) === '无可剖析项' + && await page.locator('[data-catalog-row="order_summary"] [data-row-profile]').count() === 0 + && await page.locator('[data-catalog-row="legacy_coupons"] [data-row-profile]').count() === 0) + const title = await page.locator('[data-catalog-row="visits"] [data-row-profile]').getAttribute('title') + check('……悬停写上次剖析的时间、结果和为什么需要剖析', title.startsWith('上次剖析(') && title.includes('部分完成') && title.includes('上次剖析没做完'), title) + const due = page.locator('[data-catalog-profile-due]') + check('「需要剖析」开关带数量 5,审阅进度的四项筛选照旧', (await textOf(due)).replace(/\s+/g, '') === '需要剖析5' + && (await page.locator('[data-catalog-filter] [data-filter]').count()) === 4, await textOf(due)) + await due.click() + const names = await page.locator('[data-catalog-row]').evaluateAll((els) => els.map((e) => e.getAttribute('data-catalog-row'))) + check('打开:只列需要剖析的 5 张', names.join(',') === 'orders,visits,products,refunds,tmp_log', names.join(',')) + check('……写进地址栏(profile=due),按钮标为按下', await until(async () => new URL(page.url()).searchParams.get('profile') === 'due') + && (await due.getAttribute('aria-pressed')) === 'true') + await page.reload({ waitUntil: 'networkidle' }) + await page.locator('[data-catalog-row]').first().waitFor() + check('……刷新之后还在', (await page.locator('[data-catalog-row]').count()) === 5 && (await page.locator('[data-catalog-profile-due]').getAttribute('aria-pressed')) === 'true') + // 筛出来一键剖析:当前筛选结果就是这 5 张 + await openDialog(page) + check('……打开剖析:多一项「当前筛选结果中的 5 张表」', await box(page).locator('[data-profile-scope="filtered"]').count() === 1 + && (await inBox(page, '[data-profile-scope="filtered"]')).startsWith('当前筛选结果中的 5 张表'), await inBox(page, '[data-profile-scope="filtered"]')) + await page.keyboard.press('Escape') + await page.locator('[data-catalog-profile-due]').click() + check('关掉:回到全部 9 张,地址里不留', await until(async () => (await page.locator('[data-catalog-row]').count()) === 9 && new URL(page.url()).search === '')) + await shot(page, 'profile-list') + // 没开剖析的源:每行不写「剖析」,没有开关 + await goto(page, `/data/catalog/${S2}`) + await page.locator('[data-catalog-row]').first().waitFor() + check('没开剖析的源(也没剖析过):不写「剖析」一格,没有「需要剖析」开关', await page.locator('[data-row-profile]').count() === 0 + && await page.locator('[data-catalog-profile-due]').count() === 0) + check('只读、不发写请求', writes(sent, /./).length === 0) + check('没有运行时报错', errors.length === 0, errors[0] ?? '') + await close() +}) + +await section('剖析 · 表详情的「数据剖析」一节', async () => { + const state = freshState() + const replies = { create: [] } + const { page, sent, errors, natives, close } = await open(`/data/catalog/${S1}/customers`, { handlers: handlers(state, replies) }) + const sec = page.locator('[data-catalog-section="profile"]') + await sec.waitFor() + const old = state.pjobs[0] + check('上次剖析:结果、时间、发起人、发了几条查询,给「查看这次剖析」', await until(async () => (await textOf(sec.locator('[data-catalog-profile-last]'))).includes('完成')) + && (await textOf(sec.locator('[data-catalog-profile-last]'))).includes('上次剖析:') && (await textOf(sec.locator('[data-catalog-profile-last]'))).includes(`${OTHER} 发起`) + && (await textOf(sec.locator('[data-catalog-profile-last]'))).includes('4 条查询') && await sec.locator(`[data-catalog-profile-job="${old.id}"]`).count() === 1, + await textOf(sec.locator('[data-catalog-profile-last]'))) + check('……行数估算和来源', (await textOf(sec.locator('[data-profile-rows]'))) === '行数:约 12,000 行(来自数据库统计信息)', await textOf(sec.locator('[data-profile-rows]'))) + check('……发现摘要、码值候选', (await textOf(sec.locator('[data-catalog-profile-findings]'))) === '取到 1 列码值候选(2 个含义待填写)' + && await sec.locator('[data-profile-finding="codes"][data-column="level"]').count() === 1) + check('……日期范围', (await textOf(sec.locator('[data-profile-date-range="joined_at"]'))).includes('取值从 2021-03-01 08:00:00 到 2026-10-01 21:30:00')) + check('……跳过的项(按规则不查的遮罩列)', (await textOf(sec.locator('[data-profile-skip="masked"]'))).includes('列已遮罩')) + check('……发过的查询(折着)', (await textOf(sec.locator('[data-profile-queries-log] summary'))) === '看发过的查询(3 条)') + await sec.locator(`[data-catalog-profile-job="${old.id}"]`).click() + check('「查看这次剖析」:取来那个任务,打开它的结果', await until(async () => (await phase(page)) === 'done') + && (await inBox(page, '[data-profile-counts]')).startsWith('4 张表') && count(sent, new RegExp(`^GET .*profile-jobs/${old.id}$`)) >= 1) + await page.locator('[data-profile-close]').click() + await shot(page, 'profile-detail') + + // 剖析这张表:建只含这张表的任务;这张表在任务里时写「正在剖析这张表」和正在做哪一项 + await sec.locator('[data-catalog-profile-table]').click() + check('「剖析这张表」:建只含这张表的任务,提示已开始、可以查看进度', await until(async () => JSON.stringify(lastBody(sent, /profile-jobs$/)) === JSON.stringify({ tables: ['customers'] })) + && await until(async () => (await page.locator('[data-toast]').filter({ hasText: '已开始剖析 customers' }).count()) > 0), await toasts(page)) + const job = state.pjobs.at(-1) + settle(state, job, 'customers', { status: 'running', done: { codes: 1 }, queries: 2, current: '正在取 customers.joined_at 的最小值和最大值' }) + check('……这一节写「正在剖析这张表」和正在做哪一项,按钮不能再点', await until(async () => (await textOf(sec.locator('[data-catalog-profile-live="running"]'))).includes('正在剖析这张表')) + && (await textOf(sec.locator('[data-catalog-profile-live]'))).includes('正在取 customers.joined_at 的最小值和最大值') + && await sec.locator('[data-catalog-profile-table]').isDisabled()) + // 正有剖析任务时剖析别的表(409 正忙):写明进行中的那个到哪了、这张表在不在里面 + await page.locator('[data-catalog-row="products"] [data-catalog-open-row]').click() + await until(async () => (await page.locator('[data-catalog-detail="products"][data-version]').count()) === 1) + await sec.locator('[data-catalog-profile-table]').click() + const blocked = sec.locator('[data-catalog-profile-blocked="busy"]') + check('正忙(409):写服务端原话、进行中的那个到哪了、这张表不在其中', await until(async () => (await blocked.count()) === 1) + && (await textOf(blocked)).includes('正在进行数据剖析') && (await textOf(sec.locator('[data-catalog-profile-busy-where]'))).includes('这张表不在其中,等它结束后再剖析这张表'), + await textOf(blocked)) + await blocked.locator('[data-catalog-profile-progress]').click() + check('……「查看进度」打开进行中的那一个', await until(async () => (await phase(page)) === 'running') && (await inBox(page, '[data-profile-progress]')) === '0 / 1 张表') + await page.locator('[data-profile-hide]').click() + settle(state, job, 'customers', { queries: 4, elapsed_ms: 2000, row_estimate: { rows: 12500, method: 'stats', at_least: null } }) + finish(job) + await page.locator('[data-catalog-row="customers"] [data-catalog-open-row]').click() + check('剖析完:这一节换成这次的结果(行数更新),发起人是本机', await until(async () => (await textOf(sec.locator('[data-profile-rows]'))).includes('约 12,500 行')) + && (await textOf(sec.locator('[data-catalog-profile-last]'))).includes(`${ACTOR} 发起`)) + // 没剖析过、视图、没有可剖析项 + await page.locator('[data-catalog-row="orders"] [data-catalog-open-row]').click() + check('从没剖析过的表:写「还没有剖析过」,给「剖析这张表」', await until(async () => (await textOf(sec.locator('[data-catalog-profile-never]'))) === '这张表还没有剖析过。') + && !(await sec.locator('[data-catalog-profile-table]').isDisabled())) + await page.locator('[data-catalog-row="order_summary"] [data-catalog-open-row]').click() + check('视图:写「视图不剖析」,不给按钮', await until(async () => (await textOf(sec.locator('[data-catalog-profile-never]'))) === '视图不剖析。') + && await sec.locator('[data-catalog-profile-table]').count() === 0) + await page.locator('[data-catalog-row="settings_kv"] [data-catalog-open-row]').click() + check('没有可剖析的项:写明,不给按钮', await until(async () => (await textOf(sec.locator('[data-catalog-profile-never]'))).includes('没有可剖析的项')) + && await sec.locator('[data-catalog-profile-table]').count() === 0) + // 没开剖析的源:引到剖析设置 + await goto(page, `/data/catalog/${S2}/orders`) + await sec.waitFor() + check('没开剖析的源:写明未开启,「剖析这张表」不能点,给「开启数据剖析」', (await textOf(sec.locator('[data-catalog-profile-disabled]'))).includes('未开启数据剖析') + && await sec.locator('[data-catalog-profile-table]').isDisabled()) + await sec.locator('[data-catalog-profile-enable]').click() + check('……点开剖析设置', await until(async () => (await page.locator('[data-profile-settings-dialog]').count()) === 1)) + await page.keyboard.press('Escape') + check('写请求只有建任务', writes(sent, /./).every((s) => /profile-jobs$/.test(s.key)), JSON.stringify(writes(sent, /./).map((s) => s.key))) + check('没有原生对话框', natives.length === 0, natives.join(' | ')) + check('没有运行时报错', errors.length === 0, errors[0] ?? '') + await close() +}) + +await section('剖析 · 窄屏', async () => { + const state = freshState() + const { page, errors, close } = await open(`/data/catalog/${S1}`, { handlers: handlers(state), viewport: { width: 390, height: 844 } }) + await page.locator('[data-catalog-row]').first().waitFor() + const overflow = () => page.evaluate(() => { + const root = document.querySelector('[data-catalog-page]') + return root ? root.scrollWidth - root.clientWidth : 99 + }) + check('390 宽:表清单带「剖析」一格,不横向滚动', await page.locator('[data-row-profile]').count() > 0 && await overflow() <= 1, String(await overflow())) + await openDialog(page) + await page.locator('[data-profile-start]').click() + await until(async () => (await phase(page)) === 'running') + const job = state.pjobs.at(-1) + settle(state, job, 'orders', { status: 'running', done: { relation: 3, codes: 5, date: 1 }, queries: 23, elapsed_ms: 18000, + current: '正在核对 orders.coupon_id → coupons.id,这一条要先抽样再到被指向表里核对' }) + await until(async () => (await inBox(page, '[data-profile-task="orders"] [data-profile-task-checks]')) !== '') + await taskLine(page, 'orders').locator('[data-profile-task-toggle]').click() + await until(async () => (await box(page).locator('[data-profile-checks]').count()) === 1) + const d = await page.locator('[role="dialog"]').last().boundingBox() + const inner = await box(page).evaluate((el) => el.scrollWidth - el.clientWidth) + check('……进行中、展开一张表:对话框在视口里,正文不横向滚动', !!d && d.x >= 0 && d.x + d.width <= 390 && inner <= 1, `${JSON.stringify(d)} ${inner}`) + await shot(page, 'profile-narrow') + stopJob(job) + check('没有运行时报错', errors.length === 0, errors[0] ?? '') + await close() +}) + +await browser.close() + +const after = await realCounts() +check('真库里的数据源和目录数量没有变化', JSON.stringify(after) === JSON.stringify(before), `${JSON.stringify(before)} → ${JSON.stringify(after)}`) +check('检查用的假数据源没有写进真库', !(await (await fetch(`${API}/datasources`)).json()).some((s) => NAMES.includes(s.name))) + +console.log(failed ? `\n✗ ${failed} 项未通过` : '\n✓ 数据剖析全部通过') +process.exit(failed ? 1 : 0) diff --git a/scripts/check-catalog.mjs b/scripts/check-catalog.mjs index 2e762729..7bfa38f5 100644 --- a/scripts/check-catalog.mjs +++ b/scripts/check-catalog.mjs @@ -6,8 +6,7 @@ // 起草是后台任务:查的是「起草要求和补充说明有没有随请求发出」「关掉对话框任务照常进行、页面上有进度、再打开就是它」 // 「能停」「哪张表没写全、为什么写得清」「能只重试失败的」「已有任务、模型用不了这两种 409 各有去处」「清单上标出上次 // 起草失败的表、能单独重起草」。任务怎么往前走由检查自己推,页面每 1.5 秒取一次。 -// 数据剖析(阶段 4B)会对业务库发查询:剖析接口、保存数据源一律在浏览器层答掉,查的是「确认前写没写预算」「进行中是否 -// 明说停不下来」「报告说没说清用了几条查询、为什么停、跳过了什么」「各种 409 有没有下一步」「设置被拒时报错落没落到那一项」。 +// 数据剖析(后台任务、逐表进行)另有 check-catalog-profile;这里只在窄屏一节看剖析对话框放不放得下,剖析的建任务一律拦成 503。 // // 写法同 check-versions:沙箱里的数据源多半没探查过结构,目录也是空的,所以数据源列表、目录清单、单表目录都用 // page.route 伪造(虚构的景区业务库和一个导入表格的源);写请求(整份提交、单项审阅、起草)也在浏览器层答掉, @@ -646,8 +645,9 @@ function catalogHandlers(state, replies = {}) { const name = parts(key)[4] return json({ table: name, templates: state.impact?.[name] ?? [] })(route) })], - // 剖析没配答复的一律 503:检查脚本不能让剖析真的发到业务库 - [/^POST \/datasources\/check-cat-[a-z]+\/catalog\/profile$/, reply('profile', json({ detail: '检查脚本没有为这次剖析准备答复' }, 503))], + // 剖析任务(数据剖析的检查在 check-catalog-profile):这里没有剖析任务,建任务一律 503,不让剖析真的发到业务库 + [/^GET \/datasources\/check-cat-[a-z]+\/catalog\/profile-jobs\/(active|latest)$/, json({ job: null })], + [/^POST \/datasources\/check-cat-[a-z]+\/catalog\/profile-jobs(\/.*)?$/, json({ detail: '检查脚本没有为这次剖析准备答复' }, 503)], // 保存数据源(剖析设置):照提交改假状态,回整条数据源 [/^PATCH \/datasources\/check-cat-[a-z]+$/, reply('patch', (route, { key, body }) => { const id = parts(key)[2] @@ -2259,91 +2259,8 @@ await section('数据目录 · 空状态与导入表格', async () => { await close() }) -// --------------------------------------------------------------------------- -// 数据剖析(阶段 4B):剖析会对业务库发查询,这里一条都不能真的发出去——剖析接口和保存数据源都在浏览器层答掉, -// 照服务端(data/catalog_profile.py)的结论改假状态 -// --------------------------------------------------------------------------- -const GATE_REL = relId(['gate_id'], 'gates', ['id']) -const MEMBER_REL = relId(['member_id'], 'members', ['id']) -const PROFILE_DAY = '2026-10-03' -const PROFILE_ON = { enabled: true, max_queries: 60, query_timeout_s: 10 } -const withProfile = (state, src, value) => { - state.sources = state.sources.map((s) => (s.id === src ? { ...s, options: { ...s.options, catalog_profile: value } } : s)) -} -const BUDGET_DETAIL = '本次剖析的查询次数已用完(上限 60 条),未检查' - -/** - * 剖析的默认答复:入园记录的闸机关系核实成已验证(覆盖率 98.7%),会员关系覆盖率不够保持推断,状态列的码值候选多出一个 - * 没有含义的取值 2;订单只剖析了一半就用完了查询次数,订单明细一条没查。ms 是服务端「执行」多久 - */ -function profileReply(state, { ms = 0, extraCodes = false } = {}) { - return async (route) => { - if (ms) await sleep(ms) - const visits = state.tables[S1].find((t) => t.name === 'visits') - const notes = clone(visits.notes) - const at = `${PROFILE_DAY}T03:00:00+00:00` - // 剖析之前这两条关系的状态(服务端 previous_status):第一次闸机关系是命名推断,再剖析一次时已经是已验证 - const before = (id) => visits.notes.relations.find((r) => r.id === id)?.status ?? null - const previous = { gate: before(GATE_REL), member: before(MEMBER_REL) } - notes.relations = notes.relations.map((r) => (r.id === GATE_REL ? { - ...r, source: 'profile', status: 'verified', coverage: 0.987, cardinality: 'many_to_one', cardinality_checked: true, updated_at: at, - note: '数据剖析:子表抽样 300 个不同键值,父表对上 296 个,覆盖率 98.7%;被指向列是主键;子表一侧有重复值,多对一。', - } : r)) - notes.relations.push(rel(['member_id'], 'members', ['id'], 'profile', { coverage: 0.62, updated_at: at, - note: '数据剖析:抽样覆盖率 62%,可能不是这条关系;子表抽样 200 个不同键值,父表对上 124 个;被指向列是主键。' })) - notes.columns.status.codes = { ...item({ 1: '已入园', 9: '已作废', 2: '' }, 'profile', 'proposed'), updated_at: at } - notes.columns.status.codes.note = '数据剖析:统计全表,182000 行非空值共 3 个取值:1(94%)、9(5%)、2(1%)。' - // 统计的是全表:服务端给码值候选标「已列全」 - notes.columns.status.codes.complete = true - // extraCodes:票种列也取到码值候选(两个取值,含义都空着)——报告里有两列要填,填完一列回到报告接着填下一列 - if (extraCodes) notes.columns.ticket_type_id = { codes: item({ 11: '', 12: '' }, 'profile', 'proposed') } - writeEntry(visits, notes) - const tables = [ - { - table_name: 'visits', queries: 38, row_estimate: { rows: 182000, method: 'stats', at_least: null }, - findings: [ - { kind: 'relation', path: `relations.${GATE_REL}`, target: 'gate_id → gates.id', columns: ['gate_id'], to_table: 'gates', to_columns: ['id'], - status: 'verified', confirmed: false, previous_status: previous.gate, coverage: 0.987, cardinality: 'many_to_one', cardinality_checked: true, - sample: 300, matched: 296, - summary: previous.gate === 'verified' ? '覆盖率 98.7%,仍为有确证' : '覆盖率 98.7%,升为有确证' }, - { kind: 'relation', path: `relations.${MEMBER_REL}`, target: 'member_id → members.id', columns: ['member_id'], to_table: 'members', - to_columns: ['id'], status: 'proposed', confirmed: false, previous_status: previous.member, coverage: 0.62, cardinality: 'many_to_one', - cardinality_checked: false, sample: 200, matched: 124, - summary: '抽样覆盖率 62%,保持推断' }, - { kind: 'codes', path: 'columns.status.codes', column: 'status', values: [{ value: '1', rows: 171080 }, { value: '9', rows: 9100 }, - { value: '2', rows: 1820 }], rows: 182000, status: 'proposed', pending: 1, summary: '3 个取值' }, - ...(extraCodes ? [{ kind: 'codes', path: 'columns.ticket_type_id.codes', column: 'ticket_type_id', values: [{ value: '11', rows: 120000 }, - { value: '12', rows: 62000 }], rows: 182000, status: 'proposed', pending: 2, summary: '2 个取值' }] : []), - ], - skipped: [ - { kind: 'codes', target: 'member_level', path: 'columns.member_level.codes', reason: 'masked', detail: 'visits.member_level 在数据源设置中被遮罩,不取值' }, - { kind: 'date', target: 'visit_time', path: 'business_date', reason: 'too_large', - detail: 'visits 的行数超过整表统计的行数上限(100000 行),不取日期列的最小值和最大值' }, - ], - date_ranges: [], added: 1, updated: 2, removed: 0, version: visits.version, error: null, - }, - { - table_name: 'orders', queries: 22, row_estimate: { rows: null, method: 'count', at_least: 100001 }, findings: [], - skipped: [{ kind: 'relation', target: 'member_id → members.id', path: 'relations.x', reason: 'budget', detail: BUDGET_DETAIL }], - date_ranges: [], added: 0, updated: 0, removed: 0, version: 1, error: null, - }, - { - table_name: 'order_items', queries: 0, row_estimate: null, findings: [], - skipped: [{ kind: 'table', target: 'order_items', path: null, reason: 'budget', detail: BUDGET_DETAIL }], - date_ranges: [], added: 0, updated: 0, removed: 0, version: 1, error: null, - }, - ] - return json({ - profiled_at: at, actor: ACTOR, settings: { ...PROFILE_ON, sample_size: 2000, max_scan_rows: 100000, max_total_s: 120 }, - queries_used: 60, stopped: 'budget', tables, total: { added: 1, updated: 2, removed: 0 }, note: null, - empty_tables: ['ext_log_001', 'ext_log_002'], - })(route) - } -} +/** 数据剖析对话框(窄屏一节看它放不放得下) */ const profileBox = (page) => page.locator('[data-profile-dialog]') -const profilePhase = (page) => profileBox(page).getAttribute('data-profile-dialog').catch(() => null) -/** 报告里不该露的机读码:停止原因、跳过原因、状态、基数的英文值 */ -const PROFILE_CODE_RE = /\b(?:budget|deadline|failed|masked|too_large|high_cardinality|no_data|unsupported|verified|proposed|many_to_one|row_estimate)\b/ await section('数据目录 · 只探查了一部分表', async () => { // 探查结构每个数据源最多取 200 张表:多出来的不在清单里,助手也看不到。顶部要说出来,并给出下一步 @@ -2377,405 +2294,6 @@ await section('数据目录 · 只探查了一部分表', async () => { await close() }) -await section('数据目录 · 数据剖析设置', async () => { - const state = freshState() - const replies = { patch: [] } - const { page, sent, errors, natives, close } = await open('/data/databases', { handlers: catalogHandlers(state, replies) }) - // 手工登记的库:编辑框里有「数据剖析」一段,默认关闭,预算收着 - await page.locator('[data-source="zzcatscenic"]').getByRole('button', { name: '编辑', exact: true }).click() - const box = page.locator('[data-profile-settings]') - await box.waitFor() - check('编辑框里有「数据剖析」,默认关闭、预算收着', (await box.getAttribute('data-profile-settings')) === 'off' - && await box.locator('[data-profile-field]').count() === 0 && (await box.innerText()).includes('默认关闭')) - await box.locator('[data-profile-enabled]').check() - const labels = await box.locator('label.label').allInnerTexts() - check('开启后五项预算展开,中文标签', labels.join('、') === '查询次数上限、单条查询时限、抽样键值数、整表统计行数上限、总时长上限', labels.join('、')) - const units = await box.locator('[data-profile-field] span').allInnerTexts() - check('……每项带单位', units.join(',') === '条,秒,个,行,秒', units.join(',')) - const hints = await box.locator('[id$="-hint"]').allInnerTexts() - check('……写明范围和默认值,整表统计写明 0 的含义', hints.some((h) => h.startsWith('范围 1–500 条,默认 60')) - && hints.some((h) => h.includes('范围 0–10,000,000 行') && h.includes('填 0 表示一律不做整表统计')), hints.join(' | ')) - const risk = () => box.locator('[data-profile-risk]').innerText().catch(() => '') - check('……风险说明:会发出只读查询,每次最多 60 条、单条 10 秒(缺省)', (await risk()).includes('只读查询:最多 60 条,单条不超过 10 秒'), await risk()) - await page.locator('#ds-profile-max_queries').fill('80') - check('……风险说明跟着填的数走', (await risk()).includes('最多 80 条'), await risk()) - await page.locator('#ds-profile-max_queries').fill('0') - const err = await page.locator('#ds-profile-max_queries-error').innerText().catch(() => '') - check('填错(0 条):就地写明范围,保存按钮禁用', err === '请填写 1 到 500 之间的整数' && await page.locator('[data-source-save]').isDisabled(), err) - check('……输入框标出错(aria-invalid)', (await page.locator('#ds-profile-max_queries').getAttribute('aria-invalid')) === 'true') - await page.locator('#ds-profile-max_queries').fill('80') - await page.locator('#ds-profile-query_timeout_s').fill('20') - // 服务端按字段拒收(422):报错落在那一项下面,光标放进去,不只弹提示 - const REJECT = '数据剖析的「单条查询时限(秒)」需要填写 1 到 60 之间的数;当前为「20」' - // 服务端带机读码和出错的那一项(api/coded.py:code + field),界面按它落到那一格,不从原话里认 - replies.patch.push(json({ detail: REJECT, code: 'profile_settings_invalid', field: 'query_timeout_s' }, 422)) - await page.locator('[data-source-save]').click() - check('保存被拒(422):服务端原话写在「单条查询时限」下面', await until(async () => - (await page.locator('#ds-profile-query_timeout_s-error').innerText().catch(() => '')) === REJECT), - await page.locator('#ds-profile-query_timeout_s-error').innerText().catch(() => '')) - check('……光标落在那一项上,框标出错', await until(async () => page.evaluate(() => document.activeElement?.id === 'ds-profile-query_timeout_s')) - && (await page.locator('#ds-profile-query_timeout_s').getAttribute('aria-invalid')) === 'true') - check('……没有落到「高级连接参数」的查询时限上', await page.locator('#ds-adv-query_timeout_s-error').count() === 0) - if (SHOTS) await sleep(350) - await shot(page, 'catalog-profile-settings-422') - await page.locator('#ds-profile-query_timeout_s').fill('15') - check('改了这一项,报错收起', await page.locator('#ds-profile-query_timeout_s-error').count() === 0) - await page.locator('[data-source-save]').click() - await until(async () => (await page.locator('[data-profile-settings]').count()) === 0) - const saved = writes(sent, /^PATCH /).at(-1)?.body - check('保存:options 里写剖析设置(开关和填了的项,数字)', JSON.stringify(saved?.options?.catalog_profile) - === JSON.stringify({ enabled: true, max_queries: 80, query_timeout_s: 15 }), JSON.stringify(saved?.options)) - check('……卡片上标「数据剖析已开启」,悬停写明预算', await until(async () => (await page.locator('[data-source="zzcatscenic"] [data-profile-on]').count()) === 1) - && (await page.locator('[data-source="zzcatscenic"] [data-profile-on]').getAttribute('title')) === '数据剖析已开启:每次最多 80 条只读查询,单条不超过 15 秒') - // 再打开:剖析设置回显,没有摊进「高级连接参数」变成一串文字 - await page.locator('[data-source="zzcatscenic"]').getByRole('button', { name: '编辑', exact: true }).click() - await page.locator('[data-profile-settings="on"]').waitFor() - check('再打开编辑框:开关和填的数回显', await page.locator('#ds-profile-max_queries').inputValue() === '80' - && await page.locator('#ds-profile-query_timeout_s').inputValue() === '15') - const advanced = await page.locator('[data-profile-settings] ~ details, details').allInnerTexts() - check('……剖析设置没有摊进「高级连接参数」', !advanced.join(' ').includes('catalog_profile') && !advanced.join(' ').includes('[object Object]')) - await page.getByRole('button', { name: '取消', exact: true }).click() - - // 导入表格的源没有编辑框:卡片上单独设置,options 的其余键原样带上 - withProfile(state, S2, undefined) - state.sources = state.sources.map((s) => (s.id === S2 ? { ...s, options: { mask_columns: ['联系电话'] } } : s)) - await goto(page, '/data/tables') - await page.locator('[data-source="zzcatupload"] [data-edit-profile]').click() - const dialog = page.locator('[data-profile-settings-dialog]') - await dialog.waitFor() - check('导入表格的源:卡片上「设置数据剖析」打开只改这一项的弹窗', (await page.locator('[role="dialog"]').last().innerText()).includes('「zzcatupload」的数据剖析设置')) - check('……没改时保存禁用', await page.locator('[data-profile-save]').isDisabled()) - await dialog.locator('[data-profile-enabled]').check() - await page.locator('[data-profile-save]').click() - await until(async () => (await dialog.count()) === 0) - const body = writes(sent, /^PATCH \/datasources\/check-cat-upload$/).at(-1)?.body - check('……保存只换剖析设置,遮罩列原样带上', JSON.stringify(body) === JSON.stringify({ options: { mask_columns: ['联系电话'], catalog_profile: { enabled: true } } }), - JSON.stringify(body)) - check('……卡片上标「数据剖析已开启」', await until(async () => (await page.locator('[data-source="zzcatupload"] [data-profile-on]').count()) === 1)) - check('写请求只有保存数据源', writes(sent, /./).every((s) => /^PATCH \/datasources\/check-cat-/.test(s.key)), - JSON.stringify(writes(sent, /./).map((s) => s.key))) - check('没有原生对话框', natives.length === 0, natives.join(' | ')) - check('没有运行时报错', errors.length === 0, errors[0] ?? '') - await close() -}) - -await section('数据目录 · 数据剖析', async () => { - const state = freshState() - withProfile(state, S1, PROFILE_ON) - const replies = { profile: [] } - const { page, sent, errors, natives, close } = await open(`/data/catalog/${S1}`, { handlers: catalogHandlers(state, replies) }) - await page.locator('[data-catalog-row]').first().waitFor() - const head = page.locator('[data-catalog-profile]') - check('页头有「数据剖析」', (await head.getAttribute('aria-label')) === '数据剖析') - await head.click() - await profileBox(page).waitFor() - const setup = await profileBox(page).innerText() - check('没选表、没筛选:范围只有「使用次数最多、有待核实关联关系的表」,写明至多 10 张', - (await profileBox(page).locator('[data-profile-scope]').evaluateAll((els) => els.map((e) => e.getAttribute('data-profile-scope')))).join(',') === 'default' - && setup.includes('使用次数最多、有待核实关联关系的表') && setup.includes('至多 10 张')) - const budget = await profileBox(page).locator('[data-profile-budget]').innerText() - check('确认前写明预算:查询条数、单条和总时长、抽样规模、整表统计门槛,按哪个源的设置', - ['最多 60 条只读查询,单条不超过 10 秒', '总时长不超过 120 秒', '抽样 2,000 个键值', '行数超过 100,000 的表不做整表统计', '按「zzcatscenic」的数据剖析设置'] - .every((s) => budget.includes(s)), budget.replace(/\s+/g, ' ')) - check('……写明开始后无法中途停止', setup.includes('开始后无法中途停止')) - check('……还没开始就没有写请求', writes(sent, /./).length === 0) - if (SHOTS) await sleep(350) - await shot(page, 'catalog-profile-setup') - await page.keyboard.press('Escape') - await until(async () => (await profileBox(page).count()) === 0) - - // 打开入园记录的详情,选两张表剖析:完成后清单和正在看的详情都要重新载入 - await openDetail(page, S1, 'visits') - const v0 = await version(page) - check('剖析前:闸机关系是「推断 · 命名推断」,没有覆盖率', (await markStatus(page, `relations.${GATE_REL}`)) === 'proposed' - && (await page.locator(`[data-relation="${GATE_REL}"]`).innerText()).includes('—')) - await page.locator('[data-catalog-select="visits"]').click() - await page.locator('[data-catalog-select="orders"]').click() - await page.locator('[data-catalog-profile-selected]').click() - await profileBox(page).waitFor() - check('剖析所选:范围默认是「已选的 2 张表」', await profileBox(page).locator('[data-profile-scope="selected"] input').isChecked() - && (await profileBox(page).locator('[data-profile-scope="selected"]').innerText()).includes('已选的 2 张表')) - replies.profile.push(profileReply(state, { ms: 2600 })) - const lists0 = count(sent, /^GET \/datasources\/check-cat-scenic\/catalog$/) - await page.locator('[data-profile-start]').click() - check('进行中:写「正在剖析…」和已用时长、总时长上限', await until(async () => (await profilePhase(page)) === 'running') - && (await profileBox(page).locator('[data-profile-title]').innerText()) === '正在剖析…' - && (await profileBox(page).locator('[data-profile-elapsed]').innerText()).includes('(总时长上限 2 分 00 秒)')) - check('……时长在走(不是一直「不到 1 秒」)', await until(async () => Number(await profileBox(page).locator('[data-profile-elapsed]').getAttribute('data-profile-elapsed')) >= 1, 2500)) - check('……不给「停止」,明说无法中途停止', await profileBox(page).getByRole('button', { name: /停止|取消/ }).count() === 0 - && (await profileBox(page).locator('[data-profile-no-cancel]').innerText()).includes('无法中途停止')) - const req = writes(sent, /catalog\/profile$/).at(-1) - check('……请求带所选的两张表和本机署名', JSON.stringify([...(req?.body?.tables ?? [])].sort()) === JSON.stringify(['orders', 'visits']) - && decodeURIComponent(req?.actor ?? '') === ACTOR, JSON.stringify(req?.body)) - if (SHOTS) await sleep(350) - await shot(page, 'catalog-profile-running') - // 关掉窗口剖析照常进行:页头换成计时,完成后提示并可以打开报告 - await page.locator('[data-profile-background]').click() - check('「在后台继续」:窗口关掉,页头换成「正在剖析」', await until(async () => (await profileBox(page).count()) === 0) - && (await page.locator('[data-catalog-profile="running"]').innerText()).includes('正在剖析')) - const toastBox = page.locator('[data-toast]').filter({ hasText: '数据剖析已停止' }) - check('完成:提示「数据剖析已停止:查询次数已用完」,带「查看报告」', await until(async () => (await toastBox.count()) > 0, 6000), - await page.locator('[data-toast]').allInnerTexts().then((t) => t.join(' | ')).catch(() => '')) - check('……页头回到「数据剖析」', await page.locator('[data-catalog-profile="running"]').count() === 0) - check('……写进去的表重新载入:清单重取、正在看的详情版本更新', await until(async () => count(sent, /^GET \/datasources\/check-cat-scenic\/catalog$/) > lists0) - && await until(async () => (await version(page)) === v0 + 1)) - await toastBox.getByRole('button', { name: '查看报告' }).click() - await profileBox(page).waitFor() - const report = profileBox(page).locator('[data-profile-report]') - check('报告:停止原因说中文', (await report.locator('[data-profile-title]').innerText()) === '剖析已停止:查询次数已用完') - check('……用了多少条查询、剖析几张表', (await report.locator('[data-profile-summary]').innerText()).startsWith('用了 60 / 60 条查询,剖析 3 张表')) - check('……目录的变化', (await report.locator('[data-profile-changes]').innerText()) === '新增 1 项、更新 2 项') - check('……停下之后怎么办:调高查询次数上限或缩小范围,给「修改设置」', (await report.locator('[data-profile-stopped="budget"]').innerText()).includes('调高查询次数上限') - && await report.locator('[data-profile-stopped] [data-profile-settings-open]').count() === 1) - const visitsCard = report.locator('[data-profile-table="visits"]') - const gate = visitsCard.locator(`[data-profile-finding="relation"][data-path="relations.${GATE_REL}"]`) - check('关系核实:覆盖率、基数、抽样规模,升为已验证', (await gate.locator('[data-profile-coverage]').innerText()) === '覆盖率 98.7%' - && (await gate.locator('[data-profile-cardinality]').innerText()) === '多对一(已用数据核实)' && (await gate.innerText()).includes('抽样 300 个键值,对上 296 个') - && (await gate.locator('[data-profile-outcome]').innerText()) === '升为已验证' && (await gate.getAttribute('data-status')) === 'verified', - (await gate.innerText()).replace(/\s+/g, ' ')) - // 结论是状态说明,不是链接:带框的小标签,文字用次要文字色(不是强调蓝),不能点 - const outcomeLook = await gate.locator('[data-profile-outcome]').evaluate((el) => { - const probe = document.createElement('span') - probe.style.color = 'var(--accent)' - document.body.append(probe) - const accent = getComputedStyle(probe).color - probe.style.color = 'var(--text-dim)' - const dim = getComputedStyle(probe).color - probe.remove() - const cs = getComputedStyle(el) - return { tag: el.tagName, color: cs.color, accent, dim, border: cs.borderTopStyle, cursor: cs.cursor, kind: el.getAttribute('data-profile-outcome'), - clickable: !!el.closest('a, button') } - }) - check('……结论用状态标签的样子:带框、次要文字色,不是蓝色正文、不像链接', outcomeLook.kind === 'raised' && outcomeLook.color === outcomeLook.dim - && outcomeLook.color !== outcomeLook.accent && outcomeLook.border === 'solid' && outcomeLook.cursor !== 'pointer' && !outcomeLook.clickable, - JSON.stringify(outcomeLook)) - // 结构化的读数:基数核实读 cardinality_checked,几个含义待填写读 pending(假答复的 summary 里故意不写),空表读 empty_tables - check('……基数用数据核实过没有,读 cardinality_checked', (await gate.locator('[data-profile-cardinality]').getAttribute('data-checked')) === 'true' - && (await visitsCard.locator(`[data-path="relations.${MEMBER_REL}"] [data-profile-cardinality]`).innerText()) === '多对一(子表一侧未核实,按表结构推断)') - check('……挑表时跳过的空表读 empty_tables', (await report.locator('[data-profile-empty-tables="2"]').innerText().catch(() => '')) - === '挑表时跳过 2 张空表:ext_log_001、ext_log_002') - const member = visitsCard.locator(`[data-profile-finding="relation"][data-path="relations.${MEMBER_REL}"]`) - check('……覆盖率不够的保持推断,写明原因', (await member.locator('[data-profile-outcome]').innerText()) === '保持推断:覆盖率不足,可能不是这条关系' - && (await member.locator('[data-profile-coverage]').innerText()) === '覆盖率 62%') - const codes = visitsCard.locator('[data-profile-finding="codes"]') - check('码值候选:列、各取值的行数、几个含义待填写,给「填写含义」', (await codes.getAttribute('data-column')) === 'status' - && (await codes.innerText()).includes('3 个取值,共 182,000 行') && (await codes.innerText()).includes('171,080 行') - && (await codes.locator('[data-codes-pending]').innerText()) === '1 个含义待填写' && await codes.locator('[data-profile-fill="status"]').count() === 1, - (await codes.innerText()).replace(/\s+/g, ' ')) - const skipped = visitsCard.locator('[data-profile-skipped]') - check('跳过的项:原因说中文,附服务端的整句', (await skipped.innerText()).includes('列已遮罩') && (await skipped.innerText()).includes('visits.member_level 在数据源设置中被遮罩,不取值') - && (await skipped.innerText()).includes('表太大'), (await skipped.innerText()).replace(/\s+/g, ' ')) - check('……行数估算写成人话:统计信息约 182,000 行、数到上限的超过 100,000 行', - (await visitsCard.locator('header').innerText()).includes('约 182,000 行') && (await report.locator('[data-profile-table="orders"] header').innerText()).includes('超过 100,000 行')) - check('……一条没查的表写明「查询次数用完」', (await report.locator('[data-profile-table="order_items"] [data-profile-skip="budget"]').innerText()).includes('查询次数用完')) - const reportText = await report.innerText() - check('报告不露机读码', !PROFILE_CODE_RE.test(reportText), reportText.match(PROFILE_CODE_RE)?.[0] ?? '') - if (SHOTS) await sleep(350) - await shot(page, 'catalog-profile-report') - await page.locator('[data-profile-close]').click() - await until(async () => (await profileBox(page).count()) === 0) - - // 表详情:关系的来源写「数据剖析」,带覆盖率和基数;码值的含义空着写「含义待填写」 - const gateRow = page.locator(`[data-relation="${GATE_REL}"]`) - check('表详情:闸机关系「已验证 · 数据剖析」,覆盖率 98.7%、多对一', (await markStatus(page, `relations.${GATE_REL}`)) === 'verified' - && (await gateRow.innerText()).includes('数据剖析') && (await gateRow.innerText()).includes('98.7%') && (await gateRow.innerText()).includes('多对一'), - (await gateRow.innerText().catch(() => '')).replace(/\s+/g, ' ')) - await openMark(page, `relations.${GATE_REL}`) - check('……点开来源:写明抽样规模和覆盖率的剖析说明', (await panel(page).innerText()).includes('子表抽样 300 个不同键值,父表对上 296 个')) - // 说明里不带日期:时间看这一项的 updated_at(按本地时间,2026-10-03 03:00 UTC 是 11:00) - check('……说明不带日期,时间按这一项的 updated_at 写', !/\d{4}-\d{2}-\d{2}/.test(await panel(page).locator('.whitespace-pre-wrap').innerText()) - && /更新于 (2026\/)?10\/3 11:00/.test(await panel(page).innerText()), (await panel(page).innerText()).replace(/\s+/g, ' ')) - check('……关系表上写基数已用数据核实', await page.locator(`[data-relation="${GATE_REL}"] [data-relation-cardinality][data-checked="true"]`).count() === 1 - && (await page.locator(`[data-relation="${GATE_REL}"] [data-relation-cardinality]`).innerText()).includes('已核实')) - await page.keyboard.press('Escape') - const statusCodes = page.locator('[data-column="status"] [data-cell="codes"]') - check('……码值的含义空着写「含义待填写」,给「填写含义」', (await statusCodes.locator('[data-code="2"] [data-code-pending]').innerText()) === '含义待填写' - && (await statusCodes.locator('[data-codes-fill]').innerText()).includes('填写含义')) - // 从报告点「填写含义」:窗口关掉,打开这张表那一列的码值弹窗,光标在空着的那一个上 - await page.locator('[data-catalog-profile]').click() - await profileBox(page).waitFor() - check('页头再点:上一次的报告看过了就回到选范围', (await profilePhase(page)) === 'setup') - await page.keyboard.press('Escape') - await statusCodes.locator('[data-codes-fill]').click() - const codesBox = page.locator('[data-codes-dialog="status"]') - await codesBox.waitFor() - check('码值弹窗:每个码值一个输入框,空着的写「含义待填写」,光标在它上面', await codesBox.locator('input[data-code]').count() === 3 - && (await codesBox.locator('input[data-code="2"]').getAttribute('placeholder')) === '含义待填写' - && await until(async () => page.evaluate(() => document.activeElement?.getAttribute('data-code') === '2'))) - check('……写明剖析说明(各取值的占比),时间按这一项的 updated_at', (await codesBox.innerText()).includes('1(94%)、9(5%)、2(1%)') - && /^((2026\/)?10\/3 11:00)$/.test(await codesBox.locator('[data-codes-note-at]').innerText().catch(() => '')), - await codesBox.locator('[data-codes-note]').innerText().catch(() => '')) - check('……没填时保存禁用', await page.locator('[data-codes-save]').isDisabled()) - check('……写明当前已列出全部取值(剖析统计了全表),可以取消', await codesBox.locator('[data-codes-complete]').isChecked() - && (await codesBox.innerText()).includes('已列出全部取值') && (await statusCodes.locator('[data-codes-complete-mark]').count()) === 1) - await codesBox.locator('[data-codes-complete]').uncheck() - check('……只改勾选也能保存', !(await page.locator('[data-codes-save]').isDisabled())) - await codesBox.locator('[data-codes-complete]').check() - await codesBox.locator('input[data-code="2"]').fill('已退票') - if (SHOTS) await sleep(350) - await shot(page, 'catalog-codes-fill') - const v1 = await version(page) - await page.keyboard.press('Enter') - check('回车保存:整份提交,只换这一列码值的含义,带读到的版本', await until(async () => (await codesBox.count()) === 0) - && JSON.stringify(lastBody(sent, /^PUT .*catalog\/visits$/)?.notes?.columns?.status?.codes?.value) === JSON.stringify({ 1: '已入园', 9: '已作废', 2: '已退票' }) - && lastBody(sent, /^PUT .*catalog\/visits$/)?.notes?.columns?.status?.codes?.complete === true - && lastBody(sent, /^PUT .*catalog\/visits$/)?.if_version === v1, JSON.stringify(lastBody(sent, /^PUT .*catalog\/visits$/)?.notes?.columns?.status)) - check('……保存后不再有「含义待填写」,这一项记为人工确认', await until(async () => (await statusCodes.locator('[data-code-pending]').count()) === 0) - && (await statusCodes.innerText()).includes('已退票') && (await markStatus(page, 'columns.status.codes')) === 'confirmed') - // 报告里的「填写含义」直达 - await page.locator('[data-catalog-profile]').click() - replies.profile.push(profileReply(state, { extraCodes: true })) - await page.locator('[data-profile-start]').click() - await until(async () => (await profilePhase(page)) === 'done') - check('窗口开着时完成:直接看报告,不另弹提示', (await toastBox.count()) <= 1) - // 重新剖析:闸机关系上一次已经核实成已验证,这次照样核实——写「仍为已验证」,不说「升为已验证」 - const gateAgain = profileBox(page).locator(`[data-profile-finding="relation"][data-path="relations.${GATE_REL}"] [data-profile-outcome]`) - check('重新剖析:原本就是已验证的关系写「仍为已验证,覆盖率 98.7%」,不说「升为」', (await gateAgain.innerText()) === '仍为已验证,覆盖率 98.7%' - && (await gateAgain.getAttribute('data-profile-outcome')) === 'same', await gateAgain.innerText()) - const memberAgain = profileBox(page).locator(`[data-profile-finding="relation"][data-path="relations.${MEMBER_REL}"] [data-profile-outcome]`) - check('……原本就是推断、这次仍不够的写「保持推断」', (await memberAgain.innerText()) === '保持推断:覆盖率不足,可能不是这条关系' - && (await memberAgain.getAttribute('data-profile-outcome')) === 'kept') - // 点「填写含义」:报告收起、弹出那一列的码值;保存或取消之后回到报告,这一列改写成「含义已填写」,焦点落在下一列上 - const fillLine = (col) => profileBox(page).locator(`[data-profile-finding="codes"][data-column="${col}"]`) - await fillLine('status').locator('[data-profile-fill]').click() - const fillBox = page.locator('[data-codes-dialog="status"]') - check('报告里点「填写含义」:报告收起,弹出这一列的码值', await until(async () => (await fillBox.count()) === 1) - && (await profileBox(page).count()) === 0) - await fillBox.locator('input[data-code="2"]').fill('已退票') - await page.keyboard.press('Enter') - check('填完保存:回到报告(不用重新剖析)', await until(async () => (await profilePhase(page)) === 'done') && (await fillBox.count()) === 0 - && count(sent, /catalog\/profile$/) === 2, String(await profilePhase(page))) - const statusLine = fillLine('status') - check('……刚填的这一列写「含义已填写」,按钮改成「修改含义」', (await statusLine.locator('[data-codes-filled]').innerText().catch(() => '')).includes('含义已填写') - && await statusLine.locator('[data-codes-pending]').count() === 0 && (await statusLine.locator('[data-profile-fill]').innerText()) === '修改含义', - (await statusLine.innerText().catch(() => '')).replace(/\s+/g, ' ')) - check('……还没填的下一列照旧写待填写,焦点落在它的「填写含义」上', (await fillLine('ticket_type_id').locator('[data-codes-pending]').innerText()) === '2 个含义待填写' - && await until(async () => page.evaluate(() => document.activeElement?.getAttribute('data-profile-fill') === 'ticket_type_id'), 2000), - await page.evaluate(() => document.activeElement?.outerHTML?.slice(0, 120) ?? '')) - if (SHOTS) await sleep(350) - await shot(page, 'catalog-profile-report-filled') - await page.keyboard.press('Enter') - const nextBox = page.locator('[data-codes-dialog="ticket_type_id"]') - check('……回车接着填下一列', await until(async () => (await nextBox.count()) === 1)) - await page.keyboard.press('Escape') - check('码值弹窗取消:同样回到报告,那一列仍写待填写', await until(async () => (await profilePhase(page)) === 'done') && (await nextBox.count()) === 0 - && (await fillLine('ticket_type_id').locator('[data-codes-pending]').innerText()) === '2 个含义待填写') - await page.locator('[data-profile-close]').click() - await until(async () => (await profileBox(page).count()) === 0) - await openDetail(page, S1, 'orders') - await page.locator('[data-catalog-profile]').click() - await profileBox(page).waitFor() - check('换了一张表、再点页头:回到选范围(报告已看过)', (await profilePhase(page)) === 'setup') - await page.keyboard.press('Escape') - - check('写请求只有剖析和整份提交', writes(sent, /./).every((s) => /catalog\/profile$|^PUT .*catalog\/visits$/.test(s.key)), - JSON.stringify(writes(sent, /./).map((s) => s.key))) - check('没有原生对话框', natives.length === 0, natives.join(' | ')) - check('没有运行时报错', errors.length === 0, errors[0] ?? '') - await close() -}) - -await section('数据目录 · 数据剖析开始不了(409)', async () => { - const state = freshState() - const replies = { profile: [], patch: [] } - const { page, sent, errors, natives, close } = await open(`/data/catalog/${S1}`, { handlers: catalogHandlers(state, replies) }) - await page.locator('[data-catalog-row]').first().waitFor() - // 数据源列表里就没开:不让开始,就地开启 - await page.locator('[data-catalog-profile]').click() - await profileBox(page).waitFor() - check('没开启:写明需要先开启,「开始剖析」禁用,没有预算', (await profileBox(page).locator('[data-profile-disabled]').innerText()).includes('未开启数据剖析') - && await page.locator('[data-profile-start]').isDisabled() && await profileBox(page).locator('[data-profile-budget]').count() === 0) - await profileBox(page).locator('[data-profile-disabled] [data-profile-enable]').click() - const settings = page.locator('[data-profile-settings-dialog]') - await settings.waitFor() - await settings.locator('[data-profile-enabled]').check() - await page.locator(`#profile-${S1}-max_queries`).fill('40') - // 保存被拒:只认机读码和 field。没带机读码的(老服务端)不从原话里猜是哪一项,只弹通用提示,弹窗留着 - const QUERIES_REJECT = '数据剖析的「查询次数上限」需要填写 1 到 500 之间的整数;当前为「40」' - replies.patch.push(json({ detail: QUERIES_REJECT }, 422)) - await page.locator('[data-profile-save]').click() - check('422 没带机读码:不从原话里猜是哪一项,只弹通用提示,弹窗留着', - await until(async () => (await page.locator('[data-toast]').filter({ hasText: '查询次数上限' }).count()) > 0) - && await settings.locator('[aria-invalid="true"]').count() === 0 && await settings.count() === 1) - replies.patch.push(json({ detail: QUERIES_REJECT, code: 'profile_settings_invalid', field: 'max_queries' }, 422)) - await page.locator('[data-profile-save]').click() - check('422 带机读码和 field:报错落在「查询次数上限」那一格,光标放进去', - await until(async () => (await page.locator(`#profile-${S1}-max_queries`).getAttribute('aria-invalid')) === 'true') - && await until(async () => page.evaluate((id) => document.activeElement?.id === id, `profile-${S1}-max_queries`))) - replies.patch.push(json({ detail: '数据剖析设置中有无法识别的项「rows」,请删除后重试', code: 'profile_settings_invalid' }, 422)) - await page.locator('[data-profile-save]').click() - check('422 带机读码、没有 field:报错写在剖析设置这一节上', await until(async () => - (await settings.locator('[data-profile-error="section"]').innerText().catch(() => '')).includes('无法识别的项'))) - await page.locator('[data-profile-save]').click() - check('就地开启:保存后回到剖析,预算按刚存的设置', await until(async () => (await settings.count()) === 0) - && await until(async () => (await profileBox(page).locator('[data-profile-budget]').innerText().catch(() => '')).includes('最多 40 条只读查询')) - && !(await page.locator('[data-profile-start]').isDisabled())) - check('……保存的是这个源的剖析设置', JSON.stringify(writes(sent, /^PATCH /).at(-1)?.body) === JSON.stringify({ options: { catalog_profile: { enabled: true, max_queries: 40 } } }), - JSON.stringify(writes(sent, /^PATCH /).at(-1)?.body)) - - // 服务端的 409 都带机读码(api/coded.py),界面按它分五种情况,不从原话里认 - const cases = [ - ['disabled', '数据源「zzcatscenic」未开启数据剖析。剖析会对业务库发查询,请先在数据源设置中开启数据剖析', '未开启数据剖析', 'data-profile-enable', 'profile_disabled'], - ['inactive', '数据源「zzcatscenic」已停用,无法进行数据剖析。请先在数据源设置中启用', '数据源已停用', 'data-profile-go-source', 'datasource_inactive'], - ['noSchema', '尚未探查结构,无法进行数据剖析。请先在数据源卡片上点「探查结构」', '还没有表结构', 'data-profile-go-source', 'schema_missing'], - ['busy', '数据源「zzcatscenic」正在进行数据剖析,请等待完成后再试', '这个数据源正在剖析', 'data-profile-retry', 'profile_busy'], - ['tampered', '数据文件与登记的版本不一致,可能被修改过,已拒绝查询', '数据文件核对未通过', 'data-profile-go-source', 'snapshot_tampered'], - ] - // 没带机读码的 409(老服务端):原话里明明写着「正在进行数据剖析」,也不去猜,按「无法开始剖析」给重试 - replies.profile.push(json({ detail: cases[3][1] }, 409)) - await page.locator('[data-profile-start]').click() - check('409 没带机读码:不从原话里猜,按「无法开始剖析」说,给重试', await until(async () => - (await profileBox(page).locator('[data-profile-blocked="other"]').count()) === 1) - && await profileBox(page).locator('[data-profile-blocked="other"] [data-profile-retry]').count() === 1) - await page.locator('[data-profile-back]').click() - await until(async () => (await profilePhase(page)) === 'setup') - for (const [kind, detail, title, action, code] of cases) { - replies.profile.push(json({ detail, code }, 409)) - await page.locator('[data-profile-start]').click() - const blocked = profileBox(page).locator(`[data-profile-blocked="${kind}"]`) - const ok = await until(async () => (await blocked.count()) === 1) - const text = ok ? await blocked.innerText() : '' - check(`409(${title}):写明是哪种情况、服务端原话和下一步,给对应的按钮`, ok && text.includes(title) && text.includes(detail) - && await blocked.locator(`[${action}]`).count() === 1, text.replace(/\s+/g, ' ').slice(0, 140)) - if (kind === 'busy') { - // 稍后重试:同一个范围再发一次,这次成功 - replies.profile.push(profileReply(state)) - await blocked.locator('[data-profile-retry]').click() - check('……「重试」再发一次,成功后看到报告', await until(async () => (await profilePhase(page)) === 'done')) - } - if (kind === 'disabled') { - // 服务端说没开(列表里的设置过时了):同样就地开启,开了回到选范围 - await blocked.locator('[data-profile-enable]').click() - await settings.waitFor() - await page.locator(`#profile-${S1}-max_queries`).fill('45') - await page.locator('[data-profile-save]').click() - check('……就地开启后回到选范围', await until(async () => (await profilePhase(page)) === 'setup')) - continue - } - // 底栏的按钮不在弹窗正文里 - await page.locator(kind === 'busy' ? '[data-profile-again]' : '[data-profile-back]').click() - await until(async () => (await profilePhase(page)) === 'setup') - } - if (SHOTS) await sleep(350) - await shot(page, 'catalog-profile-blocked') - // 不是 409 的失败:照常的报错,能重试 - replies.profile.push(json({ detail: '服务端处理剖析时出错,请稍后重试' }, 500)) - await page.locator('[data-profile-start]').click() - check('其余失败:写「剖析未能完成」和原因,能重试', await until(async () => (await profileBox(page).locator('[data-profile-failed]').count()) === 1) - && (await profileBox(page).locator('[data-profile-failed]').innerText()).includes('剖析未能完成')) - // 去数据源:停用、没有表结构、快照被改动时 - replies.profile.push(json({ detail: cases[1][1], code: cases[1][4] }, 409)) - await profileBox(page).locator('[data-profile-failed]').getByRole('button', { name: /重试/ }).click() - await profileBox(page).locator('[data-profile-go-source]').click() - await page.waitForURL('**/data/databases', { timeout: 5000 }).catch(() => {}) - check('「前往数据源」回到数据源卡片', new URL(page.url()).pathname === '/data/databases', page.url()) - check('写请求只有剖析和保存剖析设置', writes(sent, /./).every((s) => /catalog\/profile$|^PATCH \/datasources\/check-cat-scenic$/.test(s.key)), - JSON.stringify(writes(sent, /./).map((s) => s.key))) - check('没有原生对话框', natives.length === 0, natives.join(' | ')) - check('没有运行时报错', errors.length === 0, errors[0] ?? '') - await close() -}) - await section('数据目录 · 窄屏', async () => { const state = freshState() const { page, errors, close } = await open(`/data/catalog/${S1}`, { handlers: catalogHandlers(state), viewport: { width: 390, height: 844 } }) From e3bd8e3d19339a4841598152b876fe4e4b548570 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Thu, 15 Oct 2026 18:38:52 +0800 Subject: [PATCH 3/4] =?UTF-8?q?=E8=A1=A8=E6=B8=85=E5=8D=95=E7=9A=84?= =?UTF-8?q?=E3=80=8C=E9=9C=80=E8=A6=81=E5=89=96=E6=9E=90=E3=80=8D=E5=BC=80?= =?UTF-8?q?=E5=85=B3=E5=8F=AA=E5=9C=A8=E6=9C=8D=E5=8A=A1=E7=AB=AF=E7=BB=99?= =?UTF-8?q?=E4=BA=86=20profile=5Fdue=20=E6=97=B6=E5=87=BA=E7=8E=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 老服务端(或检查脚本里没给新字段的假清单)每行没有 profile_due:开着剖析的源以前也会出一个「需要剖析 0」的开关,点了 就筛空清单,像是所有表都不用剖析。改成服务端给了这个口径才出(地址里已经带着 profile=due 时照样出,好让人关掉)。 --- frontend/src/pages/catalog/CatalogPage.tsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frontend/src/pages/catalog/CatalogPage.tsx b/frontend/src/pages/catalog/CatalogPage.tsx index c8b0b738..679c0643 100644 --- a/frontend/src/pages/catalog/CatalogPage.tsx +++ b/frontend/src/pages/catalog/CatalogPage.tsx @@ -176,6 +176,8 @@ export function CatalogPage() { const visible = useMemo(() => visibleRows(rows, query, filter, sort, usedOnly, dueOnly), [rows, query, filter, sort, usedOnly, dueOnly]) const counts = useMemo(() => filterCounts(rows, query, usedOnly, dueOnly), [rows, query, usedOnly, dueOnly]) const dueTotal = useMemo(() => dueCount(rows, query, usedOnly), [rows, query, usedOnly]) + /** 服务端给不给「需要剖析」的口径(老服务端每行没有 profile_due):不给就不出「需要剖析」开关 */ + const dueKnown = useMemo(() => rows.some((r) => r.profile_due !== undefined), [rows]) const base = `/data/catalog/${encodeURIComponent(sourceId)}` const openTable = useCallback((t: string) => navigate(`${base}/${encodeURIComponent(t)}${searchRef.current}`), [navigate, base]) const backToList = useCallback(() => navigate(`${base}${searchRef.current}`), [navigate, base]) @@ -528,7 +530,7 @@ export function CatalogPage() { profileOn={profileSettings.enabled} profileLive={profileLive} dueOnly={dueOnly} - onDueOnly={setDueOnly} + onDueOnly={dueKnown || dueOnly ? setDueOnly : undefined} dueTotal={dueTotal} /> From 6c728aaacd815764403602000cdead846294c484 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Thu, 15 Oct 2026 19:18:17 +0800 Subject: [PATCH 4/4] =?UTF-8?q?=E6=95=B0=E6=8D=AE=E5=89=96=E6=9E=90?= =?UTF-8?q?=E6=8C=89=E5=A4=8D=E6=A0=B8=E6=84=8F=E8=A7=81=E4=BF=AE=E4=B8=89?= =?UTF-8?q?=E5=A4=84=EF=BC=9A=E6=B2=A1=E6=9D=A5=E5=BE=97=E5=8F=8A=E6=94=B6?= =?UTF-8?q?=E5=B0=BE=E7=9A=84=E8=A1=A8=E7=85=A7=E5=AE=9E=E8=AF=B4=E3=80=81?= =?UTF-8?q?=E5=81=9C=E6=AD=A2=E4=B8=AD=E7=9A=84=E8=AF=B4=E6=B3=95=E3=80=81?= =?UTF-8?q?=E5=B1=95=E5=BC=80=E7=9A=84=E8=BF=9B=E8=A1=8C=E4=B8=AD=E4=B8=80?= =?UTF-8?q?=E8=A1=8C=E4=B8=8D=E8=B7=9F=E7=9D=80=E6=9F=A5=E8=AF=A2=E6=95=B0?= =?UTF-8?q?=E9=87=8D=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 服务重启的说明不再一律说「做完的项已写进数据目录」(R1 前端这一侧)。目录只在一张表结束时和它的结果一起写:停止、关停 服务时正在剖析的那张来得及收尾,做完的项照常写进去(服务端记部分完成,原因写「已做完的 N 项照常写进了数据目录」);进程 被杀、下次启动时才标中断的那张没来得及收尾,这次的结论一项都没写进去,服务端把它记成已停止、查完的项改记跳过。界面按逐表 的状态认出这样的表(已停止却有开始时刻:正常收尾的表不会是已停止,没开始的表没有开始时刻),服务重启、停止的说明里点名它 「剖析到一半服务就停了,没来得及收尾,这次的结论没有写进目录,保留原来的结论」,有别的表收了尾才说「其余做完的项已写进数据 目录」;逐表一行用服务端给这张表的原话(查完的几项没写进目录),一项都还没查完、原因就是任务那一句时写「没来得及收尾」。 不说「做完的项没写进去」:被杀时它可能一项都还没查完。 - 「正在停止」不再说等正在执行的那条查询结束(R4)。停止时服务端不等这条查询(它在发过的查询里记失败),库那边按单条时限截止 它,收尾写完停止就答;单条时限设得长时,旧说法会让人以为要等满时限。改成「正在执行的那条查询不再等待,由数据库按单条时限 截止;已做完的项照常写进目录…」,和服务端的说法一致;接口、停止逻辑的注释同步改。 - 展开进行中的那一行,只在一项检查开始、结束时重取这张表的完整结果(R5)。服务端只在这两个时刻写检查清单、正在做哪一项和发过 的查询,其间每发一条查询只加查询数和用时;以前重取的依据带着查询数,进行中几乎每次轮询(1.5 秒)都多取一份没变的结果, 一张 40 条查询的表约多 40 次。依据改成状态、结束的项数和正在做哪一项。 检查脚本 check-catalog-profile:伪造的停止照服务端分「来得及收尾」(正在剖析的那张记部分完成、原因写做完的已写进目录)和 「没来得及收尾」(记已停止、查完的项改记跳过、说法照服务端)两种;停止那一节跟着改(正在剖析的那张是部分完成,汇总 「完成 1,部分完成 1,已停止 3」),「正在停止」的说法改查新句子;服务重启一节加关停时收了尾、进程被杀(查完一项、一项都 没查完)三种;表详情一节加剖析到一半进程被杀;进行中一节加一条:展开进行中的一行后只多发查询时不再请求 tasks/{seq},一项 结束、换下一项时重取一次。先在改动前的代码上跑过,这几条照复核说的那样不通过(展开后多取了 2~3 次、停止中还是旧说法、 服务重启一律说已写进),改完通过。 --- frontend/src/api/client.ts | 4 +- frontend/src/lib/terms.ts | 27 +++- frontend/src/pages/catalog/ProfileDialog.tsx | 25 +++- frontend/src/pages/catalog/profileJob.ts | 3 +- scripts/check-catalog-profile.mjs | 150 ++++++++++++++++--- 5 files changed, 177 insertions(+), 32 deletions(-) diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index f6f12789..832089de 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -492,8 +492,8 @@ export const api = { get<{ task: CatalogProfileTaskDetail }>( `${catalogBase(sourceId)}/profile-jobs/${encodeURIComponent(jobId)}/tasks/${encodeURIComponent(String(seq))}`, opts), /** - * 停止:排队的直接停;进行中的那条查询由数据库按单条时限截止,这张表已做完的项照常写进目录、没做的保留旧结论, - * 后面的表记已停止。回停止之后的状态;已经结束的回 409(catalog_profile_finished) + * 停止:排队的直接停;进行中的不再等正在执行的那条查询(数据库按单条时限截止它),这张表已做完的项照常写进目录、 + * 没做的保留旧结论,后面的表记已停止。收尾写完才回停止之后的状态;已经结束的回 409(catalog_profile_finished) */ cancel: (sourceId: string, jobId: string) => post<{ job: CatalogProfileJob }>(`${catalogBase(sourceId)}/profile-jobs/${encodeURIComponent(jobId)}/cancel`), diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index e15408e5..340dd609 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -2769,6 +2769,8 @@ export const SCHEMA_PARTIAL_TEXT = { // - 「数据剖析」:对业务库发只读查询,核实推断的关联关系(覆盖率、基数),取状态类列的码值候选,提议业务日期。 // 动作写「剖析」,不写扫描、探测。默认关闭,按数据源开启。 // - 剖析是后台任务:逐张表进行,每张表剖析完就写进目录;查询一条一条地发,可以随时停止。关掉对话框照常进行。 +// 目录只在整张表结束时写一次:停止、关停服务时正在做的那张来得及收尾,做完的项照常写进去;进程被杀时那张表没来得及收尾, +// 这次做完的项一项都没写进去。说停下以后写进了什么,按逐表的状态照实说(cancelledNote / interruptedNote),不一律说「已写进」。 // - 预算按表算:「每张表最多查询数」「每张表时长上限」用完只停这张表,下一张表重新计数;另有单条查询时限、抽样键值数、 // 整表统计行数上限。标签和服务端校验报错里的叫法一致(catalog_profile._NUMBER_FIELDS),保存被拒时按机读码认出是哪一项。 // - 没有整次总量上限:任务自己在连续失败、多数超时时停下(止损),写明原因。 @@ -2801,6 +2803,9 @@ export const PROFILE_FIELD_HINT: Record = { max_scan_rows: '不超过此行数的表才做去重计数、最小值和最大值等整表统计', } +/** 表名点名:「visits」、「refunds」 */ +const quoteNames = (names: string[]) => names.map((n) => `「${n}」`).join('、') + export const PROFILE_TEXT = { // ---- 设置 section: '数据剖析', @@ -2888,8 +2893,9 @@ export const PROFILE_TEXT = { interrupted: '服务重启,剖析中断', } as Record, waitingDraft: '排在起草任务后面,起草结束后自动开始', - // 停止时正在执行的那条查询由数据库按单条时限截止,不是立刻停 - stopping: '正在停止:等正在执行的那条查询结束(不超过单条查询时限),已做完的项照常写进目录…', + // 停止时服务端不等正在执行的那条查询(它的结果记「失败」),库那边按单条时限截止它;收尾写完(至多十来秒)停止就答。 + // 别写成「等这条查询结束」:单条时限设得长时,界面会让人以为要等满时限 + stopping: '正在停止:正在执行的那条查询不再等待,由数据库按单条时限截止;已做完的项照常写进目录…', progress: (done: number, total: number) => `${formatNumber(done)} / ${formatNumber(total)} 张表`, /** 结束以后:做完了几张(停止、失败、跳过的不算) */ progressDone: (ok: number, total: number) => `完成 ${formatNumber(ok)} / ${formatNumber(total)} 张表`, @@ -2910,8 +2916,21 @@ export const PROFILE_TEXT = { failed: '后面的表没有剖析。请先在数据源卡片上测试连接、确认账号有只读权限,再重试没做完的表。', slow: '后面的表没有剖析。可以在业务库不忙的时候重试没做完的表,或在剖析设置中调低单条查询时限、每张表最多查询数。', } as Record, - cancelledNote: '做完的项已写进数据目录,没做的项保留原来的结论;后面的表没有剖析,可以重试没做完的表。', - interruptedNote: '服务重启时这次剖析还没有完成:做完的项已写进数据目录,没做的项保留原来的结论。不会自动续上,可以重试没做完的表。', + /** + * 停止、服务重启以后的说明。停下时正在剖析的那张表通常来得及收尾,做完的项照常写进目录(服务端记部分完成);lost 是没来得 + * 及收尾的表(进程被杀、下次启动时才标中断,或停止时服务端没在跑它):目录只在整张表结束时和它的结果一起写,所以它这次 + * 一项结论都没写进去,服务端把它记成已停止、查完的项也改记跳过。有这样的表时点名说,不能一律说「做完的项已写进数据目录」。 + * 它这次可能一项都还没查完,所以说「这次的结论没有写进目录」,不说「做完的项没写进去」。 + * written:有别的表收了尾(完成、部分完成),才说「其余做完的项已写进」 + */ + cancelledNote: (lost: string[], written: boolean) => (lost.length + ? `${quoteNames(lost)}剖析到一半停下,没来得及收尾,这次的结论没有写进目录,保留原来的结论${written ? ';其余做完的项已写进数据目录' : ''}。可以重试没做完的表。` + : '做完的项已写进数据目录,没做的项保留原来的结论;后面的表没有剖析,可以重试没做完的表。'), + interruptedNote: (lost: string[], written: boolean) => (lost.length + ? `服务重启时这次剖析还没有完成:${quoteNames(lost)}剖析到一半服务就停了,没来得及收尾,这次的结论没有写进目录,保留原来的结论${written ? ';其余做完的项已写进数据目录' : ''}。不会自动续上,可以重试没做完的表。` + : '服务重启时这次剖析还没有完成:做完的项已写进数据目录,没做的项保留原来的结论。不会自动续上,可以重试没做完的表。'), + /** 逐表一行:没来得及收尾的表(见 cancelledNote)。服务端给了这张表自己的原因(查完的几项没写进目录)时用原话 */ + notSaved: '剖析到一半中断,没来得及收尾,这次的结论没有写进目录,保留原来的结论', /** 结束后的汇总:「40 张表:完成 33,部分完成 4,跳过 2,失败 1」 */ counts: (total: number, c: { ok: number; partial: number; skipped: number; failed: number; cancelled: number }) => `${formatNumber(total)} 张表:${[ diff --git a/frontend/src/pages/catalog/ProfileDialog.tsx b/frontend/src/pages/catalog/ProfileDialog.tsx index 1b7ee42e..321ad9c3 100644 --- a/frontend/src/pages/catalog/ProfileDialog.tsx +++ b/frontend/src/pages/catalog/ProfileDialog.tsx @@ -455,6 +455,9 @@ function JobView({ sourceId, job, running, stopping, busy, filled, returnTo, onS const notice = running ? job.notice?.trim() ?? '' : '' const changed = job.added + job.updated + job.removed const findings = PT.findings(job.summary) + // 停下以后写进了什么按逐表照实说:没来得及收尾的表点名;收了尾的表(完成、部分完成)做完的项写进了目录 + const unsaved = job.tasks.filter(isUnsaved).map((t) => t.table_name) + const written = job.tasks.some((t) => t.status === 'ok' || t.status === 'partial') const [filter, setFilter] = useState('all') const radio = useRadioGroup(FILTERS, filter, setFilter) const tasks = running ? job.tasks : job.tasks.filter(FILTER_TEST[filter]) @@ -519,8 +522,8 @@ function JobView({ sourceId, job, running, stopping, busy, filled, returnTo, onS {!job.stop_reason && job.status === 'failed' && job.error && ( {job.error} )} - {!running && job.status === 'cancelled' &&

    {PT.cancelledNote}

    } - {job.status === 'interrupted' && {PT.interruptedNote}} + {!running && job.status === 'cancelled' &&

    {PT.cancelledNote(unsaved, written)}

    } + {job.status === 'interrupted' && {PT.interruptedNote(unsaved, written)}}
    {PT.tasks} @@ -571,14 +574,24 @@ export function checksText(t: Pick): st .filter(Boolean).join(' · ') } +/** + * 没来得及收尾的表:开始剖析了(有开始时刻),却以「已停止」结束。正常收尾的表状态按逐表算(完成、部分完成、失败、跳过), + * 不会是已停止;没开始就停下的表没有开始时刻。只有进程被杀(下次启动时才标中断)或停止时服务端没在跑它,正在做的那张才以 + * 已停止结束——目录只在整张表结束时和它的结果一起写,它这次一项结论都没写进去(服务端把查完的项改记跳过,这张表的原因 + * 写明查完的几项没写进目录;一项都还没查完时原因就是任务的那一句) + */ +export const isUnsaved = (t: Pick): boolean => t.status === 'cancelled' && !!t.started_at + /** * 一张表的结论:进行中写正在做哪一项;结束了写发现(核实了几条关系、取到几列码值……),没查成的写几项,整张表的原因照服务端 - * 的原话。jobError:任务整体的原因(止损、服务重启时服务端把后面的表也记上同一句),和它一样的不再逐表重复 + * 的原话。jobError:任务整体的原因(止损、服务重启时服务端把后面的表也记上同一句),和它一样的不再逐表重复。 + * 没来得及收尾的表写明这次的结论没写进目录(服务端给了自己的原因时用原话) */ export function taskNote(t: CatalogProfileTaskRow, jobError: string | null): string { if (t.status === 'pending') return '' if (t.status === 'running') return t.current ?? '' const own = t.error && t.error !== jobError ? t.error : '' + if (isUnsaved(t)) return own || PT.notSaved if (t.status === 'skipped' || t.status === 'failed') return own const parts = [PT.findings(t.summary, true), t.unfinished ? PT.unfinished(t.unfinished) : '', own] if (t.status === 'ok' && !parts.some(Boolean)) return PT.noFindings @@ -648,7 +661,9 @@ function TaskItem({ sourceId, job, task: t, open, onToggle, filled, focusAfter, } /** - * 展开的一张表:按 seq 取完整结果(检查清单、发现、发过的查询)。还在剖析的表,每多结束一项、多发一条查询就重取一次 + * 展开的一张表:按 seq 取完整结果(检查清单、发现、发过的查询)。还在剖析的表只在一项检查开始、结束时重取:服务端这时才 + * 写检查清单、正在做哪一项和发过的查询(开始时 current 换成这一项,结束时结束的项数加一、current 清空);其间每发一条 + * 查询只加查询数和用时,这两样逐表一行上就有,跟着它重取只会每次轮询都多取一份没变的结果 */ function TaskDetailLoader({ sourceId, jobId, task: t, filled, focusAfter, onOpenTable, onFillCodes }: { sourceId: string @@ -662,7 +677,7 @@ function TaskDetailLoader({ sourceId, jobId, task: t, filled, focusAfter, onOpen const [detail, setDetail] = useState(null) const [error, setError] = useState(null) const [again, setAgain] = useState(0) - const sig = `${t.status}:${t.checks_done}:${t.queries}` + const sig = `${t.status}:${t.checks_done}:${t.current ?? ''}` useEffect(() => { let alive = true api.dataCatalog.profileJobs.task(sourceId, jobId, t.seq).then(({ task }) => { diff --git a/frontend/src/pages/catalog/profileJob.ts b/frontend/src/pages/catalog/profileJob.ts index 23dd2ca7..839b8516 100644 --- a/frontend/src/pages/catalog/profileJob.ts +++ b/frontend/src/pages/catalog/profileJob.ts @@ -133,7 +133,8 @@ export function useProfileJob(sourceId: string, active: CatalogActiveProfileJob }, [accept]) /** - * 停止。服务端停下正在执行的那条查询(由数据库按单条时限截止)之前任务还是进行中:这期间 stopping 为这个任务。 + * 停止。停止接口等服务端收尾写完才答(不等正在执行的那条查询,它由数据库按单条时限截止),这期间任务还是进行中: + * stopping 为这个任务。 * 已经结束了(409)就取一次最新的 */ const cancel = useCallback(async () => { diff --git a/scripts/check-catalog-profile.mjs b/scripts/check-catalog-profile.mjs index 3dadac4f..1ba26afa 100644 --- a/scripts/check-catalog-profile.mjs +++ b/scripts/check-catalog-profile.mjs @@ -4,8 +4,9 @@ // 「开始前的范围和个数是不是按清单算的」「预算是不是按表写的、估了这次最多几条查询」「明说只读、一条一条发、可以停」 // 「同源有起草任务时是不是写排队」「进行中每张表到哪了(各类检查几项、发了几条查询、正在做哪一项)」「停得下来、停下来 // 写清做完的保留了什么」「展开一张表看得到检查清单和发过的查询(字面量换成了 ?)」「结束后能按有发现 / 有跳过 / 失败筛、 -// 只重试没做完的表」「止损、服务重启的原因照实写」「各种 409 各有下一步」「表清单的剖析一列和需要剖析的筛选」「表详情的一节 -// 和剖析这张表」「设置的两项新字段、被拒时报错落到那一项」「起草排在剖析后面时怎么说」。 +// 只重试没做完的表」「止损、服务重启的原因照实写(进程被杀时没来得及收尾的那张表点名,不说做完的已写进目录)」 +// 「各种 409 各有下一步」「表清单的剖析一列和需要剖析的筛选」「表详情的一节和剖析这张表」「设置的两项新字段、被拒时报错 +// 落到那一项」「起草排在剖析后面时怎么说」「展开进行中的表时只在一项检查开始、结束时重取,不跟着查询数每次轮询都取」。 // // 写法同 check-catalog:数据源列表、目录清单、单表目录、剖析任务、起草任务都用 page.route 伪造(虚构的网店业务库,表名 // orders、customers、visits……都是通用的假名字),任务怎么往前走由检查自己推(settle、finish、stopJob),页面每 1.5 秒 @@ -360,8 +361,16 @@ function finish(job, status) { job.started_at ??= job.finished_at delete job.stopAsked } -/** 停下(使用者停止、止损、服务重启):正在做的表写入已做完的项、其余项记原因;没开始的表记已停止,原因同一句 */ -function stopJob(job, { status = 'cancelled', reason = 'cancelled', current = null, rest = null, error = null } = {}) { +/** + * 停下(使用者停止、止损、服务重启)。正在做的那张表照服务端分两种: + * - 来得及收尾(saved,缺省:停止、止损、关停服务):写入已做完的项,其余项记原因,状态按逐表算(有项没查是部分完成), + * error 是 rest(服务端的「已做完的 N 项照常写进了数据目录,剩下 M 项没查」); + * - 没来得及收尾(saved: false:进程被杀,下次启动时才标中断):结论一项都没写进目录,表记已停止,查完的项也改记跳过, + * 说法照服务端(catalog_profile_jobs._close_tasks):查完的项「{原因},查完了但结论没有写进数据目录」,表的原因 + * 「{原因}:已查完的 N 项没有写进数据目录,剩下 M 项没查」;一项都还没查完时表的原因就是 error。 + * 没开始的表记已停止,原因同一句 + */ +function stopJob(job, { status = 'cancelled', reason = 'cancelled', current = null, rest = null, error = null, saved = true } = {}) { for (const t of job.tasks) { if (t.status === 'running') { const left = t.checks.filter((c) => c.state === 'pending' || c.state === 'running') @@ -369,7 +378,17 @@ function stopJob(job, { status = 'cancelled', reason = 'cancelled', current = nu Object.assign(c, { state: 'skipped', reason, detail: current ?? '已停止,这一项没有查' }) t.skipped.push({ kind: c.kind, target: c.target, path: c.path, reason, detail: c.detail }) } - Object.assign(t, { status: 'cancelled', current: null, finished_at: isoNow(), error: rest ?? error }) + if (saved) { + Object.assign(t, { status: left.length ? 'partial' : 'ok', current: null, finished_at: isoNow(), error: rest ?? error }) + continue + } + const done = t.checks.filter((x) => x.state === 'done') + for (const c of done) { + Object.assign(c, { state: 'skipped', reason, detail: `${error},查完了但结论没有写进数据目录` }) + t.skipped.push({ kind: c.kind, target: c.target, path: c.path, reason, detail: c.detail }) + } + const own = done.length ? `${error}:已查完的 ${done.length} 项没有写进数据目录${left.length ? `,剩下 ${left.length} 项没查` : ''}` : error + Object.assign(t, { status: 'cancelled', current: null, finished_at: isoNow(), error: own }) } else if (t.status === 'pending') { Object.assign(t, { status: 'cancelled', finished_at: isoNow(), error: error ?? '已停止,这张表没有剖析' }) } @@ -830,6 +849,30 @@ await section('剖析 · 进行中:逐表进度、后台继续与停止', asyn && (await inBox(page, '[data-profile-queries]')).includes('已发 23 条查询') && (await inBox(page, '[data-profile-elapsed]')).startsWith('· 用时'), await textOf(box(page).locator('[role="status"]'))) check('……标题写「正在剖析…」', (await inBox(page, '[data-profile-title]')) === '正在剖析…') + // 展开进行中的那一行:服务端只在一项检查开始、结束时写检查清单和发过的查询,其间每发一条查询只加查询数。所以只多了 + // 查询数时不重取这张表的完整结果,一项结束了、换了一项在做才重取(开发模式下 StrictMode 挂载时会多取一次,按展开后的 + // 次数为准) + const detailGets = () => count(sent, new RegExp(`^GET .*profile-jobs/${job.id}/tasks/1$`)) + const jobPolls = () => count(sent, new RegExp(`^GET .*profile-jobs/${job.id}$`)) + const ordersTask = job.tasks.find((t) => t.table_name === 'orders') + await taskLine(page, 'orders').locator('[data-profile-task-toggle]').click() + const live = box(page).locator('[data-profile-task-detail="orders"]') + check('进行中展开一行:取这张表的完整结果,正在做的那一项写「进行中」', await until(async () => (await live.locator('[data-profile-checks]').count()) === 1) + && detailGets() >= 1 && (await live.locator('[data-profile-check][data-state="running"]').count()) === 1, `${detailGets()} 次`) + await sleep(300) + const gets0 = detailGets() + for (const n of [25, 27]) { + const p0 = jobPolls() + Object.assign(ordersTask, { queries: n, elapsed_ms: ordersTask.elapsed_ms + 1500 }) + await until(async () => jobPolls() >= p0 + 2, 5000) + } + check('……这一项没结束、只多发了几条查询:逐表一行跟着走,不重取这张表的完整结果', (await inBox(page, '[data-profile-task="orders"] [data-profile-task-queries]')) === '27 条查询' + && detailGets() === gets0, `展开后 ${gets0} 次 → ${detailGets()} 次`) + settle(state, job, 'orders', { status: 'running', done: { relation: 4, codes: 5, date: 1 }, queries: 29, current: '正在取 orders.source 的码值候选' }) + check('……这一项结束、换了下一项:重取一次,检查清单跟上', await until(async () => detailGets() === gets0 + 1 + && (await live.locator('[data-profile-check="relation"][data-state="done"]').count()) === 4), `展开后 ${gets0} 次 → ${detailGets()} 次`) + await taskLine(page, 'orders').locator('[data-profile-task-toggle]').click() + await until(async () => (await live.count()) === 0) settle(state, job, 'orders', { queries: 31, elapsed_ms: 26000, added: 3, updated: 2, row_estimate: { rows: 250000, method: 'stats', at_least: null }, findings: [ @@ -883,21 +926,25 @@ await section('剖析 · 进行中:逐表进度、后台继续与停止', asyn check('点页头:回到对话框,直接是进行中的任务,提醒写在进度条下面', await until(async () => (await phase(page)) === 'running') && (await inBox(page, '[data-profile-notice]')) === NOTICE && await box(page).locator('[data-profile-scope]').count() === 0) - // 停止:调停止接口;正在执行的那条查询由数据库按单条时限截止,这期间写「正在停止」 + // 停止:调停止接口。服务端不再等正在执行的那条查询(数据库按单条时限截止它),收尾写完就答,这期间写「正在停止」 await page.locator('[data-profile-stop]').click() - check('点「停止」:调停止接口,写「正在停止:等正在执行的那条查询结束…」,停止按钮不能再点', await until(async () => - (await inBox(page, '[data-profile-title]')).startsWith('正在停止:等正在执行的那条查询结束')) + check('点「停止」:调停止接口,写「正在停止:正在执行的那条查询不再等待,由数据库按单条时限截止…」,停止按钮不能再点', await until(async () => + (await inBox(page, '[data-profile-title]')).startsWith('正在停止:正在执行的那条查询不再等待,由数据库按单条时限截止')) + && !(await inBox(page, '[data-profile-title]')).includes('等正在执行的那条查询结束') && count(sent, new RegExp(`^POST .*profile-jobs/${job.id}/cancel$`)) === 1 && await page.locator('[data-profile-stop]').isDisabled(), await inBox(page, '[data-profile-title]')) - stopJob(job, { rest: '已停止:做完的 1 项已写进目录,其余 3 项保留原来的结论' }) + // 服务端:正在剖析的访问记录来得及收尾,做完的 1 项写进目录、剩下 3 项没查(部分完成),后面的表已停止 + stopJob(job, { rest: '剖析已停止:已做完的 1 项照常写进了数据目录,剩下 3 项没查' }) check('……停下来:标题写「已停止」,写明做完的已写进目录、没做的保留原来的结论', await until(async () => (await phase(page)) === 'done') - && (await inBox(page, '[data-profile-title]')) === '已停止' && (await inBox(page, '[data-profile-cancelled]')).includes('没做的项保留原来的结论'), + && (await inBox(page, '[data-profile-title]')) === '已停止' && (await inBox(page, '[data-profile-cancelled]')).includes('没做的项保留原来的结论') + && !(await inBox(page, '[data-profile-cancelled]')).includes('没来得及'), `${await inBox(page, '[data-profile-title]')} · ${await inBox(page, '[data-profile-cancelled]')}`) - check('……正在剖析的那张写已停止和服务端的原因;后面的表都记已停止', (await inBox(page, '[data-profile-task="visits"] [data-profile-task-status]')) === '已停止' - && (await inBox(page, '[data-profile-task="visits"] [data-profile-task-note]')).includes('其余 3 项保留原来的结论') + check('……正在剖析的那张写部分完成和服务端的原因;后面的表都记已停止', (await inBox(page, '[data-profile-task="visits"] [data-profile-task-status]')) === '部分完成' + && (await inBox(page, '[data-profile-task="visits"] [data-profile-task-note]')).includes('已做完的 1 项照常写进了数据目录') && (await inBox(page, '[data-profile-task="tmp_log"] [data-profile-task-status]')) === '已停止' - && await taskLine(page, 'tmp_log').locator('[data-profile-task-checks]').count() === 0) - check('……汇总照实写:5 张表,完成 1,已停止 4', (await inBox(page, '[data-profile-counts]')) === '5 张表:完成 1,已停止 4', await inBox(page, '[data-profile-counts]')) + && await taskLine(page, 'tmp_log').locator('[data-profile-task-checks]').count() === 0, await inBox(page, '[data-profile-task="visits"]')) + check('……汇总照实写:5 张表,完成 1,部分完成 1,已停止 3', (await inBox(page, '[data-profile-counts]')) === '5 张表:完成 1,部分完成 1,已停止 3', + await inBox(page, '[data-profile-counts]')) check('……可以重试没做完的 4 张', (await textOf(page.locator('[data-profile-retry]'))).includes('重试没做完的表(4 张)')) check('……页头回到「数据剖析」', await until(async () => (await head(page).getAttribute('data-catalog-profile')) === '')) const polls = () => count(sent, new RegExp(`^GET .*profile-jobs/${job.id}$`)) @@ -1109,16 +1156,67 @@ await section('剖析 · 止损与服务重启', async () => { && (await inBox(page, '[data-profile-stop-error]')) === FAIL && (await inBox(page, '[data-profile-stopped="failed"]')).includes('测试连接')) await page.locator('[data-profile-close]').click() - // 服务重启:任务标中断,不自动续跑 + // 服务重启:任务标中断,不自动续跑。关停服务时正在剖析的那张表来得及收尾(做完的项写进目录),后面的表已停止 + const RESTART = '服务重启,剖析中断' await openDialog(page) await page.locator('[data-profile-start]').click() await until(async () => (await phase(page)) === 'running') const job3 = state.pjobs.at(-1) - settle(state, job3, job3.tasks[0].table_name, { queries: 5 }) - stopJob(job3, { status: 'interrupted', error: '服务重启,剖析中断' }) - check('服务重启:标题写「服务重启,剖析中断」,写明不会自动续上、可以重试没做完的表', await until(async () => (await phase(page)) === 'done') - && (await inBox(page, '[data-profile-title]')) === '服务重启,剖析中断' && (await inBox(page, '[data-profile-interrupted]')).includes('不会自动续上')) - check('……和任务相同的原因不逐表重复', await box(page).locator('[data-profile-task-note]').filter({ hasText: '服务重启' }).count() === 0) + const [first3, second3] = job3.tasks.map((t) => t.table_name) + settle(state, job3, first3, { queries: 5 }) + settle(state, job3, second3, { status: 'running', done: { relation: 1 }, queries: 4 }) + const left3 = job3.tasks[1].checks.filter((c) => c.state !== 'done').length + stopJob(job3, { status: 'interrupted', error: RESTART, rest: `${RESTART}:已做完的 1 项照常写进了数据目录,剩下 ${left3} 项没查` }) + check('服务重启(关停时收了尾):标题写「服务重启,剖析中断」,写明做完的项已写进目录、不会自动续上、可以重试', await until(async () => (await phase(page)) === 'done') + && (await inBox(page, '[data-profile-title]')) === RESTART && (await inBox(page, '[data-profile-interrupted]')).includes('不会自动续上') + && (await inBox(page, '[data-profile-interrupted]')).includes('做完的项已写进数据目录') + && !(await inBox(page, '[data-profile-interrupted]')).includes('没来得及'), await inBox(page, '[data-profile-interrupted]')) + check('……正在剖析的那张:部分完成,写服务端的原因(做完的写进了目录)', (await inBox(page, `[data-profile-task="${second3}"] [data-profile-task-status]`)) === '部分完成' + && (await inBox(page, `[data-profile-task="${second3}"] [data-profile-task-note]`)).includes('已做完的 1 项照常写进了数据目录'), + await inBox(page, `[data-profile-task="${second3}"]`)) + check('……和任务相同的原因不逐表重复(只有收了尾的那张写它自己的一句)', await box(page).locator('[data-profile-task-note]').filter({ hasText: RESTART }).count() === 1) + await page.locator('[data-profile-close]').click() + + // 进程被杀:下次启动时才标中断,正在剖析的那张表没来得及收尾,这次的结论一项都没写进目录(服务端把查完的项改记跳过)。 + // 不能说「做完的项已写进数据目录」:点名这张表,写明它这次的结论没写进去 + await openDialog(page) + await page.locator('[data-profile-start]').click() + await until(async () => (await phase(page)) === 'running') + const job4 = state.pjobs.at(-1) + const [first4, lost4] = job4.tasks.map((t) => t.table_name) + settle(state, job4, first4, { queries: 5 }) + settle(state, job4, lost4, { status: 'running', done: { relation: 1 }, queries: 4 }) + stopJob(job4, { status: 'interrupted', error: RESTART, current: `${RESTART},未检查`, saved: false }) + const note = () => inBox(page, '[data-profile-interrupted]') + check('服务重启(进程被杀、没来得及收尾):点名剖析到一半的那张表,写明它这次的结论没写进目录、保留原来的结论', await until(async () => (await phase(page)) === 'done') + && (await note()).includes(`「${lost4}」剖析到一半服务就停了`) && (await note()).includes('这次的结论没有写进目录') && (await note()).includes('保留原来的结论') + && (await note()).includes('其余做完的项已写进数据目录') && (await note()).includes('不会自动续上'), await note()) + check('……不再一律说「做完的项已写进数据目录」', !(await note()).startsWith('服务重启时这次剖析还没有完成:做完的项已写进数据目录'), await note()) + check('……那张表记已停止,这一行照服务端写明查完的项没写进目录;前面剖析完的表照常', (await inBox(page, `[data-profile-task="${lost4}"] [data-profile-task-status]`)) === '已停止' + && (await inBox(page, `[data-profile-task="${lost4}"] [data-profile-task-note]`)) === `${RESTART}:已查完的 1 项没有写进数据目录,剩下 1 项没查` + && (await inBox(page, `[data-profile-task="${first4}"] [data-profile-task-status]`)) === '完成', await inBox(page, `[data-profile-task="${lost4}"]`)) + await taskLine(page, lost4).locator('[data-profile-task-toggle]').click() + const lostBox = box(page).locator(`[data-profile-task-detail="${lost4}"]`) + check('……展开:检查清单没有标「完成」的项,查完的那项写明结论没写进目录', await until(async () => (await lostBox.locator('[data-profile-checks]').count()) === 1) + && await lostBox.locator('[data-profile-check][data-state="done"]').count() === 0 + && (await textOf(lostBox.locator('[data-profile-check="relation"]'))).includes('查完了但结论没有写进数据目录'), await textOf(lostBox.locator('[data-profile-checks]'))) + check('……没开始的表不写这一句', await box(page).locator('[data-profile-task-note]').filter({ hasText: '写进' }).count() === 1) + await shot(page, 'profile-interrupted') + await page.locator('[data-profile-close]').click() + + // 进程被杀时那张表一项都还没查完(还在估行数):服务端给它的原因就是任务的那一句,这一行改写「没来得及收尾」; + // 也没有别的表收了尾,不说「其余做完的项已写进」 + await openDialog(page) + await page.locator('[data-profile-start]').click() + await until(async () => (await phase(page)) === 'running') + const job5 = state.pjobs.at(-1) + const lost5 = job5.tasks[0].table_name + settle(state, job5, lost5, { status: 'running', queries: 1, current: `正在估算 ${lost5} 的行数` }) + stopJob(job5, { status: 'interrupted', error: RESTART, current: `${RESTART},未检查`, saved: false }) + check('……一项都还没查完就被杀:这一行写「没来得及收尾,这次的结论没有写进目录」,说明里不提「其余做完的项」', await until(async () => (await phase(page)) === 'done') + && (await inBox(page, `[data-profile-task="${lost5}"] [data-profile-task-note]`)) === '剖析到一半中断,没来得及收尾,这次的结论没有写进目录,保留原来的结论' + && (await note()).includes(`「${lost5}」剖析到一半服务就停了`) && !(await note()).includes('其余做完的项'), + `${await inBox(page, `[data-profile-task="${lost5}"]`)} · ${await note()}`) check('没有运行时报错', errors.length === 0, errors[0] ?? '') await close() }) @@ -1378,6 +1476,18 @@ await section('剖析 · 表详情的「数据剖析」一节', async () => { await page.locator('[data-catalog-row="customers"] [data-catalog-open-row]').click() check('剖析完:这一节换成这次的结果(行数更新),发起人是本机', await until(async () => (await textOf(sec.locator('[data-profile-rows]'))).includes('约 12,500 行')) && (await textOf(sec.locator('[data-catalog-profile-last]'))).includes(`${ACTOR} 发起`)) + // 再剖析一次,剖析到一半进程被杀(下次启动时才标中断):这张表没来得及收尾,这一节照服务端写明查完的项没写进目录, + // 跳过的项里那一项写明结论没写进去 + await sec.locator('[data-catalog-profile-table]').click() + await until(async () => state.pjobs.at(-1) !== job) + const killed = state.pjobs.at(-1) + settle(state, killed, 'customers', { status: 'running', done: { codes: 1 }, queries: 2 }) + await until(async () => (await sec.locator('[data-catalog-profile-live="running"]').count()) === 1) + stopJob(killed, { status: 'interrupted', error: '服务重启,剖析中断', current: '服务重启,剖析中断,未检查', saved: false }) + check('剖析到一半进程被杀:这一节写已停止和服务端的原因(查完的项没写进目录),跳过的项写明结论没写进去', await until(async () => + (await textOf(sec.locator('[data-profile-table-error]'))) === '服务重启,剖析中断:已查完的 1 项没有写进数据目录,剩下 1 项没查') + && (await textOf(sec.locator('[data-catalog-profile-last]'))).includes('已停止') + && (await textOf(sec)).includes('查完了但结论没有写进数据目录') && !(await textOf(sec)).includes('照常写进'), await textOf(sec)) // 没剖析过、视图、没有可剖析项 await page.locator('[data-catalog-row="orders"] [data-catalog-open-row]').click() check('从没剖析过的表:写「还没有剖析过」,给「剖析这张表」', await until(async () => (await textOf(sec.locator('[data-catalog-profile-never]'))) === '这张表还没有剖析过。')