From aa3bd65cfc7b490bf02109bb16fb309f2d1f7671 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Fri, 16 Oct 2026 19:46:14 +0800 Subject: [PATCH] =?UTF-8?q?W4-d=20=E5=89=8D=E7=AB=AF=EF=BC=9A=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E7=9B=AE=E5=BD=95=E9=A1=B5=E7=9A=84=E3=80=8C=E5=8F=98?= =?UTF-8?q?=E5=8C=96=E3=80=8D=E8=A7=86=E5=9B=BE=E3=80=81=E8=A1=A8=E8=AF=A6?= =?UTF-8?q?=E6=83=85=E3=80=8C=E7=BB=93=E6=9E=84=E4=B8=8E=E5=8F=96=E5=80=BC?= =?UTF-8?q?=E5=8F=98=E5=8C=96=E3=80=8D=E4=B8=80=E8=8A=82=E3=80=81=E5=90=8E?= =?UTF-8?q?=E5=8F=B0=E4=BB=BB=E5=8A=A1=E6=9D=A1=E3=80=81=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E6=BA=90=E7=9A=84=E8=A7=92=E6=A0=87=E5=92=8C=E5=AE=9A=E6=97=B6?= =?UTF-8?q?=E7=BB=B4=E6=8A=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 第 3 波的结构变化、取值观察、变化记录、定时维护都只有接口，这一提交给它们界面，能走通（收口）： - 数据目录页左栏加第三个视图「变化」（view=changes）：按要处理 / 提醒 / 说明分组（组内照服务端排好的先后），能按开着的 / 处理过的、结构变化 / 取值变化筛，筛选和打开的那一条记在地址里；视图标签上写几条要处理。点开一条，右栏写服务端的一句说法、 依据（谁、经什么测到的；取值变化写观察口径，结构变化写比对的两版结构）、前后各是什么、影响面（说到这些列的目录项和它们 现在的状态、写死了这张表的已发布模板、「这条 SQL 现在会报列不存在」、有效记录条件）、能做的处理。动作和做不了的原因都照 服务端给的 actions 画；清理目录项先确认，加入码值可以先填含义，误报必须写理由（写之前按钮不能点，服务端也会拒）。处理完 照实写结果：谁、什么时候、目录第几版、建了的任务（点得进起草、剖析的进度）。 - 表详情加「结构与取值变化」一节：这张表的变化（点「去处理」到「变化」视图里那一条）、撤销过的「已列全」、最近几次取值观察 （行数、码值分布、最新业务日期、按日行数）和各自的口径，口径换过的说一句不互相比较；按日行数的小柱图只给眼睛看，数字写在 字里和读屏名称里。 - 后台任务条（05 (g)）：页头的「起草」「数据剖析」改回纯入口（进行中转圈、排队中时钟，读屏名称写状态、不写数字）；DraftStrip 换成 JobStrip，按服务端给的同源队先后列出探查、起草、剖析，每项一行加进度条和提醒（剖析的「最近查询多数超时」也挪到这里， 窄屏、触屏看得见全文），「查看」打开对应的对话框，探查的到数据源卡片上（#source- 滚过去）。 - 数据源卡片：要处理的变化条数的角标（读屏写全，点开到「变化」视图）；「定时维护」对话框（和「探查范围」一样是卡片上的 独立对话框，不塞进编辑表单：两个保存混在一起出错时说不清哪一样没存上）：开关缺省关、开启时提示选低峰时段，选步骤（PUT /maintenance），频率用 components/FreqPicker，保存成一条 target_kind=source_maintenance 的定时任务（新建 ScheduleTargetCreate， 改频率、开关 SchedulePatch 带版本）；照实写下一次什么时候、最近一次建了哪些任务、哪几步没执行和原因（触发记录的 inputs）， 剖析没开、调度没开时说到点会怎样。上传表格、记录表不给这一节。开着的维护在卡片上带一个标签写频率。 - 地址里带 pjob / djob 时打开那个剖析、起草任务的进度（定时维护的「最近一次」从数据源页链过来）。 新文件：typesSource.ts（接口约定）、api/sourceChanges.ts、lib/termsSource.ts（terms.ts 末尾转出）、pages/catalog/changes.ts、 ChangesQueue.tsx、ChangeDetail.tsx、TableChanges.tsx、JobStrip.tsx、pages/SourceMaintenanceDialog.tsx。变化的叫法和服务端逐字 一致，tests/test_source_change_terms.py 守着（维护步骤的叫法同样）。观察的按日行数服务端补上窗口里没数据的日子（不然小柱图 看不出断档）。 --- backend/app/data/catalog_profile_observe.py | 11 +- backend/tests/test_source_change_terms.py | 44 ++ backend/tests/test_source_upkeep.py | 20 + frontend/src/api/client.ts | 3 + frontend/src/api/sourceChanges.ts | 41 ++ frontend/src/lib/terms.ts | 2 + frontend/src/lib/termsSource.ts | 263 ++++++++++ frontend/src/pages/DataSourcesTab.tsx | 63 ++- .../src/pages/SourceMaintenanceDialog.tsx | 297 ++++++++++++ frontend/src/pages/catalog/CatalogPage.tsx | 274 +++++++---- frontend/src/pages/catalog/ChangeDetail.tsx | 450 ++++++++++++++++++ frontend/src/pages/catalog/ChangesQueue.tsx | 186 ++++++++ frontend/src/pages/catalog/JobStrip.tsx | 116 +++++ frontend/src/pages/catalog/TableChanges.tsx | 188 ++++++++ frontend/src/pages/catalog/TableDetail.tsx | 9 +- frontend/src/pages/catalog/changes.ts | 72 +++ frontend/src/typesSource.ts | 232 +++++++++ 17 files changed, 2165 insertions(+), 106 deletions(-) create mode 100644 backend/tests/test_source_change_terms.py create mode 100644 frontend/src/api/sourceChanges.ts create mode 100644 frontend/src/lib/termsSource.ts create mode 100644 frontend/src/pages/SourceMaintenanceDialog.tsx create mode 100644 frontend/src/pages/catalog/ChangeDetail.tsx create mode 100644 frontend/src/pages/catalog/ChangesQueue.tsx create mode 100644 frontend/src/pages/catalog/JobStrip.tsx create mode 100644 frontend/src/pages/catalog/TableChanges.tsx create mode 100644 frontend/src/pages/catalog/changes.ts create mode 100644 frontend/src/typesSource.ts diff --git a/backend/app/data/catalog_profile_observe.py b/backend/app/data/catalog_profile_observe.py index 60cee5ef..78b07a31 100644 --- a/backend/app/data/catalog_profile_observe.py +++ b/backend/app/data/catalog_profile_observe.py @@ -556,7 +556,16 @@ def _shown(kind: str, data: Mapping[str, Any]) -> dict[str, Any]: "truncated": bool(data.get("truncated"))} if kind == "daily_volume": days = data.get("days") if isinstance(data.get("days"), Mapping) else {} - series = sorted((str(d), int(n)) for d, n in days.items() if isinstance(n, (int, float))) + counts = {str(d)[:10]: int(n) for d, n in days.items() if isinstance(n, (int, float))} + # 窗口里没数据的日子补 0：只列有数据的日子，断了的那几天在图上就看不出来 + try: + first, last = date.fromisoformat(str(data.get("since"))[:10]), date.fromisoformat(str(data.get("until"))[:10]) + except ValueError: + first = last = None + if first and last and 0 <= (last - first).days <= WINDOW_DAYS + 7: + for i in range((last - first).days + 1): + counts.setdefault((first + timedelta(days=i)).isoformat(), 0) + series = sorted(counts.items()) return {"since": data.get("since"), "until": data.get("until"), "days": [list(p) for p in series], "total": sum(n for _, n in series), "days_with_data": sum(1 for _, n in series if n > 0)} if kind == "freshness": diff --git a/backend/tests/test_source_change_terms.py b/backend/tests/test_source_change_terms.py new file mode 100644 index 00000000..6b8b2885 --- /dev/null +++ b/backend/tests/test_source_change_terms.py @@ -0,0 +1,44 @@ +"""变化的叫法前后端一致（总方案 X10：一种说法）。 + +服务端给人看的一句（source_changes.summary）用 KIND_LABEL，前端「变化」标签页、表详情的那一节用 lib/termsSource.ts 的 +SOURCE_CHANGE_KIND_LABEL：两边逐字对照，一边改了另一边没跟上，同一条变化在接口和界面上就是两种说法。写法同 +test_notify_events.py 对照通知事件的叫法。 +""" +from __future__ import annotations + +import re +from pathlib import Path + +import app.db.models # noqa: F401 - 模型都登记上（source_objects 的外键指向 data_sources） +from app.data import source_changes as sc +from app.data import source_maintenance as sm + +TERMS = Path(__file__).resolve().parents[2] / "frontend" / "src" / "lib" / "termsSource.ts" + + +def _block(source: str, head: str) -> str: + start = source.index(head) + start = source.index("{", start) + 1 + depth, i = 1, start + while depth: + depth += {"{": 1, "}": -1}.get(source[i], 0) + i += 1 + return source[start:i - 1] + + +def _entries(block: str) -> dict[str, str]: + return dict(re.findall(r"'?([a-z_]+)'?:\s*'([^']*)'", block)) + + +def test_change_kind_labels_match_word_for_word(): + source = TERMS.read_text(encoding="utf-8") + labels = _entries(_block(source, "export const SOURCE_CHANGE_KIND_LABEL")) + assert labels == sc.KIND_LABEL, "「变化」标签页的叫法和服务端的对不上" + assert set(labels) == set(sc.KINDS) + + +def test_maintenance_step_labels_match_word_for_word(): + source = TERMS.read_text(encoding="utf-8") + block = _block(source, "export const MAINTENANCE_TEXT") + steps = _entries(_block(block, "step:")) + assert steps == sm.STEP_LABEL, "定时维护的步骤叫法和服务端的对不上（没执行的原因里用的是服务端那一份）" diff --git a/backend/tests/test_source_upkeep.py b/backend/tests/test_source_upkeep.py index 95a4acc7..0aaab038 100644 --- a/backend/tests/test_source_upkeep.py +++ b/backend/tests/test_source_upkeep.py @@ -341,3 +341,23 @@ def test_schedule_target_columns_are_the_ones_the_page_uses(): """数据源设置里建的维护定时：target_kind / target_id 写法和调度那一套同一处封装（X4）。""" cols = scheduler_service.target_columns("source_maintenance", "src-x") assert cols["target_kind"] == "source_maintenance" and cols["target_id"] == "src-x" and cols["workflow_id"] == "" + + +async def test_daily_volume_fills_the_days_without_data(client, shop): + """按日行数只记了有数据的日子：窗口里断了的那几天补 0，界面上的小柱图才看得出断档。""" + sid = shop["id"] + async with SessionLocal() as session: + session.add(TableObservation(source_id=sid, table_name="orders", column_name="paid_on", kind="daily_volume", + basis={"method": "window", "column": "paid_on", "days": 35, "since": "2026-10-01"}, + basis_key="window:paid_on:35", + data={"since": "2026-10-01", "until": "2026-10-05", + "days": {"2026-10-01": 30, "2026-10-02": 28, "2026-10-05": 31}}, + at=utcnow())) + await session.commit() + body = (await client.get(f"/api/datasources/{sid}/observations", params={"table": "orders"})).json() + [daily] = [s for s in body["series"] if s["kind"] == "daily_volume"] + point = daily["points"][0] + assert point["data"]["days"] == [["2026-10-01", 30], ["2026-10-02", 28], ["2026-10-03", 0], ["2026-10-04", 0], + ["2026-10-05", 31]] + assert point["data"]["days_with_data"] == 3 and point["data"]["total"] == 89 + assert point["basis_text"] == "业务日期 paid_on 最近 35 天" diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index d37f2e4b..2481ff42 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -23,6 +23,7 @@ import { bundle, dispositionName } from './bundle' import { compare } from './compare' import { inquiry } from './inquiry' import { schedules } from './schedules' +import { sourceChanges } from './sourceChanges' import { whatif } from './whatif' import { reportExport } from './reportExport' // 应用平台：各自的接口文件，这里只挂上去 @@ -948,6 +949,8 @@ export const api = { // ---- 复核套件：各功能的接口在自己的文件里（api/<名字>.ts），这里只挂上去 ---- schedules, + // 数据源的变化、取值观察、定时维护的步骤、后台任务条（下一轮第 4 波，typesSource.ts 是接口约定） + sourceChanges, compare, bundle, inquiry, diff --git a/frontend/src/api/sourceChanges.ts b/frontend/src/api/sourceChanges.ts new file mode 100644 index 00000000..2f61cb8e --- /dev/null +++ b/frontend/src/api/sourceChanges.ts @@ -0,0 +1,41 @@ +/** + * 数据源的变化、取值观察、定时维护的步骤、后台任务条的接口（服务端 app/api/source_changes.py），挂在 api.sourceChanges 上。 + * 定时本身（cron、时区、开关）走 api.schedules（target_kind=source_maintenance，总方案 X4：定时只有一套）。 + * + * 请求函数从 ./http 导入，不从 ./client 导入（client.ts 要导入这个文件，反过来就成了循环，见 http.ts 开头的说明）。 + * 误报没写理由 422、处理过的再处理 409、目录被别人刚改过 409，message 是服务端给的中文 + */ +import type { + MaintenanceStep, SourceBackgroundJob, SourceChange, SourceChangeList, SourceChangeQuery, SourceChangeResolveBody, + SourceChangeResolveOut, SourceMaintenance, SourceUpkeep, TableObservations, +} from '../typesSource' +import { get, post, put, qs, type RequestOptions } from './http' + +const at = (sourceId: string) => `/datasources/${encodeURIComponent(sourceId)}` + +export const sourceChanges = { + /** 数据源列表一次取齐：各源要处理的变化条数（卡片角标）、指向它的维护定时 */ + upkeep: (opts?: RequestOptions) => get<{ items: SourceUpkeep[] }>('/datasources/upkeep', opts), + /** 一个源的变化：缺省开着的（要处理的先）；status=handled 是处理过的。category 只看结构 / 取值变化 */ + list: (sourceId: string, q: SourceChangeQuery = {}, opts?: RequestOptions) => + get(`${at(sourceId)}/changes${qs({ + status: q.status ?? 'pending', category: q.category ?? undefined, table: q.table ?? undefined, + limit: q.limit ?? 100, offset: q.offset ?? 0, + })}`, opts), + get: (sourceId: string, changeId: string, opts?: RequestOptions) => + get<{ change: SourceChange }>(`${at(sourceId)}/changes/${encodeURIComponent(changeId)}`, opts), + /** 处理一条变化：动作落到目录已有的写入接口并记目录版本、建了的任务编号 */ + resolve: (sourceId: string, changeId: string, body: SourceChangeResolveBody) => + post(`${at(sourceId)}/changes/${encodeURIComponent(changeId)}/resolve`, body), + /** 表详情「结构与取值变化」一节：这张表最近几次观察，各带口径 */ + observations: (sourceId: string, table: string, opts?: RequestOptions) => + get(`${at(sourceId)}/observations${qs({ table })}`, opts), + /** 这个源进行中和排队的后台任务，按同源队的先后排好 */ + backgroundJobs: (sourceId: string, opts?: RequestOptions) => + get<{ items: SourceBackgroundJob[] }>(`${at(sourceId)}/background-jobs`, opts), + /** 定时维护：步骤、剖析开没开、调度开没开、指向这个源的维护定时（定时任务页同一个形状，最近一次触发带 inputs） */ + maintenance: (sourceId: string, opts?: RequestOptions) => get(`${at(sourceId)}/maintenance`, opts), + /** 改要做哪几步（至少一步）。上传表格、记录表 409 */ + putMaintenance: (sourceId: string, steps: MaintenanceStep[]) => + put(`${at(sourceId)}/maintenance`, { steps }), +} diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index c3744e5b..0745fc99 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -6278,3 +6278,5 @@ export const CAPABILITY_REVIEW_TEXT: typeof CATALOG_REVIEW_TEXT = { // 应用平台使用面的文案（FE-5 维护）。调用方照旧从 lib/terms 导入 export * from './termsApp' export * from './termsBlueprint' +// 数据源的变化、定时维护、后台任务条（下一轮第 4 波 W4-d） +export * from './termsSource' diff --git a/frontend/src/lib/termsSource.ts b/frontend/src/lib/termsSource.ts new file mode 100644 index 00000000..3e851b7f --- /dev/null +++ b/frontend/src/lib/termsSource.ts @@ -0,0 +1,263 @@ +/** + * 数据源的变化（结构变化、取值变化）、表详情的「结构与取值变化」一节、定时维护、目录页的后台任务条的文案（next-round 03 B2–B4、 + * 05 (g)）。规矩同 terms.ts 开头的术语表。 + * + * 叫法（总方案 X10）：「结构变化」「取值变化」，和已经在用的「目录有变化」（目录版本变了，CATALOG_DRIFT_TEXT）分开；严重程度 + * 「要处理 / 提醒 / 说明」同 SCHEMA_DRIFT_TEXT.severity。 + * + * terms.ts 末尾 `export * from './termsSource'`：调用方照旧从 lib/terms 导入。 + */ +import type { MaintenanceStep, SourceChangeAction, SourceChangeCategory, SourceChangeSeverity, SourceChangeStatus } from '../typesSource' +import type { ObservationKind, SourceJobKind } from '../typesSource' +import { formatNumber } from './format' + +const pct = (share: number) => { + const v = share * 100 + return `${v >= 10 || v === 0 ? Math.round(v) : v.toFixed(1)}%` +} + +/** + * 变化的种类：和服务端 app/data/source_changes.KIND_LABEL 逐字一致（tests/test_source_change_terms.py 守着），服务端给人看 + * 的一句（summary）也用它 + */ +export const SOURCE_CHANGE_KIND_LABEL: Record = { + table_added: '新出现的表', table_removed: '表已删除', table_left_scope: '表离开了探查范围', + table_reflect_failed: '这次读不到表结构', column_added: '新加了列', column_removed: '删除了列', + column_type_changed: '列的类型变了', column_nullable_changed: '列的可空变了', key_changed: '键变了', fk_changed: '外键变了', + comment_changed: '注释变了', scope_changed: '探查范围的规则变了', + codes_new: '出现了新码值', codes_shift: '码值的比例变化大', date_stalled: '最新业务日期没有前进', + date_gap: '有的日子没有数据', volume_drop: '按日行数大幅下跌', rows_drop: '行数大幅下跌', rows_jump: '行数大幅上涨', + coverage_drop: '关联的覆盖率下降', +} + +export const SOURCE_CHANGE_TEXT = { + /** 数据目录页左栏的第三个视图 */ + view: '变化', + viewHint: '探查、取值观察测到的结构变化和取值变化，按要不要处理分组', + category: { struct: '结构变化', value: '取值变化' } as Record, + categoryAll: '全部', + categoryLabel: '按类别筛选', + severity: { action: '要处理', warning: '提醒', info: '说明' } as Record, + severityHint: { + action: '现在大概率已经不对了：已确认的目录项或写死的 SQL 指向了被删的列，或者「已列全」被撤销了', + warning: '值得核对：可能影响 SQL 的写法或结果', + info: '知道一下：一般不影响已有的目录和模板', + } as Record, + status: { open: '开着', acknowledged: '知道了，还没处理', resolved: '已处理', dismissed: '标为误报' } as Record, + statusFilter: { pending: '开着的', handled: '处理过的' } as Record<'pending' | 'handled', string>, + statusLabel: '按状态筛选', + listLabel: '数据源的变化', + loading: '正在载入…', + refresh: '刷新', + more: (n: number) => `再显示 ${formatNumber(n)} 条`, + total: (n: number) => `${formatNumber(n)} 条`, + group: (label: string, n: number) => `${label} ${formatNumber(n)} 条`, + empty: '没有开着的变化', + emptyBody: '探查结构、取值观察测到表结构或取值有变化时，会列在这里。', + emptyFiltered: '没有符合筛选的变化', + emptyHandled: '还没有处理过的变化', + /** 视图标签上的计数：要处理的条数 */ + badge: (n: number) => `${formatNumber(n)} 条要处理`, + seen: (n: number) => `测到 ${formatNumber(n)} 次`, + detectedAt: (when: string) => `测到于 ${when}`, + overviewTitle: '从左侧挑一条变化开始处理', + overviewBody: '要处理的排在最前：已确认的目录项、写死的 SQL 说到了被删的列，或者「已列全」被撤销了。点开一条，右侧写依据、影响面和能做的处理；每个处理都记下谁、什么时候、目录第几版。', + backToList: '返回变化列表', + missing: '这条变化不存在，可能已随数据源删除', + openTable: '查看这张表', + // ---- 依据 + basisTitle: '依据', + detectedBy: (job: string, trigger: string, actor: string | null) => + `${trigger}的${job}${actor ? `（${actor}）` : ''}`, + jobKind: { introspect: '探查结构', profile: '取值观察', on_demand: '按需加入' } as Record, + trigger: { manual: '手动发起', schedule: '定时维护', on_demand: '助手点名' } as Record, + structBasis: (before: number | null, after: number | null) => (before && after + ? `比对的是这张表的结构第 ${formatNumber(before)} 版和第 ${formatNumber(after)} 版` + : before ? `比对的是这张表的结构第 ${formatNumber(before)} 版和这次探查读到的` : '比对的是上一次和这次探查读到的结构'), + valueBasis: (text: string) => `观察口径：${text}`, + valueBasisNote: '只和口径相同的历史观察比', + // ---- 前后各是什么（按种类） + factsTitle: '前后', + before: '之前', + after: '现在', + none: '（无）', + nullable: (v: boolean) => (v ? '可空' : '不可空'), + rename: (from: string, to: string) => `可能是 ${from} 改名为 ${to}（未确认：库里只看得到删了一列、加了一列）`, + newCodes: (codes: string) => `新出现的码值：${codes}`, + codeShare: (code: string, share: number) => `${code}（占 ${pct(share)}）`, + knownCodes: (n: number) => `目录里列出的 ${formatNumber(n)} 个取值之外`, + completeRevoked: (version: number | null) => + `取值观察撤销了码值项的「已列全」${version ? `（目录第 ${formatNumber(version)} 版）` : ''}：SQL 检查不再按旧的码值表提示，新取值的含义要人来填`, + completeKept: '「已列全」在观察之前已被别人改掉，没有撤销', + shift: (psi: number) => `比例的变化程度（PSI）${psi.toFixed(2)}`, + shiftTop: (code: string, from: number, to: number) => `${code}：${pct(from)} → ${pct(to)}`, + stalled: (max: string, age: number, threshold: number) => + `最新业务日期停在 ${max}，已经 ${formatNumber(age)} 天（以前通常 ${formatNumber(threshold)} 天之内会前进）`, + gap: (n: number, days: string) => `窗口里有 ${formatNumber(n)} 天没有数据：${days}`, + volume: (day: string, rows: number, median: number) => + `${day} 只有 ${formatNumber(rows)} 行，前 28 天的中位数是 ${formatNumber(median)} 行`, + rows: (before: number, after: number, estimated: boolean) => + `${formatNumber(before)} 行 → ${formatNumber(after)} 行${estimated ? '（数据库统计的估算值，连续两次都这样才报）' : ''}`, + coverage: (target: string, before: number, after: number) => `指向 ${target} 的关联覆盖率 ${pct(before)} → ${pct(after)}`, + scopeRules: (entered: number, left: number) => `这次探查新进范围 ${formatNumber(entered)} 张，离开范围 ${formatNumber(left)} 张`, + reflectFailed: (error: string) => `读不到结构的原因：${error}`, + // ---- 影响面 + impactTitle: '影响面', + impactNone: '目录里没有说到它的项，已发布模板里也没有用到它的。', + impactCatalog: (n: number) => `说到这些列的目录项 ${formatNumber(n)} 项`, + impactNow: '现在', + impactGone: '已不在目录里', + impactNeeds: '人工确认过、还说着被删的列', + impactRules: '有效记录条件用到了这一列：取值有变时，按它过滤的结果可能漏掉新状态的行', + impactTemplates: (n: number) => `用到这张表的已发布模板 ${formatNumber(n)} 个`, + templateImpact: { direct: '写死的 SQL 直接引用', possible: '可能涉及' } as Record, + templateVersion: (v: number) => `v${v}`, + breaks: (cols: string[]) => (cols.length + ? `这条 SQL 现在会报列不存在（${cols.join('、')}）` + : '这条 SQL 现在会报表或列不存在'), + // ---- 处理 + actionsTitle: '处理', + action: { + clean_catalog: '清理目录项', add_codes: '加入码值', redraft: '重新起草这张表', reprofile: '重新剖析这张表', + acknowledge: '知道了', dismiss: '误报', + } as Record, + actionHint: { + clean_catalog: '把影响面里说到这些列的目录项从目录里去掉（目录升一版，记下是因为这条变化）', + add_codes: '把新出现的码值加进这一列的码值表，含义可以先空着', + redraft: '建一个只含这张表的起草任务：按新的结构重写推断项，人工确认过的项不动', + reprofile: '建一个只含这张表的剖析任务：对业务库发只读查询，重新核对关系、码值和日期', + acknowledge: '记下已经看过，还留在开着的变化里', + dismiss: '不是真的变化：写明理由后不再算要处理', + } as Record, + cleanTitle: (n: number) => `清理 ${formatNumber(n)} 项目录？`, + cleanConsequences: ['这些项从数据目录里去掉，目录升一版；从已发布版本发起的正式运行会提醒「数据目录有变化」', + '人工确认过的项也会去掉：它们说的列已经不在了', '去掉之后要恢复，只能重新填写'], + cleanConfirm: '清理', + codesTitle: '加入码值', + codesHint: '含义可以先空着，之后在表详情的码值表里补；加入的项记为人工填写、已确认，依据写这条变化的编号', + codesMeaning: (code: string) => `码值 ${code} 的含义`, + codesSubmit: '加入', + noteLabel: '误报的理由', + notePlaceholder: '例如：测试库里故意删的列', + noteRequired: '标为误报要写明理由', + noteSubmit: '标为误报', + cancel: '取消', + done: { + clean_catalog: '已清理目录项', add_codes: '已加入码值', redraft: '已建起草任务', reprofile: '已建剖析任务', + acknowledge: '已记为知道了', dismiss: '已标为误报', + } as Record, + // ---- 处理过的结果 + resolutionTitle: '处理结果', + resolvedBy: (action: string, by: string | null, when: string) => `${action} · ${by ?? '未署名'} · ${when}`, + resolvedVersion: (v: number) => `目录第 ${formatNumber(v)} 版`, + resolvedVersions: (pairs: string) => `各表的目录版本：${pairs}`, + resolvedNote: (note: string) => `理由：${note}`, + resolvedJob: { draft: '起草任务', profile: '剖析任务' } as Record, + jobLink: (kind: string, id: string) => `${kind} #${id.slice(0, 6)}`, + viewProgress: '查看进度', + /** 不能做的动作的读屏说明：「重新剖析这张表：这个数据源未开启数据剖析」 */ + unavailable: (label: string, reason: string) => `${label}：${reason}`, +} + +/** 数据源卡片上的角标：要处理的变化条数（读屏写全） */ +export const SOURCE_BADGE_TEXT = { + chip: (n: number) => `${formatNumber(n)} 个变化要处理`, + aria: (name: string, n: number) => `数据源「${name}」有 ${formatNumber(n)} 个变化要处理，打开数据目录的「变化」`, + hint: '探查结构、取值观察测到的变化里要处理的条数，点开到数据目录的「变化」处理', +} + +/** 表详情的「结构与取值变化」一节 */ +export const TABLE_CHANGES_TEXT = { + title: '结构与取值变化', + hint: '这张表的结构变化、取值变化，和最近几次取值观察', + changesHead: '这张表的变化', + changesNone: '这张表还没有测到结构变化或取值变化。', + changesMore: (n: number) => `…另有 ${formatNumber(n)} 条，在「变化」里查看`, + openChange: '去处理', + viewChange: '查看', + observeHead: '最近几次观察', + observeNone: '这张表还没有取值观察：定时维护选了「取值观察」，或者手动剖析时带上观察，才会有。', + observeKind: { rows: '行数', codes_observe: '码值分布', freshness: '最新业务日期', daily_volume: '按日行数' } as Record, + basis: (text: string) => `口径：${text}`, + basisChanged: '这几次的口径不全相同：口径不同的不互相比较', + at: (when: string) => `观察于 ${when}`, + rows: (rows: number | null, atLeast: boolean) => (rows == null ? '—' : `${atLeast ? '至少 ' : ''}${formatNumber(rows)} 行`), + codesTop: (n: number, rows: number) => `${formatNumber(n)} 个取值，共 ${formatNumber(rows)} 行`, + codesTruncated: '取值太多，只记了最多的 50 个：不判新码值、不算比例', + code: (code: string, share: number) => `${code} ${pct(share)}`, + freshness: (max: string | null) => max ?? '—', + daily: (days: number, total: number) => `有数据的 ${formatNumber(days)} 天，共 ${formatNumber(total)} 行`, + dailyRange: (since: string, until: string) => `${since} 至 ${until}`, + dailyChart: (since: string, until: string, days: number, total: number) => + `${since} 至 ${until} 每天的行数：有数据的 ${formatNumber(days)} 天，共 ${formatNumber(total)} 行`, + history: (n: number) => `前 ${formatNumber(n)} 次`, + revokedHead: '撤销过「已列全」', + revoked: (column: string, when: string, version: number | null) => + `${when}：取值观察在 ${column} 上测到目录之外的取值，撤销了「已列全」${version ? `（目录第 ${formatNumber(version)} 版）` : ''}`, +} + +/** 目录页的后台任务条（05 (g)）：页头只做入口，进度只在这一条上 */ +export const JOB_STRIP_TEXT = { + label: '这个数据源的后台任务', + kind: { introspect: '探查', draft: '起草', profile: '剖析' } as Record, + running: (kind: string, done: number, total: number) => `${kind} ${formatNumber(done)}/${formatNumber(total)}`, + queued: (kind: string) => `${kind} 排队中`, + queuedBehind: (kind: string, behind: string) => `${kind} 排队中，${behind}结束后开始`, + behind: { introspect: '探查', draft: '起草', profile: '剖析', import: '导入目录', sync_draft: '起草' } as Record, + progress: (kind: string, done: number, total: number) => `${kind}：已完成 ${formatNumber(done)} / ${formatNumber(total)} 张表`, + view: '查看', + viewHint: { introspect: '到数据源卡片上看探查的进度、停止', draft: '打开起草的进度', profile: '打开剖析的进度' } as Record, + /** 页头的入口：读屏名称写状态，不写数字 */ + headRunning: (label: string) => `${label}（进行中）`, + headQueued: (label: string) => `${label}（排队中）`, +} + +/** 数据源的定时维护（数据源卡片上的「定时维护」） */ +export const MAINTENANCE_TEXT = { + open: '定时维护', + openHint: '按设定的时段自动探查结构、剖析需要的表、观察取值，缺省关闭', + title: (name: string) => `「${name}」的定时维护`, + intro: '到点按顺序探查结构 → 剖析需要的表 → 观察取值，和起草、剖析在同一条队里排队，一个接一个做。只发只读查询，每张表的查询次数和时长按剖析设置。', + enable: '开启定时维护', + enableHint: '缺省关闭：开启后会定期对业务库发只读查询，请选业务库的低峰时段（比如凌晨）', + offPeak: '请选业务库的低峰时段（比如凌晨），避开白天的业务高峰', + stepsTitle: '做哪几步', + stepsHint: '至少选一步', + step: { introspect: '探查结构', profile_needed: '剖析需要的表', observe: '取值观察' } as Record, + stepHint: { + introspect: '读取探查范围内的表结构，前后比出结构变化', + profile_needed: '剖析从没剖析过、上次没做完、目录里多出待核实项的表', + observe: '观察码值分布、最新业务日期、按日行数和行数，和同口径的历史比出取值变化', + } as Record, + stepsNone: '至少选一步', + profileOff: '这个数据源未开启数据剖析：到点时「剖析需要的表」「取值观察」两步不执行，只探查结构（照实记下）。剖析在数据源的「编辑」里开启。', + schedulerOff: '这个服务的定时调度没有开启：到点不会自动维护。', + freqTitle: '时段', + scheduleName: (name: string) => `「${name}」定时维护`, + next: (when: string) => `下一次：${when}`, + nextNone: '已暂停，不会触发', + lastTitle: '最近一次', + lastNone: '还没有触发过', + lastAt: (when: string, status: string) => `${when} · ${status}`, + fireStatus: { started: '已开始', skipped_overlap: '已跳过（上一次还没做完）', skipped_misfire: '已跳过（错过了时间）', failed_to_start: '没有执行' } as Record, + lastJob: (step: string, kind: string, id: string, tables: number | null) => + `${step}：${kind} #${id.slice(0, 6)}${tables != null ? `（${formatNumber(tables)} 张表）` : ''}`, + lastJobKind: { introspect: '探查任务', profile: '剖析任务' } as Record, + lastSkipped: (step: string, reason: string) => `${step}没有执行：${reason}`, + lastReason: (reason: string) => `原因：${reason}`, + viewJob: '查看进度', + history: '在定时任务页查看触发记录', + extra: (n: number) => `另有 ${formatNumber(n)} 条指向这个数据源的维护定时，在定时任务页管理`, + save: '保存', + cancel: '取消', + saving: '正在保存…', + saved: '已保存定时维护', + savedOff: '已保存：定时维护已关闭', + nothing: '没有改动', + notMaintained: '上传的表格、记录表的结构随版本或表单走，不需要定时维护', + /** 卡片上的标签：开着时写频率 */ + chip: (desc: string) => `定时维护 · ${desc}`, + chipPaused: '定时维护 · 已暂停', + chipHint: '到点自动探查结构、剖析需要的表、观察取值', + loadFailed: '无法载入定时维护的设置', +} diff --git a/frontend/src/pages/DataSourcesTab.tsx b/frontend/src/pages/DataSourcesTab.tsx index b8dee988..175d9fea 100644 --- a/frontend/src/pages/DataSourcesTab.tsx +++ b/frontend/src/pages/DataSourcesTab.tsx @@ -1,8 +1,8 @@ import { useEffect, useId, useMemo, useRef, useState } from 'react' import { useNavigate } from 'react-router-dom' import { - AlertTriangle, BookMarked, ChevronRight, Database, EyeOff, FileSpreadsheet, History, Info, KeyRound, Layers, Lock, Plug, Plus, - RefreshCw, ScanSearch, Search, Table2, Upload, X, + AlertOctagon, AlertTriangle, BookMarked, CalendarClock, ChevronRight, Database, EyeOff, FileSpreadsheet, History, Info, KeyRound, Layers, + Lock, Plug, Plus, RefreshCw, ScanSearch, Search, Table2, Upload, X, } from 'lucide-react' import clsx from 'clsx' import { ApiError, api, uploadDecision } from '../api/client' @@ -23,8 +23,8 @@ import { checkHealth, forgetHealth, healthFromServer, setHealth, useHealth } fro import type { HealthRecord } from '../lib/health' import { workflowList, workflowsMentioning } from '../lib/mentions' import { - CATALOG_TEXT, INTROSPECT_SCOPE_TEXT, INTROSPECT_TEXT, MASK_BOUNDARY_TEXT, PROFILE_TEXT, RAW_STATE_LABEL, RECIPE_ORIGIN_LABEL, - RECIPE_TEXT, UPLOAD_CONVERSION_LABEL, UPLOAD_SHAPE_LABEL, UPLOAD_TEXT, VERSIONS_TEXT, + CATALOG_TEXT, INTROSPECT_SCOPE_TEXT, INTROSPECT_TEXT, MAINTENANCE_TEXT, MASK_BOUNDARY_TEXT, PROFILE_TEXT, RAW_STATE_LABEL, + RECIPE_ORIGIN_LABEL, RECIPE_TEXT, SOURCE_BADGE_TEXT, UPLOAD_CONVERSION_LABEL, UPLOAD_SHAPE_LABEL, UPLOAD_TEXT, VERSIONS_TEXT, } from '../lib/terms' import { useLoginOn } from '../store/auth' import { useRunClock } from '../run/useRunClock' @@ -36,6 +36,8 @@ import { import type { ProfileForm } from './catalog/profile' import { ImportWizard } from './import/ImportWizard' import { IntrospectScopeDialog } from './IntrospectScopeDialog' +import { MaintenanceChip, SourceMaintenanceDialog } from './SourceMaintenanceDialog' +import type { SourceUpkeep } from '../typesSource' import type { WizardEntry } from './import/ImportWizard' import { VersionsDialog } from './import/VersionsDialog' @@ -89,8 +91,13 @@ export function DataSourcesTab({ view = 'databases' }: { view?: DataView }) { /** 按配方导入的向导（首次、上传新一期、修改配方、继续未完成的导入） */ const [wizard, setWizard] = useState(null) const [kick, setKick] = useState<{ id: string; seq: number } | null>(null) + /** 各源要处理的变化条数（角标）和定时维护（next-round 03 B3、B4）：一次取齐；取不到只是少了角标，不挡列表 */ + const [upkeep, setUpkeep] = useState>(() => new Map()) const loadSeq = useRef(0) const hasRows = useRef(false) + const loadUpkeep = () => { + api.sourceChanges.upkeep().then((r) => setUpkeep(new Map(r.items.map((i) => [i.source_id, i]))), () => {}) + } // 首次加载才画骨架；之后的刷新保留列表、不卸载——展开的表清单、滚动位置、 // 卡片上的忙碌态都长在卡片上，以前整页换成 Spinner，这些全丢，还跳回顶部 @@ -109,8 +116,18 @@ export function DataSourcesTab({ view = 'databases' }: { view?: DataView }) { else setLoadError(e) } } - useEffect(() => { void load() }, []) - useOnReconnect(load) + useEffect(() => { void load(); loadUpkeep() }, []) + useOnReconnect(() => { void load(); loadUpkeep() }) + // 从别处链过来指名一张卡片（#source-，目录页后台任务条上探查的「查看」）：列表载入后滚过去 + const scrolledTo = useRef('') + useEffect(() => { + const hash = window.location.hash + if (!rows || !hash.startsWith('#source-') || scrolledTo.current === hash) return + const el = document.getElementById(decodeURIComponent(hash.slice(1))) + if (!el) return + scrolledTo.current = hash + el.scrollIntoView({ block: 'center' }) + }, [rows]) const upsert = (row: any) => setRows((rs) => { if (!rs) return [row] @@ -198,6 +215,8 @@ export function DataSourcesTab({ view = 'databases' }: { view?: DataView }) { // 版本页里清除原件之后服务端不返回数据源，卡片上的原件状态靠重新取列表 onReload={() => void load()} kick={kick && kick.id === row.id ? kick.seq : 0} + upkeep={upkeep.get(row.id) ?? null} + onUpkeep={loadUpkeep} /> ))} @@ -271,8 +290,13 @@ export function DataSourcesTab({ view = 'databases' }: { view?: DataView }) { /** 结构同步超过这么久就提示可能过期：库表会变，Copilot 照旧表写 SQL 只会报错 */ const STALE_SCHEMA_MS = 7 * 24 * 3600_000 -function SourceCard({ row, meta, onChange, onRemoved, onEdit, onReupload, onRecipeReupload, onRedraft, onResume, onReload, kick }: { +function SourceCard({ row, meta, onChange, onRemoved, onEdit, onReupload, onRecipeReupload, onRedraft, onResume, onReload, kick, + upkeep = null, onUpkeep }: { row: any; meta?: any + /** 要处理的变化条数、定时维护（GET /api/datasources/upkeep）；取不到为 null */ + upkeep?: SourceUpkeep | null + /** 定时维护存过了：重取角标和维护 */ + onUpkeep?: () => void onChange: (row: any) => void /** 重新取整个列表（版本页里的写操作没有返回数据源时用） */ onReload: () => void @@ -324,6 +348,11 @@ function SourceCard({ row, meta, onChange, onRemoved, onEdit, onReupload, onReci const [job, setJob] = useState(row.introspect_job ?? null) const [stopping, setStopping] = useState(false) const [scoping, setScoping] = useState(false) + const [maintaining, setMaintaining] = useState(false) + /** 要处理的变化条数（数据源卡片的角标，next-round 03 B3 ③） */ + const actionChanges = upkeep?.changes.by_severity.action ?? 0 + /** 定时维护只给手工登记的库：上传表格、记录表的结构随版本或表单走 */ + const maintainable = (row.origin ?? 'manual') === 'manual' const jobActive = !!job && (job.status === 'queued' || job.status === 'running') useEffect(() => { const next = row.introspect_job ?? null @@ -530,7 +559,7 @@ function SourceCard({ row, meta, onChange, onRemoved, onEdit, onReupload, onReci const failed = record && !record.ok && !checkingSince return ( -
+
{uploaded ? @@ -544,6 +573,14 @@ function SourceCard({ row, meta, onChange, onRemoved, onEdit, onReupload, onReci 可写 } {!row.enabled && 已停用} + {actionChanges > 0 && ( + // 要处理的变化：点开到数据目录的「变化」视图（读屏写全：哪个源、几条、点了去哪） + + )}
@@ -573,6 +610,12 @@ function SourceCard({ row, meta, onChange, onRemoved, onEdit, onReupload, onReci {INTROSPECT_SCOPE_TEXT.open} )} + {!uploaded && maintainable && ( + + )} {row.open_staging && ( )} + {maintainable && !uploaded && upkeep && } {!!row.tools?.length && {row.tools.join(' · ')}}
+ {maintaining && ( + setMaintaining(false)} onSaved={() => onUpkeep?.()} /> + )} {scoping && ( setScoping(false)} onSaved={onReload} onRun={() => void introspect()} /> diff --git a/frontend/src/pages/SourceMaintenanceDialog.tsx b/frontend/src/pages/SourceMaintenanceDialog.tsx new file mode 100644 index 00000000..4a051770 --- /dev/null +++ b/frontend/src/pages/SourceMaintenanceDialog.tsx @@ -0,0 +1,297 @@ +import { useEffect, useMemo, useRef, useState } from 'react' +import { Link, useNavigate } from 'react-router-dom' +import { AlertTriangle, CalendarClock, Info } from 'lucide-react' +import { ApiError, api } from '../api/client' +import type { SchedulePatch, SchedulePreview, ScheduleTargetCreate } from '../types' +import type { MaintenanceSchedule, MaintenanceStep, SourceMaintenance } from '../typesSource' +import { FreqPicker } from '../components/FreqPicker' +import { ErrorState, Modal, Skeleton, Spinner, toast, useTicker } from '../components/ui' +import { formatDateTime, parseServerTime } from '../lib/format' +import { DEFAULT_FREQ, DEFAULT_GRACE_S, browserTimezone, cronOf, freqOf, freqProblem } from '../lib/freq' +import type { Freq } from '../lib/freq' +import { MAINTENANCE_TEXT as T } from '../lib/terms' +import { EnableSwitch, FireStatusChip, NextRun } from './schedules/parts' + +// =========================================================================== +// 数据源的定时维护（docs/design/next-round/03-数据接入与目录维护.md B4，目录-D5）：按数据源开、缺省关、只在设定的时段跑。 +// +// 定时只有一套（总方案 X4）：维护就是一条 target_kind=source_maintenance、target_id=数据源 id 的定时任务，这里是它的快捷入口。 +// 频率用定时任务同一个选择器（components/FreqPicker），保存成 POST /api/schedules（ScheduleTargetCreate）；已经有了的改频率、 +// 开关走 PATCH（SchedulePatch，带读到的版本）。做哪几步存在数据源上（PUT /api/datasources/{id}/maintenance）。下一次什么 +// 时候、最近一次建了哪些任务、哪几步没执行和原因都照服务端给的写（触发记录的 inputs）；剖析没开、调度没开时照实说到点会怎样。 +// +// 放在数据源卡片上的独立对话框（和「探查范围」一样），不塞进编辑表单：编辑表单只有一个「保存」，存的是连接配置；维护是一条 +// 单独的定时任务，存法不同，两个保存混在一起出错时说不清哪一样没存上。 +// =========================================================================== + +const STEPS: readonly MaintenanceStep[] = ['introspect', 'profile_needed', 'observe'] +/** 新开的维护缺省在凌晨三点：低峰时段 */ +const DEFAULT_MAINTENANCE_FREQ: Freq = { ...DEFAULT_FREQ, preset: 'daily', time: '03:00' } +const PREVIEW_DEBOUNCE_MS = 400 + +interface Form { + enabled: boolean + steps: MaintenanceStep[] + freq: Freq + timezone: string +} + +const formOf = (m: SourceMaintenance, s: MaintenanceSchedule | null): Form => ({ + enabled: s?.enabled ?? false, + steps: m.steps, + freq: s ? freqOf(s.cron) : DEFAULT_MAINTENANCE_FREQ, + timezone: s?.timezone ?? browserTimezone(), +}) + +export function SourceMaintenanceDialog({ source, onClose, onSaved }: { + source: { id: string; name: string } + onClose: () => void + /** 保存过了：卡片上的「定时维护」标签跟着变 */ + onSaved: () => void +}) { + const navigate = useNavigate() + const [data, setData] = useState(null) + const [loadError, setLoadError] = useState(null) + const [form, setForm] = useState
(null) + const initial = useRef('') + const [saving, setSaving] = useState(false) + const [error, setError] = useState<{ field?: string; message: string } | null>(null) + const [preview, setPreview] = useState<{ cron: string; tz: string; data?: SchedulePreview; error?: { field?: string; message: string } } | null>(null) + useTicker(30_000) + + const schedule = data?.schedules[0] ?? null + const extra = Math.max(0, (data?.schedules.length ?? 0) - 1) + + const load = async () => { + const got = await api.sourceChanges.maintenance(source.id) + setData(got) + const f = formOf(got, got.schedules[0] ?? null) + initial.current = JSON.stringify({ ...f, cron: cronOf(f.freq) }) + setForm(f) + setLoadError(null) + } + useEffect(() => { load().catch(setLoadError) }, [source.id]) + + const cron = form ? cronOf(form.freq) : '' + const freqIncomplete = form ? freqProblem(form.freq) : null + const sig = form ? JSON.stringify({ ...form, cron }) : '' + const dirty = !!form && sig !== initial.current + const noSteps = !!form && !form.steps.length + + // 频率的说法和接下来几次：服务端算（前端不解析 cron），停手 400 毫秒再问，新的一次发出时作废上一次 + useEffect(() => { + if (!form || !cron || freqIncomplete) { + setPreview(null) + return + } + const ctrl = new AbortController() + const tz = form.timezone + const timer = setTimeout(() => { + api.schedules.preview({ cron, timezone: tz, n: 3 }, { signal: ctrl.signal }) + .then((d) => setPreview({ cron, tz, data: d })) + .catch((e) => { + if ((e as { name?: string })?.name === 'AbortError') return + setPreview({ cron, tz, error: { field: e instanceof ApiError ? e.field : undefined, message: (e as Error).message } }) + }) + }, PREVIEW_DEBOUNCE_MS) + return () => { clearTimeout(timer); ctrl.abort() } + }, [cron, form?.timezone, freqIncomplete]) + + const freqError = error?.field === 'cron' ? error.message : preview?.error && preview.error.field !== 'timezone' ? preview.error.message + : form?.enabled ? freqIncomplete ?? undefined : undefined + const tzError = error?.field === 'timezone' ? error.message : preview?.error?.field === 'timezone' ? preview.error.message : undefined + + const save = async () => { + if (!form || !data || saving) return + if (noSteps || (form.enabled && (freqIncomplete || preview?.error))) return + setSaving(true) + setError(null) + try { + const was = JSON.parse(initial.current) as Form & { cron: string } + if (JSON.stringify(form.steps) !== JSON.stringify(was.steps)) await api.sourceChanges.putMaintenance(source.id, form.steps) + if (schedule) { + const body: SchedulePatch = { version: schedule.version } + if (cron && cron !== schedule.cron) body.cron = cron + if (form.timezone !== schedule.timezone) body.timezone = form.timezone + if (form.enabled !== schedule.enabled) body.enabled = form.enabled + if (Object.keys(body).length > 1) await api.schedules.update(schedule.id, body) + } else if (form.enabled) { + const body: ScheduleTargetCreate = { + name: T.scheduleName(source.name), target_kind: 'source_maintenance', target_id: source.id, cron, timezone: form.timezone, + enabled: true, overlap: 'skip', misfire_grace_s: DEFAULT_GRACE_S, + } + await api.schedules.create(body) + } + await load() + toast.ok(form.enabled ? T.saved : T.savedOff) + onSaved() + } catch (e) { + // 频率、时区不合规（422 带 field）落到那一项；版本不符（别人刚改过，409）、别的原因写在底部 + if (e instanceof ApiError && e.kind === 'http' && (e.status === 422 || e.status === 409)) setError({ field: e.field, message: e.message }) + else toast.error(e) + } finally { + setSaving(false) + } + } + + const set = (patch: Partial) => { + setError(null) + setForm((f) => f && ({ ...f, ...patch })) + } + const toggleStep = (step: MaintenanceStep) => { + if (!form) return + const on = form.steps.includes(step) + set({ steps: STEPS.filter((s) => (s === step ? !on : form.steps.includes(s))) }) + } + + const profileSteps = !!form && form.steps.some((s) => s !== 'introspect') + const blocked = !form || noSteps || (form.enabled && (!!freqIncomplete || !!preview?.error)) + + return ( + + {error && !error.field &&

{error.message}

} + + + + )}> + {!data || !form + ? (loadError ? void load().catch(setLoadError)} /> : ) + : !data.maintained + ?

{T.notMaintained}

+ : ( +
+

{T.intro}

+ +
+
+

{T.enable}

+ + + set({ enabled: v })} /> + +
+

{T.enableHint}

+ {form.enabled && ( +

+ {T.offPeak} +

+ )} + {!data.scheduler_on && ( +

+ {T.schedulerOff} +

+ )} +
+ +
+ {T.stepsTitle} +

+ {noSteps ? T.stepsNone : T.stepsHint} +

+ {STEPS.map((step) => ( + + ))} + {!data.profile_enabled && profileSteps && ( +

{T.profileOff}

+ )} +
+ +
+ set({ freq: { ...form.freq, ...patch } })} setTimezone={(tz) => set({ timezone: tz })} + freqError={freqError} tzError={tzError} /> + {preview?.data && ( +

+ {preview.data.description_text} + {preview.data.next.length > 0 && · {preview.data.next.map((n) => formatDateTime(n)).join('、')}} +

+ )} +
+ + {schedule && { + onClose() + navigate(`/data/catalog/${encodeURIComponent(source.id)}?pjob=${encodeURIComponent(id)}`) + }} onClose={onClose} />} + {extra > 0 &&

{T.extra(extra)}

} +
+ )} + + ) +} + +/** 已经有维护定时：下一次什么时候、最近一次建了哪些任务、哪几步没执行和原因（触发记录的 inputs），到定时任务页看全部触发 */ +function LastRun({ schedule: s, onOpenProfileJob, onClose }: { + schedule: MaintenanceSchedule; onOpenProfileJob: (id: string) => void; onClose: () => void +}) { + const fire = s.last_fire + const inputs = fire?.inputs ?? {} + const at = fire ? parseServerTime(fire.fire_at) : null + return ( +
+

+ + {s.description_text} + · + +

+

{T.lastTitle}

+ {!fire + ?

{T.lastNone}

+ : ( + <> +

+ {at ? formatDateTime(at) : fire.fire_at} + +

+ {(inputs.jobs ?? []).length > 0 && ( +
    + {(inputs.jobs ?? []).map((j) => ( +
  • + {T.lastJob(j.step.split('+').map((x) => T.step[x as MaintenanceStep] ?? x).join('、'), + T.lastJobKind[j.kind] ?? j.kind, j.id, j.tables ?? null)} + {j.kind === 'profile' + ? + // 探查任务的进度就在这张数据源卡片上：关上对话框就看得到 + : } +
    • + ))} +
+ )} + {(inputs.skipped ?? []).map((k) => ( +

+ {T.lastSkipped(T.step[k.step] ?? k.step, k.reason)} +

+ ))} + {!(inputs.jobs ?? []).length && !(inputs.skipped ?? []).length && fire.reason && ( +

{T.lastReason(fire.reason)}

+ )} + + )} + {T.history} +
+ ) +} + +/** 卡片上的「定时维护」标签：开着时写频率、暂停时照实写；没有维护定时不画 */ +export function MaintenanceChip({ maintenance }: { maintenance: { enabled: boolean; description_text: string }[] }) { + const first = maintenance[0] + const text = useMemo(() => (first ? (first.enabled ? T.chip(first.description_text) : T.chipPaused) : ''), [first]) + if (!first) return null + return ( + + {text} + + ) +} diff --git a/frontend/src/pages/catalog/CatalogPage.tsx b/frontend/src/pages/catalog/CatalogPage.tsx index f2f58f17..aefc25b7 100644 --- a/frontend/src/pages/catalog/CatalogPage.tsx +++ b/frontend/src/pages/catalog/CatalogPage.tsx @@ -2,7 +2,7 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react' import type { KeyboardEvent as ReactKeyboardEvent, ReactNode } from 'react' import { useNavigate, useParams, useSearchParams } from 'react-router-dom' import { - ArrowLeft, BookMarked, Database, Download, ListFilter as ListFilterIcon, ScanSearch, Sparkles, TriangleAlert, Upload, + ArrowLeft, BookMarked, Clock, Database, Download, ListFilter as ListFilterIcon, ScanSearch, Sparkles, Upload, } from 'lucide-react' import clsx from 'clsx' import { ApiError, api } from '../../api/client' @@ -14,12 +14,12 @@ import { useCatalog, useDatasources, useOnReconnect } from '../../store/catalog' import { errorMessage } from '../../lib/errors' import { formatNumber } from '../../lib/format' import { - CATALOG_DRAFT_TEXT as DT, CATALOG_REVIEW_TEXT as RT, CATALOG_TEXT as CT, CATALOG_TRANSFER_TEXT as XT, PROFILE_TEXT as PT, - SCHEMA_PARTIAL_TEXT as SPT, + CATALOG_DRAFT_TEXT as DT, CATALOG_REVIEW_TEXT as RT, CATALOG_TEXT as CT, CATALOG_TRANSFER_TEXT as XT, JOB_STRIP_TEXT as JT, + PROFILE_TEXT as PT, SCHEMA_PARTIAL_TEXT as SPT, SOURCE_CHANGE_TEXT as GT, } from '../../lib/terms' import { StatusLegend, SystemNotesNotice } from './parts' import { DraftDialog } from './DraftDialog' -import { isJobActive, jobBlank, retryableCount, useDraftJob, waitsForProfile } from './draftJob' +import { isJobActive, jobBlank, retryableCount, useDraftJob } from './draftJob' import type { DraftPreset, DraftStartBody } from './draftJob' import { ProfileDialog } from './ProfileDialog' import { ProfileSettingsDialog } from './ProfileSettings' @@ -32,7 +32,12 @@ import { TableIndex } from './TableIndex' import { ReviewOverview, ReviewQueue } from './ReviewQueue' import { ImportDialog } from './TransferDialog' import { itemKey, reviewParamsOf, writeReviewParams } from './review' -import type { CatalogView } from './review' +import { ChangesQueue } from './ChangesQueue' +import { ChangeDetail, ChangesOverview } from './ChangeDetail' +import { changesParamsOf, clearChanges, isChangesView, writeChangesParams, writtenTables } from './changes' +import { JobStrip, stripItems, useSourceJobs } from './JobStrip' +import type { StripItem } from './JobStrip' +import type { SourceChangeCategory, SourceChangeCounts, SourceChangeResolveOut, SourceChangeStatusFilter } from '../../typesSource' import { dueCount, filterCounts, listParamsOf, progressOf, rowFromDetail, usageSummary, visibleRows, writeListParams } from './model' import type { ListFilter, ListParams, ListSort } from './model' @@ -48,13 +53,17 @@ import type { ListFilter, ListParams, ListSort } from './model' // 清单的搜索词、筛选、排序和「只看运行中查询过的表」记在地址的查询参数里（q / filter / sort / used），刷新、前进后退 // 都不丢；在清单和表详情之间跳转时带着它们走。 // -// 左栏有两个视图：表清单（逐表看）和审阅队列（view=review：跨表按优先级列出推断项，成批确认、驳回，筛选记在 kind / -// rtable 里）。从队列点开一项，右栏是所在表的详情，地址带着那一项（item），详情滚到那一项、列出用过这张表的运行。 +// 左栏有三个视图：表清单（逐表看）、审阅队列（view=review：跨表按优先级列出推断项，成批确认、驳回，筛选记在 kind / +// rtable 里）和变化（view=changes：探查、取值观察测到的结构变化、取值变化，按要处理 / 提醒 / 说明分组，筛选记在 cstatus / +// ccat 里）。从队列点开一项，右栏是所在表的详情，地址带着那一项（item），详情滚到那一项、列出用过这张表的运行；从变化里点开 +// 一条（change），右栏是它的依据、影响面和处理。 // 页头的「导出目录」「导入目录」把整理好的目录带到另一个实例（导入先预览，确认后才写）。 // -// 起草、剖析都是服务端的后台任务（draftJob.ts、profileJob.ts），状态在这一页上：关掉对话框照常进行，页头的「起草」「数据 -// 剖析」换成进度入口（「起草 3/20」「剖析 12/40」「剖析 排队中」），点开对应的对话框；结束时右下角提示，可以回到结果。 -// 同一个数据源的起草和剖析按建立顺序排队：一个在排队、进行中时，另一个建出来是排队，前一个结束后自动开始。 +// 起草、剖析都是服务端的后台任务（draftJob.ts、profileJob.ts），状态在这一页上：关掉对话框照常进行；结束时右下角提示，可以 +// 回到结果。同一个数据源的探查、起草、剖析在同一条队里排队：一个在排队、进行中时，另一个建出来是排队，前一个结束后自动开始。 +// 进度只在页头下方的后台任务条上（JobStrip，按服务端给的队列先后列出，next-round 05 (g)）；页头的「起草」「数据剖析」只是 +// 入口，进行中换转圈、排队中换时钟，读屏名称写状态、不写数字。 +// 地址里带 pjob / djob 时打开那个剖析、起草任务的进度（定时维护的「最近一次」从数据源页链过来）。 // =========================================================================== export function CatalogPage() { @@ -135,7 +144,12 @@ export function CatalogPage() { carriedRef.current = carried /** 审阅视图：view=review 时左栏是审阅队列；kind、rtable 是队列的筛选；item 是右栏从队列打开的那一项 */ const review = useMemo(() => reviewParamsOf(params), [params]) - const inReview = review.view === 'review' + /** 变化视图：view=changes 时左栏是变化列表；cstatus、ccat 是筛选；change 是右栏打开的那一条 */ + const inChanges = isChangesView(params) + const changesParams = useMemo(() => changesParamsOf(params), [params]) + const pageView: PageView = inChanges ? 'changes' : review.view + const inReview = pageView === 'review' + const openChangeId = inChanges ? changesParams.change || null : null const [selected, setSelected] = useState>(() => new Set()) /** * 起草对话框开着没有、打开时显示什么：setup 选范围开始；job 显示页面跟着的任务（进行中的进度、刚结束的结果）。 @@ -248,12 +262,12 @@ export function CatalogPage() { const onGuidanceSaved = useCallback((g: string | null) => { setData((cur) => cur && ({ ...cur, guidance: g })) }, []) - // 上方的进度：跟着的任务在进行就按它；跟着的就是清单说的那一个、已经结束了，就不信清单（清单还没重取回来） + // 起草任务现在的样子：跟着的任务在进行就按它；跟着的就是清单说的那一个、已经结束了，就不信清单（清单还没重取回来） const listActive = isJobActive(data?.active_job) ? data?.active_job ?? null : null - const stripJob: { done: number; total: number; status: string; waiting_for?: string | null } | null = draftJob && (draftActive || draftJob.id === listActive?.id) + const stripJob: { id: string; done: number; total: number; status: string; waiting_for?: string | null } | null = draftJob && (draftActive || draftJob.id === listActive?.id) ? (draftActive ? draftJob : null) : listActive - // 调用接连没写出来的提醒：关着对话框时也要看得到（上方的进度带着它） + // 调用接连没写出来的提醒：关着对话框时也要看得到（后台任务条上起草那一行带着它） const stripNotice = draftActive ? draftJob?.notice?.trim() || null : null const options = savedOptions?.sourceId === sourceId ? savedOptions.options : source?.options @@ -288,7 +302,7 @@ export function CatalogPage() { const profileActive = isJobActive(profileJob) // 页头的剖析进度：同起草，跟着的任务在进行就按它；跟着的就是清单说的那一个、已经结束了，就不信清单 const listProfile = isJobActive(data?.active_profile_job) ? data?.active_profile_job ?? null : null - const profileStrip: { done: number; total: number; status: string; waiting_for?: string | null } | null = + const profileStrip: { id: string; done: number; total: number; status: string; waiting_for?: string | null } | null = profileJob && (profileActive || profileJob.id === listProfile?.id) ? (profileActive ? profileJob : null) : listProfile /** 跟着的剖析任务里各表的状态（表清单的「剖析」一格据此写排队、剖析中） */ const profileLive = useMemo(() => new Map( @@ -320,6 +334,70 @@ export function CatalogPage() { toast.error(e) } }, [sourceId, profile.job?.id, profile.track]) + /** 打开起草对话框显示某个任务：页面跟着的那一个直接打开；别的（变化里「重新起草」建的）先取来 */ + const openDraftJob = useCallback(async (jobId: string) => { + if (draft.job?.id === jobId) { + setDraftOpen({ view: 'job' }) + return + } + const src = sourceId + try { + const { job } = await api.dataCatalog.draftJob(src, jobId) + if (sourceRef.current !== src) return + draft.track(job) + setDraftOpen({ view: 'job' }) + } catch (e) { + toast.error(e) + } + }, [sourceId, draft.job?.id, draft.track]) + const openJob = useCallback((kind: 'draft' | 'profile', id: string) => { + if (kind === 'draft') void openDraftJob(id) + else void openProfileJob(id) + }, [openDraftJob, openProfileJob]) + // 地址里带着 pjob / djob（定时维护的「最近一次」从数据源页链过来）：打开那个任务的进度，参数用过就去掉 + const linkedJob = params.get('pjob') ? `profile:${params.get('pjob')}` : params.get('djob') ? `draft:${params.get('djob')}` : '' + const linkedSeen = useRef('') + useEffect(() => { + if (!linkedJob || linkedSeen.current === linkedJob || !data) return + linkedSeen.current = linkedJob + const [kind, id] = linkedJob.split(':') + openJob(kind as 'draft' | 'profile', id) + const next = new URLSearchParams(window.location.search) + next.delete('pjob') + next.delete('djob') + navigate({ search: next.toString() ? `?${next.toString()}` : '' }, { replace: true }) + }, [linkedJob, data, openJob, navigate]) + + /** + * 后台任务条：服务端排好的队列（探查、起草、剖析），起草、剖析换上页面跟着的任务的数和提醒。页面上的任务换了状态就重取一次 + * 队列（排在谁后面跟着变） + */ + const profileNotice = profileActive ? profileJob?.notice?.trim() || null : null + const jobsKick = `${stripJob?.id ?? ''}:${stripJob?.status ?? ''}|${profileStrip?.id ?? ''}:${profileStrip?.status ?? ''}` + const sourceJobs = useSourceJobs(sourceId, jobsKick) + const jobItems = useMemo(() => stripItems(sourceJobs.items, { + draft: stripJob ? { kind: 'draft', id: stripJob.id, status: stripJob.status, done: stripJob.done, total: stripJob.total, + waiting_for: stripJob.waiting_for ?? null, position: null, notice: stripNotice } : null, + profile: profileStrip ? { kind: 'profile', id: profileStrip.id, status: profileStrip.status, done: profileStrip.done, + total: profileStrip.total, waiting_for: profileStrip.waiting_for ?? null, position: null, notice: profileNotice } : null, + }), [sourceJobs.items, stripJob, stripNotice, profileStrip, profileNotice]) + const openStripItem = useCallback((item: StripItem) => { + if (item.kind === 'introspect') navigate(`${back}#source-${encodeURIComponent(sourceId)}`) + else openJob(item.kind, item.id) + }, [navigate, back, sourceId, openJob]) + + // 变化视图：标签上的「N 条要处理」、处理过一条之后左栏重取 + const [changeCounts, setChangeCounts] = useState<{ src: string; counts: SourceChangeCounts } | null>(null) + const [changesSeq, setChangesSeq] = useState(0) + const loadChangeCounts = useCallback(() => { + const src = sourceId + api.sourceChanges.list(src, { limit: 1 }).then((r) => setChangeCounts({ src, counts: r.counts }), () => {}) + }, [sourceId]) + useEffect(() => { loadChangeCounts() }, [loadChangeCounts]) + useOnReconnect(loadChangeCounts) + const actionCount = changeCounts?.src === sourceId ? changeCounts.counts.by_severity.action : 0 + const onChangeCounts = useCallback((counts: SourceChangeCounts) => setChangeCounts({ src: sourceId, counts }), [sourceId]) + const tableProfile: TableProfileProps = useMemo(() => ({ job: profileJob, onStart: profileTable, onShowActive: profile.showActive, onSettings: () => setProfileSettingsOpen(true), onOpenJob: (id: string | null) => void openProfileJob(id), @@ -376,10 +454,29 @@ export function CatalogPage() { const next = fn(new URLSearchParams(window.location.search)).toString() navigate({ search: next ? `?${next}` : '' }, opts) }, [navigate]) - /** 换视图（表清单 / 审阅队列）：记进地址（后退回到上一个视图），右栏正在看的表不动 */ - const setView = useCallback((v: CatalogView) => { - patchSearch((prev) => writeReviewParams(prev, { view: v, item: '' })) + /** 换视图（表清单 / 审阅队列 / 变化）：记进地址（后退回到上一个视图），右栏正在看的表不动 */ + const setView = useCallback((v: PageView) => { + patchSearch((prev) => (v === 'changes' + ? writeChangesParams(writeReviewParams(prev, { item: '' }), { open: true }) + : clearChanges(writeReviewParams(prev, { view: v, item: '' })))) }, [patchSearch]) + /** 变化视图的筛选：替换当前这条历史 */ + const setChangesFilter = useCallback((p: { status?: SourceChangeStatusFilter; category?: SourceChangeCategory | null }) => { + patchSearch((prev) => writeChangesParams(prev, p), { replace: true }) + }, [patchSearch]) + /** 打开一条变化：右栏是它，地址带着它（表详情里点「去处理」也走这里，切到变化视图） */ + const openChange = useCallback((id: string) => { + const q = writeChangesParams(clearChanges(carriedRef.current), { open: true, change: id, status: changesParams.status, + category: changesParams.category }).toString() + navigate(`${base}${q ? `?${q}` : ''}`) + }, [navigate, base, changesParams.status, changesParams.category]) + /** 窄屏：从一条变化回到变化列表 */ + const closeChange = useCallback(() => patchSearch((prev) => writeChangesParams(prev, { change: '' })), [patchSearch]) + /** 从一条变化去看它说的那张表：切回表清单 */ + const openChangeTable = useCallback((t: string) => { + const q = clearChanges(writeReviewParams(carriedRef.current, { view: 'list', item: '' })).toString() + navigate(`${base}/${encodeURIComponent(t)}${q ? `?${q}` : ''}`) + }, [navigate, base]) /** 审阅队列的筛选：替换当前这条历史（筛选不该占满后退键） */ const setReviewParams = useCallback((p: { kind?: CatalogReviewKind; table?: string }) => { patchSearch((prev) => writeReviewParams(prev, p), { replace: true }) @@ -394,6 +491,17 @@ export function CatalogPage() { setDrafted((cur) => ({ seq: cur.seq + 1, tables: new Set(tables) })) void load() }, [load]) + /** 处理过一条变化：左栏重取、计数跟着变；动了目录的重取清单、重新载入那几张表；建了任务的跟上（清单带着进行中的任务） */ + const onChangeResolved = useCallback((out: SourceChangeResolveOut) => { + setChangesSeq((n) => n + 1) + loadChangeCounts() + const tables = writtenTables(out.change, out.catalog_versions) + if (tables.length) onTablesWritten(tables) + else if (out.job) void load() + if (out.job) sourceJobs.refresh() + }, [loadChangeCounts, onTablesWritten, load, sourceJobs.refresh]) + /** 右栏有内容（打开了一张表、一条变化）：窄屏一次只显示一栏，这时显示右栏 */ + const rightOpen = !!table || !!openChangeId const activeReviewKey = inReview && table && review.item ? itemKey({ table_name: table, path: review.item }) : null const proposedTotal = useMemo(() => rows.reduce((n, r) => n + (r.in_schema ? r.counts.proposed : 0), 0), [rows]) @@ -438,36 +546,28 @@ export function CatalogPage() { {CT.draft} ) - // 页头的「数据剖析」「起草」：任务在排队、进行中时换成进度入口（「剖析 12/40」「剖析 排队中」），点开对应的对话框 - // （关掉对话框任务照常进行）；排在另一种任务后面时悬停写明。窄屏只留图标，标题才放得下（进度在读屏的名称里） - const profileQueued = profileStrip?.status === 'queued' - const profileLabel = profileStrip ? (profileQueued ? PT.headerQueued : PT.headerRunning(profileStrip.done, profileStrip.total)) : PT.action - const profileTitle = profileStrip - ? (profileQueued ? (profileStrip.waiting_for === 'draft' ? PT.headerQueuedDraft - : profileStrip.waiting_for === 'introspect' ? PT.headerQueuedIntrospect : PT.headerQueuedHint) - : PT.headerRunningHint(profileStrip.done, profileStrip.total)) - : PT.actionHint - // 最近查询多数超时的提醒：对话框关着时也要看得到（页头的入口换成提醒色，悬停写全文） - const profileNotice = profileActive ? profileJob?.notice?.trim() || null : null + // 页头的「数据剖析」「起草」只是入口（进度在下方的后台任务条上，05 (g)）：进行中换转圈图标、排队中换时钟，读屏名称写状态、 + // 不写数字；点开对应的对话框（关掉对话框任务照常进行）。窄屏只留图标，标题才放得下 + const headState = (job: { status: string } | null) => (job?.status === 'queued' ? 'queued' : job?.status === 'running' ? 'running' : null) + const headLabel = (label: string, state: 'queued' | 'running' | null) => + (state === 'running' ? JT.headRunning(label) : state === 'queued' ? JT.headQueued(label) : label) + const headIcon = (state: 'queued' | 'running' | null, idle: ReactNode) => + (state === 'running' ? : state === 'queued' ? : idle) + const profileState = headState(profileStrip) + const profileLabel = headLabel(PT.action, profileState) const headerProfile = ( - ) - const draftQueued = stripJob?.status === 'queued' - const draftLabel = stripJob ? (draftQueued ? DT.headerQueued : DT.headerRunning(stripJob.done, stripJob.total)) : CT.draft - const draftTitle = stripJob - ? (draftQueued ? (waitsForProfile(stripJob) ? DT.headerQueuedProfile - : stripJob.waiting_for === 'introspect' ? DT.headerQueuedIntrospect : DT.headerQueuedHint) - : DT.headerRunningHint(stripJob.done, stripJob.total)) - : CT.draft + const draftState = headState(stripJob) + const draftLabel = headLabel(CT.draft, draftState) const headerDraft = ( - ) @@ -495,10 +595,15 @@ export function CatalogPage() { } else { body = (
-