diff --git a/backend/app/api/source_changes.py b/backend/app/api/source_changes.py index cc95b76b..904fc00b 100644 --- a/backend/app/api/source_changes.py +++ b/backend/app/api/source_changes.py @@ -8,12 +8,26 @@ - GET /{id}/changes/{change_id} 一条变化 - POST /{id}/changes/{change_id}/resolve 处理:{action, note?, codes?}。动作落到目录已有的写入接口并记目录版本 (data/source_changes.handle);误报必须写理由(422);已经处理过的 409 -- GET /{id}/maintenance 定时维护要做哪几步、剖析开没开、指向这个源的维护定时 +- GET /{id}/maintenance 定时维护要做哪几步、剖析开没开、指向这个源的维护定时(定时任务页同一个形状,最近一次触发 + 带 inputs:建了哪些任务、哪几步没执行和原因)、调度开没开、这个源要不要维护 - PUT /{id}/maintenance 改要做哪几步:{steps: [introspect | profile_needed | observe]}。定时本身(cron、时区、开关) 在 /api/schedules 里建(target_kind=source_maintenance),这里只管步骤 -访问密钥默认不放(auth/keys.py 的 KEY_ROUTES 没有登记这几条)。界面(审阅页「变化」标签页、数据源角标、定时维护设置)在 -第 4 波。 +第 4 波(W4-d)给界面加的: + +- GET /upkeep 数据源列表一次取齐:各源要处理的变化条数(卡片角标)、指向它的维护定时(频率、开没开、 + 下一次) +- GET /{id}/observations?table= 表详情「结构与取值变化」一节:这张表最近几次观察(行数、码值分布、最新业务日期、按日 + 行数),各带口径;遮罩在读的那一刻再核 +- GET /{id}/background-jobs 这个源进行中和排队的后台任务(探查、起草、剖析),按同源队的先后排好(目录页的后台任务条, + 界面不自己排) + +变化列表、单条、处理的回答里,每条变化在 agentlab-change/1 之外另带几项只给界面看的(_views;召回拉的 list_changes 不带): +category(struct 结构 / value 取值)、basis_text(依据的口径)、actions(现在能做哪几个处理动作,做不了的写原因)、job(处理时 +建的任务,能点进进度)、影响面里每个目录项现在的状态(affects.catalog[*].now:清理过、改过的照实)。列表另能按 category 筛、 +按 status=handled 看处理过的。 + +访问密钥默认不放(auth/keys.py 的 KEY_ROUTES 没有登记这几条)。 """ from __future__ import annotations @@ -21,10 +35,11 @@ from typing import Any from fastapi import APIRouter, Depends, Header, HTTPException, Query from pydantic import BaseModel, Field +from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from app.api.deps import actor_of -from app.data import catalog, source_changes as sc, source_maintenance +from app.data import catalog, schema_diff as sd, source_changes as sc, source_maintenance from app.db.base import get_session from app.db.models import DataSource @@ -32,6 +47,14 @@ router = APIRouter(prefix="/api/datasources", tags=["source-changes"]) #: 一页最多几条 LIMIT_MAX = 200 +#: 列表的状态筛选:pending 开着的和知道了的(缺省)、handled 处理过的(解决了的、标为误报的)、all、或者某一个状态 +STATUS_FILTERS = ("pending", "handled", "all", *sc.STATUSES) +#: 变化的两类:结构变化、取值变化(叫法分开,和已有的「目录有变化」也分开,总方案 X10) +CATEGORIES = {"struct": tuple(sd.STRUCT_KINDS), "value": tuple(sc.VALUE_KINDS)} +#: 处理动作建的任务是哪一种 +_JOB_OF = {"redraft": "draft", "reprofile": "profile"} +#: 关联覆盖率下降的依据(剖析核对关系时抽样) +RELATION_BASIS = "关联关系抽样核对" async def _source(session: AsyncSession, source_id: str) -> DataSource: @@ -41,19 +64,128 @@ async def _source(session: AsyncSession, source_id: str) -> DataSource: return row +# ---- 给界面的几样(要登记在 /{source_id}/… 前面:/upkeep 只有一段) + + +@router.get("/upkeep") +async def upkeep(session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """数据源列表一次取齐:{items: [{source_id, changes: {pending, open, by_severity}, maintenance: [{id, enabled, cron, + timezone, description_text, next_run_at}]}]}。卡片角标「N 个变化要处理」按 changes.by_severity.action;没有变化、没有 + 维护定时的源也列出来(计数为 0、maintenance 为空)。""" + from app.db.models import Schedule + from app.scheduler import service + + ids = list((await session.execute(select(DataSource.id))).scalars()) + counts = await sc.counts_by_source(session, ids) + schedules: dict[str, list[dict[str, Any]]] = {} + for r in (await session.execute(select(Schedule).where(Schedule.target_kind == "source_maintenance") + .order_by(Schedule.created_at))).scalars(): + schedules.setdefault(r.target_id or "", []).append({ + "id": r.id, "enabled": r.enabled, "cron": r.cron, "timezone": r.timezone, + "description_text": _describe(r.cron, r.timezone), + "next_run_at": service.as_utc(r.next_run_at).isoformat() if r.next_run_at else None}) + empty = {"pending": 0, "open": 0, "by_severity": dict.fromkeys(sc.SEVERITIES, 0)} + return {"items": [{"source_id": i, "changes": counts.get(i, empty), "maintenance": schedules.get(i, [])} + for i in ids]} + + +def _describe(cron_expr: str, tz: str) -> str: + from app.scheduler import cron + + try: + return cron.describe(cron_expr, tz) + except cron.CronError: + return f"按 cron 表达式 {cron_expr}({tz})" + + +def _basis_text(row: Any) -> str | None: + """依据的口径(取值变化):整表、业务日期最近 N 天的窗口、块采样、按存储顺序的前 N 行、统计估算……照观察时记下的写。 + 结构变化的依据是探查读到的结构版本(前后各一版),界面照 struct_version_before / after 写,这里为 None。""" + from app.data import catalog_profile_observe as observe + + if row.kind in sd.STRUCT_KINDS: + return None + basis = row.basis if isinstance(row.basis, dict) else {} + if basis.get("method") == "relation_sample": + return RELATION_BASIS + return observe.key_text(basis.get("key")) or (observe.basis_text(basis) if basis.get("method") else None) + + +async def _views(session: AsyncSession, source: DataSource, rows: list[Any]) -> list[dict[str, Any]]: + """接口给界面的变化:change_out 之外另带 category、basis_text、actions、job、影响面里每个目录项现在的状态(见模块说明)。""" + from app.data import catalog_profile, table_versions + + if not rows: + return [] + try: + view = await table_versions.resolve_source(session, source) + except table_versions.SnapshotMissing: + view = source + tables = set(((getattr(view, "schema_cache", None) or {}).get("tables") or {})) + settings = catalog_profile.profile_settings(source.options) + profile_reason = None + if not settings.enabled: + try: + catalog_profile.ensure_enabled(source) + except catalog_profile.ProfileDisabled as e: + profile_reason = str(e) + entries = await catalog.read_catalog(session, source.id) if any( + (r.affects or {}).get("catalog") for r in rows) else {} + slots = {t: catalog._slots(e.notes) for t, e in entries.items()} + + def now_of(ref: dict[str, Any]) -> dict[str, Any] | None: + try: + item = slots.get(str(ref.get("table") or ""), {}).get(catalog.parse_path(str(ref.get("path") or ""))) + except catalog.CatalogPathError: + return None + return {"status": item.get("status"), "source": item.get("source")} if item else None + + out: list[dict[str, Any]] = [] + for r in rows: + v = sc.change_out(r) + v["category"] = "struct" if r.kind in sd.STRUCT_KINDS else "value" + v["basis_text"] = _basis_text(r) + v["actions"] = sc.available_actions(r, in_schema=bool(r.table_name) and r.table_name in tables, + profile_enabled=settings.enabled, profile_reason=profile_reason) + resolution = r.resolution or {} + kind = _JOB_OF.get(str(resolution.get("action") or "")) + v["job"] = {"kind": kind, "id": resolution["job_id"]} if kind and resolution.get("job_id") else None + affects = dict(v["affects"]) + affects["catalog"] = [{**ref, "now": now_of(ref)} for ref in affects.get("catalog") or [] + if isinstance(ref, dict)] + v["affects"] = affects + out.append(v) + return out + + @router.get("/{source_id}/changes") async def list_changes(source_id: str, status: str = Query("pending"), kind: list[str] | None = Query(None), - table: str | None = Query(None), limit: int = Query(100, ge=1, le=LIMIT_MAX), - offset: int = Query(0, ge=0), session: AsyncSession = Depends(get_session)) -> dict[str, Any]: - """{items: [变化(agentlab-change/1)], total, counts: {pending, open, by_severity}}。""" - if status not in ("pending", "all", *sc.STATUSES): - raise HTTPException(422, f"不认识的状态:{status}。可选:pending、all、{'、'.join(sc.STATUSES)}") + category: str | None = Query(None), table: str | None = Query(None), + limit: int = Query(100, ge=1, le=LIMIT_MAX), offset: int = Query(0, ge=0), + session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """{items: [变化(agentlab-change/1,另带给界面的几项)], total, counts: {pending, open, by_severity}}。 + + status=pending(缺省)按「使用次数权重 × 严重程度」排,同分新的在前;status=handled 是处理过的,最近测到的在前。 + category=struct / value 只看结构变化 / 取值变化(和 kind 一起给时取两者都对得上的;一种都对不上就是空的)。""" + if status not in STATUS_FILTERS: + raise HTTPException(422, f"不认识的状态:{status}。可选:{'、'.join(STATUS_FILTERS)}") if kind and (bad := [k for k in kind if k not in sc.KINDS]): raise HTTPException(422, f"不认识的变化种类:{'、'.join(bad)}") + if category is not None and category not in CATEGORIES: + raise HTTPException(422, f"不认识的变化类别:{category}。可选:struct(结构变化)、value(取值变化)") row = await _source(session, source_id) - rows, total = await sc.list_for_source(session, row, status=status, kinds=kind, table=table, limit=limit, - offset=offset) - return {"items": [sc.change_out(r) for r in rows], "total": total, "counts": await sc.counts(session, row.id)} + kinds = list(kind) if kind else None + if category is not None: + kinds = [k for k in (kinds or CATEGORIES[category]) if k in CATEGORIES[category]] + counts = await sc.counts(session, row.id) + if kinds == []: + return {"items": [], "total": 0, "counts": counts} + if status == "handled": + rows, total = await sc.list_handled(session, row.id, kinds=kinds, table=table, limit=limit, offset=offset) + else: + rows, total = await sc.list_for_source(session, row, status=status, kinds=kinds, table=table, limit=limit, + offset=offset) + return {"items": await _views(session, row, rows), "total": total, "counts": counts} @router.get("/{source_id}/changes/{change_id}") @@ -62,7 +194,7 @@ async def get_change(source_id: str, change_id: str, session: AsyncSession = Dep found = await sc.load(session, row.id, change_id) if found is None: raise HTTPException(404, "这条变化不存在,可能已随数据源删除") - return {"change": sc.change_out(found)} + return {"change": (await _views(session, row, [found]))[0]} class ResolveIn(BaseModel): @@ -83,8 +215,8 @@ async def resolve_change(source_id: str, change_id: str, payload: ResolveIn, x_a if found is None: raise HTTPException(404, "这条变化不存在,可能已随数据源删除") try: - return await sc.handle(session, row, found, payload.action, actor=actor_of(x_actor), note=payload.note, - codes=payload.codes) + out = await sc.handle(session, row, found, payload.action, actor=actor_of(x_actor), note=payload.note, + codes=payload.codes) except sc.ChangeInvalid as e: await session.rollback() raise HTTPException(422, str(e)) from e @@ -94,18 +226,42 @@ async def resolve_change(source_id: str, change_id: str, payload: ResolveIn, x_a except catalog.CatalogConflict as e: await session.rollback() raise HTTPException(409, "这张表的数据目录刚被别人改过,请刷新后再处理") from e + # 处理完照实给这条变化现在的样子(另带给界面的几项):目录项清理过的、动作做完的都跟着变 + await session.refresh(row) + fresh = await sc.load(session, row.id, change_id) + if fresh is not None: + out["change"] = (await _views(session, row, [fresh]))[0] + return out @router.get("/{source_id}/maintenance") async def get_maintenance(source_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: - """{steps, profile_enabled, schedules}:要做哪几步、剖析开没开(没开时剖析、观察两步到点照实记没执行)、指向这个源的 - 维护定时。""" + """{steps, profile_enabled, maintained, scheduler_on, schedules}:要做哪几步、剖析开没开(没开时剖析、观察两步到点照实记 + 没执行)、这个源要不要维护(上传表格、记录表不要)、调度开没开(关着时到点不会触发)、指向这个源的维护定时。 + + 定时是定时任务页同一个形状(api/schedules 的 Schedule:改频率、开关照样带 version 走 PATCH /api/schedules/{id}),最近 + 一次触发另带 inputs({steps, jobs: [{step, kind, id, tables}], skipped: [{step, reason}]}:建了哪些任务、哪几步没执行和原因)。""" + from app import scheduler + from app.api.schedules import _schedule_outs from app.data import catalog_profile + from app.db.models import Schedule, ScheduleFire row = await _source(session, source_id) + rows = list((await session.execute(select(Schedule).where( + Schedule.target_kind == "source_maintenance", Schedule.target_id == row.id) + .order_by(Schedule.created_at))).scalars()) + outs = await _schedule_outs(session, rows) + fire_ids = [o["last_fire"]["id"] for o in outs if o.get("last_fire")] + inputs = dict((await session.execute(select(ScheduleFire.id, ScheduleFire.inputs).where( + ScheduleFire.id.in_(fire_ids)))).all()) if fire_ids else {} + for o in outs: + if o.get("last_fire"): + o["last_fire"]["inputs"] = dict(inputs.get(o["last_fire"]["id"]) or {}) return {"steps": source_maintenance.steps_of(row.options), "profile_enabled": catalog_profile.profile_settings(row.options).enabled, - "schedules": await source_maintenance.schedules_for(session, row.id)} + "maintained": (row.origin or "manual") not in ("upload", "records"), + "scheduler_on": scheduler.scheduler_enabled(), + "schedules": outs} class MaintenanceIn(BaseModel): @@ -123,3 +279,41 @@ async def put_maintenance(source_id: str, payload: MaintenanceIn, x_actor: str | row.options = source_maintenance.with_steps(row.options, [str(s) for s in payload.steps], actor=actor_of(x_actor)) await session.commit() return await get_maintenance(source_id, session) + + +@router.get("/{source_id}/observations") +async def table_observations(source_id: str, table: str = Query(..., min_length=1, max_length=500), + session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """表详情「结构与取值变化」一节:{table, series: [{kind, column, points: [{at, basis_key, basis_text, data}], + basis_changed}]}(catalog_profile_observe.recent)。现在被遮罩的列不给;没观察过是空的。""" + from app.data import catalog_profile, catalog_profile_observe as observe + + row = await _source(session, source_id) + masked = catalog_profile.masked_of(row.options) + return {"table": table, "series": await observe.recent(session, row.id, table, masked=masked)} + + +@router.get("/{source_id}/background-jobs") +async def background_jobs(source_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """这个源进行中和排队的后台任务(目录页的后台任务条,05 (g)):{items: [{kind: introspect | draft | profile, id, status, + total, done, waiting_for, position}]}。 + + 按同源队(core/job_runtime.lanes)的先后排:position 是在队里的位置(0 是正在做的);刚建、后台还没排进队的(position 为 + null)排在队里的后面、按建的先后。waiting_for 是各任务自己的说法(排在谁后面),界面照写,不自己排。""" + from app.core import job_runtime as runtime + from app.data import catalog_jobs, catalog_profile_jobs, introspect_jobs + + row = await _source(session, source_id) + found: list[tuple[int | None, Any, dict[str, Any]]] = [] + for kind, module in (("introspect", introspect_jobs), ("draft", catalog_jobs), ("profile", catalog_profile_jobs)): + job = await module.active_job(session, row.id) + brief = await module.job_brief(session, job) if job is not None else None + if job is None or brief is None: + continue + ahead = runtime.lanes.ahead(row.id, job.id) + position = len(ahead) if ahead is not None else None + found.append((position, job.created_at, { + "kind": kind, "id": job.id, "status": brief["status"], "total": brief.get("total", 0), + "done": brief.get("done", 0), "waiting_for": brief.get("waiting_for"), "position": position})) + found.sort(key=lambda f: (f[0] is None, f[0] if f[0] is not None else 0, f[1])) + return {"items": [f[2] for f in found]} diff --git a/backend/app/data/catalog_profile_observe.py b/backend/app/data/catalog_profile_observe.py index 044355e1..78b07a31 100644 --- a/backend/app/data/catalog_profile_observe.py +++ b/backend/app/data/catalog_profile_observe.py @@ -516,3 +516,87 @@ async def observed_facts(session: AsyncSession, source_id: str, __all__ = ["KIND_LABEL", "OBSERVE_CODES_MAX", "OBSERVE_EVERY_H", "OBSERVE_KINDS", "OBSERVER", "SAMPLE_LADDER", "WINDOW_DAYS", "basis_key", "basis_text", "code_columns", "coverage_drops", "execute", "last_observed", "observe_reason", "observed_facts", "plan_observations", "record_coverage_drops", "sample_percent", "save"] + + +# ========================================================================== +# 给人看的(第 4 波 W4-d:表详情「结构与取值变化」一节、变化的依据) +# ========================================================================== + +#: 表详情里每种观察给最近几次 +RECENT_SHOWN = 6 + + +def key_text(key: str | None) -> str | None: + """口径的规范写法(basis_key)→ 说法:变化记录里只存了规范写法,依据的口径照它写。认不出的为 None。""" + text = str(key or "") + head, _, rest = text.partition(":") + if head == "window": + column, _, days = rest.rpartition(":") + return basis_text({"method": "window", "column": column, "days": days}) if column and days else None + if head == "sample": + try: + return basis_text({"method": "sample", "percent": float(rest)}) + except ValueError: + return None + if head == "first_rows" and rest: + return basis_text({"method": "first_rows", "rows": rest}) + if text in ("full", "stats", "count"): + return basis_text({"method": text}) + return None + + +def _shown(kind: str, data: Mapping[str, Any]) -> dict[str, Any]: + """一次观察给界面的样子:码值按行数从多到少、带占比;按日行数按日期排成列表;其余原样。数都由这里算,界面只照写。""" + if kind == "codes_observe": + pairs = [(str(p[0]), int(p[1])) for p in data.get("values") or [] + if isinstance(p, (list, tuple)) and len(p) == 2 and isinstance(p[1], (int, float))] + rows = int(data.get("rows") or sum(n for _, n in pairs)) + pairs.sort(key=lambda p: (-p[1], p[0])) + return {"values": [[c, n, round(n / rows, 4) if rows else 0.0] for c, n in pairs], "rows": rows, + "truncated": bool(data.get("truncated"))} + if kind == "daily_volume": + days = data.get("days") if isinstance(data.get("days"), Mapping) else {} + counts = {str(d)[:10]: int(n) for d, n in days.items() if isinstance(n, (int, float))} + # 窗口里没数据的日子补 0:只列有数据的日子,断了的那几天在图上就看不出来 + try: + first, last = date.fromisoformat(str(data.get("since"))[:10]), date.fromisoformat(str(data.get("until"))[:10]) + except ValueError: + first = last = None + if first and last and 0 <= (last - first).days <= WINDOW_DAYS + 7: + for i in range((last - first).days + 1): + counts.setdefault((first + timedelta(days=i)).isoformat(), 0) + series = sorted(counts.items()) + return {"since": data.get("since"), "until": data.get("until"), "days": [list(p) for p in series], + "total": sum(n for _, n in series), "days_with_data": sum(1 for _, n in series if n > 0)} + if kind == "freshness": + return {"max": data.get("max")} + return {"rows": data.get("rows"), "at_least": bool(data.get("at_least")), "method": data.get("method")} + + +async def recent(session: AsyncSession, source_id: str, table: str, *, masked: set[str], + per: int = RECENT_SHOWN) -> list[dict[str, Any]]: + """这张表最近几次观察:按(种类、列)分组,种类按 OBSERVE_KINDS 的顺序、同种按列名;每组最近 per 次,新的在前: + [{kind, column, points: [{at, basis_key, basis_text, data}], basis_changed}]。basis_changed:这几次的口径不全相同(口径不同 + 不比,界面照实说)。遮罩在读的这一刻再核(look D6 的同一个道理):观察时还没遮、现在被遮罩的列整组不给。""" + rows = (await session.execute(select(TableObservation).where( + TableObservation.source_id == source_id, TableObservation.table_name == table) + .order_by(TableObservation.at.desc(), TableObservation.id.desc()))).scalars() + groups: dict[tuple[str, str], list[TableObservation]] = {} + for r in rows: + if r.kind not in OBSERVE_KINDS or (r.column_name and r.column_name.lower() in masked): + continue + group = groups.setdefault((r.kind, r.column_name or ""), []) + if len(group) < per: + group.append(r) + order = {k: i for i, k in enumerate(OBSERVE_KINDS)} + out: list[dict[str, Any]] = [] + for (kind, column), group in sorted(groups.items(), key=lambda kv: (order[kv[0][0]], kv[0][1])): + out.append({"kind": kind, "column": column, + "points": [{"at": r.at.isoformat() if r.at else None, "basis_key": r.basis_key, + "basis_text": basis_text(r.basis or {}), "data": _shown(kind, r.data or {})} + for r in group], + "basis_changed": len({r.basis_key for r in group}) > 1}) + return out + + +__all__ += ["RECENT_SHOWN", "key_text", "recent"] diff --git a/backend/app/data/source_changes.py b/backend/app/data/source_changes.py index fe95b0af..8edf71af 100644 --- a/backend/app/data/source_changes.py +++ b/backend/app/data/source_changes.py @@ -538,3 +538,84 @@ __all__ = [ "record", "record_all", "resolve", "summary", "triggers_recall", "without_paths", ] + + +# ========================================================================== +# 给界面的(第 4 波 W4-d:审阅页「变化」标签页、数据源列表的角标) +# ========================================================================== + +#: 处理过的:解决了的、标为误报的(「变化」标签页的「处理过的」) +HANDLED = ("resolved", "dismissed") +#: 重新起草有用的结构变化:新表、列和键变了(起草按新的结构重写这张表的推断项;人工确认过的项照旧不动) +_REDRAFT_KINDS = frozenset({sd.TABLE_ADDED, sd.COLUMN_ADDED, sd.COLUMN_REMOVED, sd.COLUMN_TYPE, sd.COLUMN_NULLABLE, + sd.KEY, sd.FK, sd.COMMENT}) +#: 这张表现在不在表结构里、或者说的是整个源的:起草、剖析都无从做起 +_NO_TABLE_KINDS = frozenset({sd.TABLE_REMOVED, sd.TABLE_LEFT_SCOPE, sd.TABLE_REFLECT_FAILED, sd.SCOPE}) +NOT_IN_SCHEMA = "表结构里已经没有这张表,先重新探查结构" +NO_CATALOG_ITEMS = "影响面里没有目录项,不需要清理" + + +def available_actions(row: Any, *, in_schema: bool, profile_enabled: bool, profile_reason: str | None = None, + ) -> list[dict[str, Any]]: + """这条变化现在能做哪几个处理动作(「变化」标签页照它画按钮,界面不自己判):[{action, ok, reason}],按 ACTIONS 的顺序。 + + 已经处理过的(resolved / dismissed)一个都不给。只列对这种变化有意义的动作:加入码值只给「出现了新码值」,重新起草只给 + 新表和列、键的变化,重新剖析给说的是一张还在的表的变化(注释变了除外);清理目录项只在影响面里有目录项时能点。前置核对 + 和 handle 一致;建任务时才知道的(起草模型、同源正忙)不在这里判,点了照实 422。ok 为假时 reason 写为什么做不了。""" + if row.status in HANDLED: + return [] + kind = row.kind + has_table = bool(row.table_name) and kind not in _NO_TABLE_KINDS + out: list[dict[str, Any]] = [] + + def put(action: str, reason: str | None = None) -> None: + out.append({"action": action, "ok": reason is None, "reason": reason}) + + if (row.affects or {}).get("catalog"): + put("clean_catalog", None if _catalog_paths(row) else NO_CATALOG_ITEMS) + if kind == CODES_NEW and row.column_name: + put("add_codes") + if has_table and kind in _REDRAFT_KINDS: + put("redraft", None if in_schema else NOT_IN_SCHEMA) + if has_table and kind != sd.COMMENT: + reason = None if in_schema else NOT_IN_SCHEMA + if reason is None and not profile_enabled: + reason = profile_reason or "这个数据源未开启数据剖析" + put("reprofile", reason) + if row.status == "open": + put("acknowledge") + put("dismiss") + return out + + +async def counts_by_source(session: AsyncSession, source_ids: Iterable[str] | None = None) -> dict[str, dict[str, Any]]: + """各源还要人看的变化(同 counts 的形状),一条查询取齐(数据源列表的角标)。没有变化的源不在里面。""" + query = select(SourceChange.source_id, SourceChange.status, SourceChange.severity, func.count()).where( + SourceChange.status.in_(PENDING)).group_by(SourceChange.source_id, SourceChange.status, SourceChange.severity) + if source_ids is not None: + query = query.where(SourceChange.source_id.in_(list(source_ids))) + out: dict[str, dict[str, Any]] = {} + for source_id, status, severity, n in (await session.execute(query)).all(): + one = out.setdefault(source_id, {"pending": 0, "open": 0, "by_severity": dict.fromkeys(SEVERITIES, 0)}) + one["by_severity"][severity] = one["by_severity"].get(severity, 0) + n + one["pending"] += n + if status == "open": + one["open"] += n + return out + + +async def list_handled(session: AsyncSession, source_id: str, *, kinds: Iterable[str] | None = None, + table: str | None = None, limit: int = 100, offset: int = 0) -> tuple[list[SourceChange], int]: + """处理过的变化(解决了的、标为误报的),最近测到的在前:处理过的不再按要不要先处理排。返回 (这一页, 一共几条)。""" + filt = [SourceChange.source_id == source_id, SourceChange.status.in_(HANDLED)] + if kinds: + filt.append(SourceChange.kind.in_(list(kinds))) + if table: + filt.append(SourceChange.table_name == table) + total = int((await session.execute(select(func.count()).select_from(SourceChange).where(*filt))).scalar() or 0) + rows = list((await session.execute(select(SourceChange).where(*filt).order_by(SourceChange.seq.desc()) + .offset(offset).limit(limit))).scalars()) + return rows, total + + +__all__ += ["HANDLED", "available_actions", "counts_by_source", "list_handled"] diff --git a/backend/app/notify/service.py b/backend/app/notify/service.py index 98dd098e..6ec395f5 100644 --- a/backend/app/notify/service.py +++ b/backend/app/notify/service.py @@ -70,7 +70,7 @@ EVENTS: dict[str, Event] = { "review.done": Event("我提交审核的应用有结果", ("global", "user")), # ---- 本轮登记的(总方案 X6):scopes 为空的还没有推送;实现推送的那一块只改自己这一行的 scopes,消息照模块说明的规矩 ---- # 数据源有要处理的结构 / 取值变化(03 B3):只推 severity 为 action 的,按源汇总;源名、各级别条数、链接 - "source.changed": Event("数据源有变化要处理", ()), + "source.changed": Event("数据源有变化要处理", ("global",)), # 口径升版后召回重算,数真变了(02 第 7 项,notify_recall):答案的主人有 recall 待办;推全局渠道和答案主人自己的个人 # 渠道(召回运行不归哪个应用)。口径名、版本、几个答案、链接,不放新旧值和差值 "ledger.recall": Event("口径升版,答案的数变了", ("global", "user")), @@ -509,7 +509,7 @@ async def notify_source_changed(source_id: str) -> int: """数据源有要处理的变化(source.changed,03 B3 ④、目录-D9):只推还要人看(开着、知道了还没处理)的「要处理」级变化, 按源汇总;消息里只有源名、各级别条数和链接,不放表名、列名、取值。探查、剖析任务结束时有新的「要处理」变化才调。 - 这一行的 scopes 第 4 波有了「变化」标签页才打开:现在 _channels 推不出去,这里返回 0(照实:没有界面看不了的通知不发)。""" + 只推全局渠道(按源汇总,不归哪个人);链接落到数据目录页的「变化」标签页(第 4 波 W4-d 有了这个标签页才打开订阅)。""" from app.data import source_changes from app.db.models import DataSource @@ -529,7 +529,7 @@ async def notify_source_changed(source_id: str) -> int: lines = [("数据源", f"「{row.name}」"), ("要处理", f"{by['action']} 条"), ("提醒", f"{by.get('warning', 0)} 条"), ("说明", f"{by.get('info', 0)} 条")] message = {"event": event, "status": "action", "title": f"数据源「{row.name[:80]}」有 {by['action']} 个变化要处理", - "lines": lines, "links": [("去看看", f"{base}/data/catalog/{source_id}")] if base else [], + "lines": lines, "links": [("去看看", f"{base}/data/catalog/{source_id}?view=changes")] if base else [], "subject": {"source_id": source_id}} return await _push(channels, message, subject_kind="source", subject_id=source_id, event=event, run_id=None, task_id=None) diff --git a/backend/tests/test_notify_events.py b/backend/tests/test_notify_events.py index cc8ec89e..f758c9e5 100644 --- a/backend/tests/test_notify_events.py +++ b/backend/tests/test_notify_events.py @@ -39,7 +39,8 @@ PUSHED_THIS_ROUND = { "eval.failed": {"global"}, "task.digest": {"user"}, "key.expiring": {"global", "user"}, - # 第 4 波:W4-d 打开 source.changed 时在这里加它推给的几条路 + # 第 4 波 W4-d:数据源有变化要处理,按源汇总、只推全局渠道(notify_source_changed);链接落到「变化」标签页 + "source.changed": {"global"}, # (两行之间隔一行不动的,两个包各改各的、合并不撞) # 第 4 波 W4-a:口径召回数真变了的待办推全局渠道和答案主人自己的个人渠道(notify_recall) "ledger.recall": {"global", "user"}, diff --git a/backend/tests/test_source_change_terms.py b/backend/tests/test_source_change_terms.py new file mode 100644 index 00000000..6b8b2885 --- /dev/null +++ b/backend/tests/test_source_change_terms.py @@ -0,0 +1,44 @@ +"""变化的叫法前后端一致(总方案 X10:一种说法)。 + +服务端给人看的一句(source_changes.summary)用 KIND_LABEL,前端「变化」标签页、表详情的那一节用 lib/termsSource.ts 的 +SOURCE_CHANGE_KIND_LABEL:两边逐字对照,一边改了另一边没跟上,同一条变化在接口和界面上就是两种说法。写法同 +test_notify_events.py 对照通知事件的叫法。 +""" +from __future__ import annotations + +import re +from pathlib import Path + +import app.db.models # noqa: F401 - 模型都登记上(source_objects 的外键指向 data_sources) +from app.data import source_changes as sc +from app.data import source_maintenance as sm + +TERMS = Path(__file__).resolve().parents[2] / "frontend" / "src" / "lib" / "termsSource.ts" + + +def _block(source: str, head: str) -> str: + start = source.index(head) + start = source.index("{", start) + 1 + depth, i = 1, start + while depth: + depth += {"{": 1, "}": -1}.get(source[i], 0) + i += 1 + return source[start:i - 1] + + +def _entries(block: str) -> dict[str, str]: + return dict(re.findall(r"'?([a-z_]+)'?:\s*'([^']*)'", block)) + + +def test_change_kind_labels_match_word_for_word(): + source = TERMS.read_text(encoding="utf-8") + labels = _entries(_block(source, "export const SOURCE_CHANGE_KIND_LABEL")) + assert labels == sc.KIND_LABEL, "「变化」标签页的叫法和服务端的对不上" + assert set(labels) == set(sc.KINDS) + + +def test_maintenance_step_labels_match_word_for_word(): + source = TERMS.read_text(encoding="utf-8") + block = _block(source, "export const MAINTENANCE_TEXT") + steps = _entries(_block(block, "step:")) + assert steps == sm.STEP_LABEL, "定时维护的步骤叫法和服务端的对不上(没执行的原因里用的是服务端那一份)" diff --git a/backend/tests/test_source_changes.py b/backend/tests/test_source_changes.py index 1eb0cf60..95ed16ff 100644 --- a/backend/tests/test_source_changes.py +++ b/backend/tests/test_source_changes.py @@ -8,7 +8,7 @@ source_changes.record 记一条。守的是: apply_patch(含义待填写、依据写变化编号)、记目录版本;知道了只改状态;剖析没开时「重新剖析」照实 422; - 同一个变化再次测到只更新 last_seen_at、seen_count; - list_changes 按 seq 游标拉、affecting 按表反查、triggers_recall 只认结构类和触及规则的新码值; -- source.changed 的推送:订阅还没打开时推不出去;打开后(测试里临时打开)消息里只有源名、条数和链接。 +- source.changed 的推送:没有渠道订阅时推不出去;全局渠道订阅了,消息里只有源名、条数和链接(第 4 波打开了订阅)。 """ from __future__ import annotations @@ -194,7 +194,7 @@ async def test_cursor_pull_affecting_and_recall_triggers(shop): async def test_source_changed_message_has_only_name_counts_and_link(shop, monkeypatch): sid, name = shop["id"], shop["name"] - assert await service.notify_source_changed(sid) == 0, "订阅还没打开(第 4 波):推不出去" + assert await service.notify_source_changed(sid) == 0, "没有渠道订阅它:推不出去" got: list[httpx.Request] = [] def handler(request: httpx.Request) -> httpx.Response: @@ -203,7 +203,6 @@ async def test_source_changed_message_has_only_name_counts_and_link(shop, monkey monkeypatch.setattr(core, "_transport", httpx.MockTransport(handler)) monkeypatch.setattr(core, "RETRY_DELAYS", (0, 0, 0)) - monkeypatch.setitem(service.EVENTS, "source.changed", service.Event("数据源有变化要处理", ("global",))) async with SessionLocal() as session: from app.core.crypto import encrypt from app.db.models import NotifyChannel diff --git a/backend/tests/test_source_upkeep.py b/backend/tests/test_source_upkeep.py new file mode 100644 index 00000000..0aaab038 --- /dev/null +++ b/backend/tests/test_source_upkeep.py @@ -0,0 +1,363 @@ +"""数据源变化、定时维护、后台任务条的界面接口(next-round 03 B2–B4、05 (g),第 4 波 W4-d)。 + +第 3 波的变化接口只给了 agentlab-change/1 的原样;界面要的几样由服务端给,界面不自己判(界面层不放逻辑): + +- 变化列表每条另带 category(结构 / 取值)、basis_text(依据的口径,照实写整表 / 窗口 / 块采样 / 前 N 行)、actions + (现在能做哪几个处理动作、做不了的写原因)、job(处理时建的任务)、影响面里每个目录项现在的状态(清理过的照实为空); +- 列表能按 category 筛、能看「处理过的」(解决了的、标为误报的,最近的在前); +- GET /upkeep:数据源列表一次取齐各源要处理的变化条数(角标)和定时维护的频率; +- GET /{id}/observations:表详情「结构与取值变化」一节的最近几次观察,各带口径;遮罩在读的那一刻再核; +- GET /{id}/background-jobs:这个源进行中和排队的后台任务(探查、起草、剖析),按同源队的先后排好; +- GET /maintenance:定时维护的定时任务是定时任务页同一个形状,最近一次触发带上 inputs(建了哪些任务、哪几步没执行和原因), + 另说调度开没开; +- source.changed 打开订阅(只推全局渠道),链接落到目录页的「变化」标签页。 + +SQLite 夹具是虚构的店铺库;名字、数字都是假的。 +""" +from __future__ import annotations + +import json +import sqlite3 +import uuid +from datetime import timedelta + +import httpx +import pytest +from httpx import ASGITransport, AsyncClient +from sqlalchemy import delete, select + +from app.core import job_runtime as runtime +from app.data import catalog, source_changes as sc +from app.db.base import SessionLocal, utcnow +from app.db.models import CatalogDraftJob, CatalogProfileJob, DataSource, NotifyChannel, Schedule, ScheduleFire +from app.db.models_sources import IntrospectJob, SourceChange, TableObservation +from app.main import app +from app.notify import core, service +from app.scheduler import service as scheduler_service +from tests.fixtures.sources import drop_source, introspect_now + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +@pytest.fixture +async def shop(client, tmp_path): + """探查建基线、写目录(status 列上有人工确认的码值和中文名),删掉 status 列再探查:得到一条要处理的 column_removed。""" + path = tmp_path / "shop.db" + db = sqlite3.connect(path) + db.executescript(""" + CREATE TABLE orders (id INTEGER PRIMARY KEY, status VARCHAR(10), channel VARCHAR(8), amount NUMERIC(10, 2)); + INSERT INTO orders VALUES (1, 'paid', 'web', 3), (2, 'void', 'shop', 4); + """) + db.commit() + db.close() + name = f"upk_{uuid.uuid4().hex[:8]}" + r = await client.post("/api/datasources", json={"name": name, "kind": "sqlite", "database": str(path), + "readonly": True}) + sid = r.json()["id"] + await introspect_now(client, sid) + async with SessionLocal() as session: + await catalog.write_entry(session, sid, "orders", { + "label": catalog.make_item("订单", "human"), + "columns": {"status": {"codes": catalog.make_item({"paid": "已支付", "void": "作废"}, "human"), + "label": catalog.make_item("状态", "human")}, + "channel": {"codes": catalog.make_item({"web": "线上"}, "profile", "proposed")}}, + }, if_version=0, actor="测试员") + db = sqlite3.connect(path) + db.execute("ALTER TABLE orders DROP COLUMN status") + db.commit() + db.close() + await introspect_now(client, sid) + made: list[str] = [] + yield {"id": sid, "name": name, "schedules": made} + async with SessionLocal() as session: + await session.execute(delete(Schedule).where(Schedule.id.in_(made))) + await session.commit() + await drop_source(sid) + + +async def _codes_new(sid: str) -> SourceChange: + async with SessionLocal() as session: + row = await session.get(DataSource, sid) + change, _new = await sc.record(session, row, sc.NewChange( + kind=sc.CODES_NEW, table="orders", column="channel", severity="warning", + after={"codes": ["app"], "shares": {"app": 0.4}}, basis={"method": "first_rows", "key": "first_rows:100000"}, + affects={"catalog": [], "templates": [], "rules_touched": False}, key=("first_rows:100000", ("app",))), + sc.Detected(job_kind="profile", job_id="job-upk", trigger="schedule")) + await session.commit() + return change + + +def _by_kind(items: list[dict]) -> dict[str, dict]: + return {c["kind"]: c for c in items} + + +# -------------------------------------------------------------------------- +# 变化列表:界面要的几样由服务端给 +# -------------------------------------------------------------------------- + + +async def test_changes_carry_category_basis_actions_and_current_status(client, shop): + sid = shop["id"] + await _codes_new(sid) + body = (await client.get(f"/api/datasources/{sid}/changes")).json() + by = _by_kind(body["items"]) + removed, fresh = by["column_removed"], by["codes_new"] + assert removed["category"] == "struct" and fresh["category"] == "value" + assert fresh["basis_text"] == "按存储顺序取前 100000 行", "依据的口径照实写" + assert removed["basis_text"] is None, "结构变化的依据是探查读到的结构版本(界面照 struct_version 写)" + + acts = {a["action"]: a for a in removed["actions"]} + assert list(acts) == ["clean_catalog", "redraft", "reprofile", "acknowledge", "dismiss"], acts + assert acts["clean_catalog"]["ok"] and acts["redraft"]["ok"] and acts["dismiss"]["ok"] + assert not acts["reprofile"]["ok"] and "剖析" in acts["reprofile"]["reason"], "剖析没开:照实写做不了的原因" + assert "add_codes" not in acts, "只有出现了新码值才给「加入码值」" + assert [a["action"] for a in fresh["actions"]][:1] == ["add_codes"] + + refs = removed["affects"]["catalog"] + assert refs and all(r["now"]["status"] == "confirmed" for r in refs if r["path"].startswith("columns.status")) + assert removed["job"] is None + + +async def test_category_and_handled_filters(client, shop): + sid = shop["id"] + change = await _codes_new(sid) + url = f"/api/datasources/{sid}/changes" + value = (await client.get(url, params={"category": "value"})).json() + assert [c["kind"] for c in value["items"]] == ["codes_new"] and value["total"] == 1 + struct = (await client.get(url, params={"category": "struct"})).json() + assert struct["items"] and all(c["category"] == "struct" for c in struct["items"]) + none = (await client.get(url, params={"category": "value", "kind": ["column_removed"]})).json() + assert none["items"] == [] and none["total"] == 0, "种类和类别对不上:一条都没有,不是不筛" + assert (await client.get(url, params={"category": "nope"})).status_code == 422 + + assert (await client.get(url, params={"status": "handled"})).json()["items"] == [] + r = await client.post(f"{url}/{change.id}/resolve", json={"action": "dismiss", "note": "测试库造的取值"}) + assert r.status_code == 200, r.text + got = r.json()["change"] + assert got["status"] == "dismissed" and got["actions"] == [], "处理过的不再给动作" + handled = (await client.get(url, params={"status": "handled"})).json() + assert [c["id"] for c in handled["items"]] == [change.id] and handled["total"] == 1 + assert change.id not in {c["id"] for c in (await client.get(url)).json()["items"]} + + +async def test_acknowledged_change_offers_no_second_acknowledge_and_clean_shows_items_gone(client, shop): + sid = shop["id"] + url = f"/api/datasources/{sid}/changes" + removed = _by_kind((await client.get(url)).json()["items"])["column_removed"] + r = await client.post(f"{url}/{removed['id']}/resolve", json={"action": "acknowledge"}) + assert r.status_code == 200 + acts = [a["action"] for a in r.json()["change"]["actions"]] + assert "acknowledge" not in acts and "clean_catalog" in acts, "知道了还没处理:还能清理、不能再点知道了" + r = await client.post(f"{url}/{removed['id']}/resolve", json={"action": "clean_catalog"}, headers={"X-Actor": "reviewer"}) + assert r.status_code == 200, r.text + got = r.json() + assert got["catalog_version"] >= 2 and got["change"]["resolution"]["catalog_version"] == got["catalog_version"] + gone = [ref for ref in got["change"]["affects"]["catalog"] if ref["path"].startswith("columns.status")] + assert gone and all(ref["now"] is None for ref in gone), "清理过的项现在不在目录里了:照实为空" + + +# -------------------------------------------------------------------------- +# 数据源列表:角标和定时维护一次取齐 +# -------------------------------------------------------------------------- + + +async def test_upkeep_gives_badge_counts_and_maintenance_per_source(client, shop): + sid = shop["id"] + r = await client.post("/api/schedules", json={ + "name": "店铺库的定时维护", "target_kind": "source_maintenance", "target_id": sid, "cron": "0 3 * * *", + "timezone": "Asia/Shanghai", "enabled": True, "overlap": "skip", "misfire_grace_s": 3600}) + assert r.status_code in (200, 201), r.text + shop["schedules"].append(r.json()["id"]) + items = {i["source_id"]: i for i in (await client.get("/api/datasources/upkeep")).json()["items"]} + mine = items[sid] + assert mine["changes"]["by_severity"]["action"] >= 1 and mine["changes"]["pending"] >= 1 + [m] = mine["maintenance"] + assert m["enabled"] is True and m["timezone"] == "Asia/Shanghai" and "03:00" in m["description_text"] + assert m["next_run_at"], "开着的维护写下一次什么时候" + + +# -------------------------------------------------------------------------- +# 表详情:最近几次观察,各带口径;遮罩在读的那一刻再核 +# -------------------------------------------------------------------------- + + +async def test_observations_grouped_with_basis_and_masked_columns_hidden(client, shop): + sid = shop["id"] + now = utcnow() + async with SessionLocal() as session: + for i, rows in enumerate((120, 118)): + session.add(TableObservation(source_id=sid, table_name="orders", column_name="", kind="rows", + basis={"method": "count"}, basis_key="count", + data={"rows": rows, "at_least": False, "method": "count"}, + at=now - timedelta(days=i))) + session.add(TableObservation(source_id=sid, table_name="orders", column_name="channel", kind="codes_observe", + basis={"method": "sample", "percent": 1.0}, basis_key="sample:1", + data={"values": [["web", 30], ["shop", 10]], "rows": 40, "truncated": False}, at=now)) + session.add(TableObservation(source_id=sid, table_name="orders", column_name="channel", kind="codes_observe", + basis={"method": "first_rows", "rows": 1000}, basis_key="first_rows:1000", + data={"values": [["web", 5]], "rows": 5, "truncated": False}, + at=now - timedelta(days=2))) + session.add(TableObservation(source_id=sid, table_name="orders", column_name="amount", kind="codes_observe", + basis={"method": "full"}, basis_key="full", + data={"values": [["3", 1]], "rows": 1, "truncated": False}, at=now)) + await session.commit() + url = f"/api/datasources/{sid}/observations" + body = (await client.get(url, params={"table": "orders"})).json() + series = {(s["kind"], s["column"]): s for s in body["series"]} + rows = series[("rows", "")] + assert [p["data"]["rows"] for p in rows["points"]] == [120, 118], "新的在前" + assert rows["points"][0]["basis_text"] == "逐行数出" and rows["basis_changed"] is False + codes = series[("codes_observe", "channel")] + assert codes["points"][0]["basis_text"] == "块采样约 1%" + assert codes["points"][1]["basis_text"] == "按存储顺序取前 1000 行" + assert codes["basis_changed"] is True, "口径换过:照实说,不跨口径比" + assert codes["points"][0]["data"]["values"] == [["web", 30, 0.75], ["shop", 10, 0.25]], "占比由服务端算" + assert [s["kind"] for s in body["series"]][0] == "rows", "按观察种类的固定顺序" + + async with SessionLocal() as session: + row = await session.get(DataSource, sid) + row.options = {**(row.options or {}), "mask_columns": ["amount"]} + await session.commit() + body = (await client.get(url, params={"table": "orders"})).json() + assert ("codes_observe", "amount") not in {(s["kind"], s["column"]) for s in body["series"]}, \ + "现在被遮罩的列:观察时还没遮的也不给" + assert (await client.get(url, params={"table": "nope"})).json()["series"] == [] + + +# -------------------------------------------------------------------------- +# 后台任务条:按同源队的先后 +# -------------------------------------------------------------------------- + + +async def test_background_jobs_follow_the_lane_order(client, shop): + sid = shop["id"] + async with SessionLocal() as session: + intro = IntrospectJob(source_id=sid, status="queued", created_at=utcnow() - timedelta(seconds=30)) + draft = CatalogDraftJob(source_id=sid, status="running") + profile = CatalogProfileJob(source_id=sid, status="queued", scope="selected", settings={}) + session.add_all([intro, draft, profile]) + await session.commit() + ids = {"introspect": intro.id, "draft": draft.id, "profile": profile.id} + # 队里是 起草(在做)→ 剖析(排着);探查刚建、还没进队:排在队里的后面 + tickets = [runtime.lanes.enter(sid, "draft", ids["draft"]), runtime.lanes.enter(sid, "profile", ids["profile"])] + try: + items = (await client.get(f"/api/datasources/{sid}/background-jobs")).json()["items"] + assert [i["kind"] for i in items] == ["draft", "profile", "introspect"], items + assert [i["id"] for i in items] == [ids["draft"], ids["profile"], ids["introspect"]] + assert items[0]["position"] == 0 and items[1]["position"] == 1 and items[2]["position"] is None + assert items[1]["waiting_for"] == "draft", "排队的写等的是什么(服务端的 waiting_for)" + assert {"total", "done", "status"} <= set(items[0]) + finally: + for t in tickets: + t.release() + async with SessionLocal() as session: + for model, key in ((IntrospectJob, "introspect"), (CatalogDraftJob, "draft"), (CatalogProfileJob, "profile")): + await session.execute(delete(model).where(model.id == ids[key])) + await session.commit() + assert (await client.get(f"/api/datasources/{sid}/background-jobs")).json()["items"] == [] + + +# -------------------------------------------------------------------------- +# 定时维护:定时任务同一个形状,最近一次触发带 inputs +# -------------------------------------------------------------------------- + + +async def test_maintenance_returns_schedule_shape_and_last_fire_inputs(client, shop): + sid = shop["id"] + url = f"/api/datasources/{sid}/maintenance" + body = (await client.get(url)).json() + assert body["maintained"] is True and body["schedules"] == [] and isinstance(body["scheduler_on"], bool) + r = await client.post("/api/schedules", json={ + "name": "店铺库的定时维护", "target_kind": "source_maintenance", "target_id": sid, "cron": "30 2 * * *", + "timezone": "Asia/Shanghai", "enabled": True, "overlap": "skip", "misfire_grace_s": 3600}) + schedule_id = r.json()["id"] + shop["schedules"].append(schedule_id) + inputs = {"steps": ["introspect", "profile_needed", "observe"], + "jobs": [{"step": "introspect", "kind": "introspect", "id": "job-i", "tables": 3}], + "skipped": [{"step": "profile_needed", "reason": "数据源「x」未开启数据剖析"}]} + async with SessionLocal() as session: + session.add(ScheduleFire(schedule_id=schedule_id, fire_at=utcnow(), kind="scheduled", status="started", + reason="剖析需要的表没有执行:未开启", inputs=inputs)) + await session.commit() + [s] = (await client.get(url)).json()["schedules"] + assert s["id"] == schedule_id and s["version"] >= 1 and s["target_kind"] == "source_maintenance" + assert "02:30" in s["description_text"] and s["next_run_at"] + assert s["last_fire"]["status"] == "started" and s["last_fire"]["inputs"] == inputs + + async with SessionLocal() as session: + row = await session.get(DataSource, sid) + row.origin = "upload" + await session.commit() + assert (await client.get(url)).json()["maintained"] is False, "上传表格、记录表不给这一节" + + +# -------------------------------------------------------------------------- +# 通知:source.changed 打开订阅,链接落到「变化」标签页 +# -------------------------------------------------------------------------- + + +async def test_source_changed_is_subscribable_and_links_to_the_changes_tab(client, shop, monkeypatch): + assert service.EVENTS["source.changed"].scopes == ("global",), "只推全局渠道(按源汇总,不归哪个人)" + assert service.subscribe_problem("source.changed", "global") is None + assert service.subscribe_problem("source.changed", "user") is not None + got: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + got.append(request) + return httpx.Response(200, json={"ok": True}) + + monkeypatch.setattr(core, "_transport", httpx.MockTransport(handler)) + monkeypatch.setattr(core, "RETRY_DELAYS", (0, 0, 0)) + monkeypatch.setattr(service, "_base_url", _fake_base) + from app.core.crypto import encrypt + + async with SessionLocal() as session: + channel = NotifyChannel(name=f"变化-{uuid.uuid4().hex[:6]}", scope="global", format="generic", + url=encrypt("https://hooks.example.com/agentlab/upkeep-3b9e"), events=["source.changed"]) + session.add(channel) + await session.commit() + channel_id = channel.id + try: + assert await service.notify_source_changed(shop["id"]) == 1 + [request] = got + body = json.loads(request.content) + links = json.dumps(body, ensure_ascii=False) + assert f"/data/catalog/{shop['id']}?view=changes" in links, links + finally: + async with SessionLocal() as session: + await session.execute(delete(NotifyChannel).where(NotifyChannel.id == channel_id)) + await session.commit() + + +async def _fake_base(_session) -> str: + return "https://agentlab.example.com" + + +def test_schedule_target_columns_are_the_ones_the_page_uses(): + """数据源设置里建的维护定时:target_kind / target_id 写法和调度那一套同一处封装(X4)。""" + cols = scheduler_service.target_columns("source_maintenance", "src-x") + assert cols["target_kind"] == "source_maintenance" and cols["target_id"] == "src-x" and cols["workflow_id"] == "" + + +async def test_daily_volume_fills_the_days_without_data(client, shop): + """按日行数只记了有数据的日子:窗口里断了的那几天补 0,界面上的小柱图才看得出断档。""" + sid = shop["id"] + async with SessionLocal() as session: + session.add(TableObservation(source_id=sid, table_name="orders", column_name="paid_on", kind="daily_volume", + basis={"method": "window", "column": "paid_on", "days": 35, "since": "2026-10-01"}, + basis_key="window:paid_on:35", + data={"since": "2026-10-01", "until": "2026-10-05", + "days": {"2026-10-01": 30, "2026-10-02": 28, "2026-10-05": 31}}, + at=utcnow())) + await session.commit() + body = (await client.get(f"/api/datasources/{sid}/observations", params={"table": "orders"})).json() + [daily] = [s for s in body["series"] if s["kind"] == "daily_volume"] + point = daily["points"][0] + assert point["data"]["days"] == [["2026-10-01", 30], ["2026-10-02", 28], ["2026-10-03", 0], ["2026-10-04", 0], + ["2026-10-05", 31]] + assert point["data"]["days_with_data"] == 3 and point["data"]["total"] == 89 + assert point["basis_text"] == "业务日期 paid_on 最近 35 天" diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index 3ea2565a..63dc10ed 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -25,6 +25,7 @@ import { inquiry } from './inquiry' import { schedules } from './schedules' // 评测(下一轮第 4 波):接口在 api/evals.ts,这里只挂上去 import { evals } from './evals' +import { sourceChanges } from './sourceChanges' import { whatif } from './whatif' import { reportExport } from './reportExport' // 应用平台:各自的接口文件,这里只挂上去 @@ -955,6 +956,8 @@ export const api = { schedules, // 评测:题集、题目、发起与结果、对比与 A/B、目录快照、口径命中与重复度 evals, + // 数据源的变化、取值观察、定时维护的步骤、后台任务条(下一轮第 4 波,typesSource.ts 是接口约定) + sourceChanges, compare, bundle, inquiry, diff --git a/frontend/src/api/sourceChanges.ts b/frontend/src/api/sourceChanges.ts new file mode 100644 index 00000000..2f61cb8e --- /dev/null +++ b/frontend/src/api/sourceChanges.ts @@ -0,0 +1,41 @@ +/** + * 数据源的变化、取值观察、定时维护的步骤、后台任务条的接口(服务端 app/api/source_changes.py),挂在 api.sourceChanges 上。 + * 定时本身(cron、时区、开关)走 api.schedules(target_kind=source_maintenance,总方案 X4:定时只有一套)。 + * + * 请求函数从 ./http 导入,不从 ./client 导入(client.ts 要导入这个文件,反过来就成了循环,见 http.ts 开头的说明)。 + * 误报没写理由 422、处理过的再处理 409、目录被别人刚改过 409,message 是服务端给的中文 + */ +import type { + MaintenanceStep, SourceBackgroundJob, SourceChange, SourceChangeList, SourceChangeQuery, SourceChangeResolveBody, + SourceChangeResolveOut, SourceMaintenance, SourceUpkeep, TableObservations, +} from '../typesSource' +import { get, post, put, qs, type RequestOptions } from './http' + +const at = (sourceId: string) => `/datasources/${encodeURIComponent(sourceId)}` + +export const sourceChanges = { + /** 数据源列表一次取齐:各源要处理的变化条数(卡片角标)、指向它的维护定时 */ + upkeep: (opts?: RequestOptions) => get<{ items: SourceUpkeep[] }>('/datasources/upkeep', opts), + /** 一个源的变化:缺省开着的(要处理的先);status=handled 是处理过的。category 只看结构 / 取值变化 */ + list: (sourceId: string, q: SourceChangeQuery = {}, opts?: RequestOptions) => + get(`${at(sourceId)}/changes${qs({ + status: q.status ?? 'pending', category: q.category ?? undefined, table: q.table ?? undefined, + limit: q.limit ?? 100, offset: q.offset ?? 0, + })}`, opts), + get: (sourceId: string, changeId: string, opts?: RequestOptions) => + get<{ change: SourceChange }>(`${at(sourceId)}/changes/${encodeURIComponent(changeId)}`, opts), + /** 处理一条变化:动作落到目录已有的写入接口并记目录版本、建了的任务编号 */ + resolve: (sourceId: string, changeId: string, body: SourceChangeResolveBody) => + post(`${at(sourceId)}/changes/${encodeURIComponent(changeId)}/resolve`, body), + /** 表详情「结构与取值变化」一节:这张表最近几次观察,各带口径 */ + observations: (sourceId: string, table: string, opts?: RequestOptions) => + get(`${at(sourceId)}/observations${qs({ table })}`, opts), + /** 这个源进行中和排队的后台任务,按同源队的先后排好 */ + backgroundJobs: (sourceId: string, opts?: RequestOptions) => + get<{ items: SourceBackgroundJob[] }>(`${at(sourceId)}/background-jobs`, opts), + /** 定时维护:步骤、剖析开没开、调度开没开、指向这个源的维护定时(定时任务页同一个形状,最近一次触发带 inputs) */ + maintenance: (sourceId: string, opts?: RequestOptions) => get(`${at(sourceId)}/maintenance`, opts), + /** 改要做哪几步(至少一步)。上传表格、记录表 409 */ + putMaintenance: (sourceId: string, steps: MaintenanceStep[]) => + put(`${at(sourceId)}/maintenance`, { steps }), +} diff --git a/frontend/src/lib/sourceAnchor.ts b/frontend/src/lib/sourceAnchor.ts new file mode 100644 index 00000000..483d0c3e --- /dev/null +++ b/frontend/src/lib/sourceAnchor.ts @@ -0,0 +1,34 @@ +/** + * 指名数据源页上的一张卡片的地址:<数据源页>#source-,带 &maintenance 时顺带打开那张卡片的「定时维护」。 + * + * 全站只认这一种写法,拼、拆都在这里:目录页后台任务条上探查的「查看」、定时任务页上维护定时的「去数据源页设置」都链到这里。 + * 两边各写一遍的话改了一边另一边就落空——复核 W4R-04 就是这样:定时任务页链的是 ?source=,数据源页只认 #source-, + * 点了落在列表顶上,既不滚到那张卡片也不打开定时维护。 + * + * 用 #(不用查询串):页内的元素 id 就是 source-,浏览器自己也认这个锚点;数据源页的两个标签(数据库 / 表格)共用一个组件, + * 查询串留给页面别的状态。 + */ +export type SourceAnchorOpen = 'maintenance' + +const PREFIX = 'source-' +const OPENS: readonly SourceAnchorOpen[] = ['maintenance'] + +/** 页内卡片的元素 id */ +export const sourceCardId = (id: string) => `${PREFIX}${id}` + +/** 地址里的 # 部分(带 #):sourceAnchor(id) 只滚过去,sourceAnchor(id, 'maintenance') 还打开定时维护 */ +export const sourceAnchor = (id: string, open?: SourceAnchorOpen) => + `#${PREFIX}${encodeURIComponent(id)}${open ? `&${open}` : ''}` + +/** 拆开地址里的 # 部分;不是指名卡片的(没有 #、别的锚点、编码坏了)给 null。认不得的标记不管,只当滚过去 */ +export function parseSourceAnchor(hash: string): { id: string; open: SourceAnchorOpen | null } | null { + const [head, ...flags] = hash.replace(/^#/, '').split('&') + if (!head.startsWith(PREFIX) || head.length === PREFIX.length) return null + let id: string + try { + id = decodeURIComponent(head.slice(PREFIX.length)) + } catch { + return null + } + return { id, open: OPENS.find((o) => flags.includes(o)) ?? null } +} diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index 515e17f6..ac4d9aa9 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -6336,3 +6336,5 @@ export const CAPABILITY_REVIEW_TEXT: typeof CATALOG_REVIEW_TEXT = { // 应用平台使用面的文案(FE-5 维护)。调用方照旧从 lib/terms 导入 export * from './termsApp' export * from './termsBlueprint' +// 数据源的变化、定时维护、后台任务条(下一轮第 4 波 W4-d) +export * from './termsSource' diff --git a/frontend/src/lib/termsSource.ts b/frontend/src/lib/termsSource.ts new file mode 100644 index 00000000..cd04b605 --- /dev/null +++ b/frontend/src/lib/termsSource.ts @@ -0,0 +1,268 @@ +/** + * 数据源的变化(结构变化、取值变化)、表详情的「结构与取值变化」一节、定时维护、目录页的后台任务条的文案(next-round 03 B2–B4、 + * 05 (g))。规矩同 terms.ts 开头的术语表。 + * + * 叫法(总方案 X10):「结构变化」「取值变化」,和已经在用的「目录有变化」(目录版本变了,CATALOG_DRIFT_TEXT)分开;严重程度 + * 「要处理 / 提醒 / 说明」同 SCHEMA_DRIFT_TEXT.severity。 + * + * terms.ts 末尾 `export * from './termsSource'`:调用方照旧从 lib/terms 导入。 + */ +import type { MaintenanceStep, SourceChangeAction, SourceChangeCategory, SourceChangeSeverity, SourceChangeStatus } from '../typesSource' +import type { ObservationKind, SourceJobKind } from '../typesSource' +import { formatNumber } from './format' + +const pct = (share: number) => { + const v = share * 100 + return `${v >= 10 || v === 0 ? Math.round(v) : v.toFixed(1)}%` +} + +/** + * 变化的种类:和服务端 app/data/source_changes.KIND_LABEL 逐字一致(tests/test_source_change_terms.py 守着),服务端给人看 + * 的一句(summary)也用它 + */ +export const SOURCE_CHANGE_KIND_LABEL: Record = { + table_added: '新出现的表', table_removed: '表已删除', table_left_scope: '表离开了探查范围', + table_reflect_failed: '这次读不到表结构', column_added: '新加了列', column_removed: '删除了列', + column_type_changed: '列的类型变了', column_nullable_changed: '列的可空变了', key_changed: '键变了', fk_changed: '外键变了', + comment_changed: '注释变了', scope_changed: '探查范围的规则变了', + codes_new: '出现了新码值', codes_shift: '码值的比例变化大', date_stalled: '最新业务日期没有前进', + date_gap: '有的日子没有数据', volume_drop: '按日行数大幅下跌', rows_drop: '行数大幅下跌', rows_jump: '行数大幅上涨', + coverage_drop: '关联的覆盖率下降', +} + +export const SOURCE_CHANGE_TEXT = { + /** 数据目录页左栏的第三个视图 */ + view: '变化', + viewHint: '探查、取值观察测到的结构变化和取值变化,按要不要处理分组', + category: { struct: '结构变化', value: '取值变化' } as Record, + categoryAll: '全部', + categoryLabel: '按类别筛选', + severity: { action: '要处理', warning: '提醒', info: '说明' } as Record, + severityHint: { + action: '现在大概率已经不对了:已确认的目录项或写死的 SQL 指向了被删的列,或者「已列全」被撤销了', + warning: '值得核对:可能影响 SQL 的写法或结果', + info: '知道一下:一般不影响已有的目录和模板', + } as Record, + status: { open: '开着', acknowledged: '知道了,还没处理', resolved: '已处理', dismissed: '标为误报' } as Record, + statusFilter: { pending: '开着的', handled: '处理过的' } as Record<'pending' | 'handled', string>, + statusLabel: '按状态筛选', + listLabel: '数据源的变化', + loading: '正在载入…', + refresh: '刷新', + more: (n: number) => `再显示 ${formatNumber(n)} 条`, + total: (n: number) => `${formatNumber(n)} 条`, + group: (label: string, n: number) => `${label} ${formatNumber(n)} 条`, + empty: '没有开着的变化', + emptyBody: '探查结构、取值观察测到表结构或取值有变化时,会列在这里。', + emptyFiltered: '没有符合筛选的变化', + emptyHandled: '还没有处理过的变化', + /** 视图标签上的计数:要处理的条数 */ + badge: (n: number) => `${formatNumber(n)} 条要处理`, + seen: (n: number) => `测到 ${formatNumber(n)} 次`, + detectedAt: (when: string) => `测到于 ${when}`, + overviewTitle: '从左侧挑一条变化开始处理', + overviewBody: '要处理的排在最前:已确认的目录项、写死的 SQL 说到了被删的列,或者「已列全」被撤销了。点开一条,右侧写依据、影响面和能做的处理;每个处理都记下谁、什么时候、目录第几版。', + backToList: '返回变化列表', + missing: '这条变化不存在,可能已随数据源删除', + openTable: '查看这张表', + // ---- 依据 + basisTitle: '依据', + detectedBy: (job: string, trigger: string, actor: string | null) => + `${trigger}的${job}${actor ? `(${actor})` : ''}`, + jobKind: { introspect: '探查结构', profile: '取值观察', on_demand: '按需加入' } as Record, + trigger: { manual: '手动发起', schedule: '定时维护', on_demand: '助手点名' } as Record, + structBasis: (before: number | null, after: number | null) => (before && after + ? `比对的是这张表的结构第 ${formatNumber(before)} 版和第 ${formatNumber(after)} 版` + : before ? `比对的是这张表的结构第 ${formatNumber(before)} 版和这次探查读到的` : '比对的是上一次和这次探查读到的结构'), + valueBasis: (text: string) => `观察口径:${text}`, + valueBasisNote: '只和口径相同的历史观察比', + // ---- 前后各是什么(按种类) + factsTitle: '前后', + before: '之前', + after: '现在', + none: '(无)', + nullable: (v: boolean) => (v ? '可空' : '不可空'), + rename: (from: string, to: string) => `可能是 ${from} 改名为 ${to}(未确认:库里只看得到删了一列、加了一列)`, + newCodes: (codes: string) => `新出现的码值:${codes}`, + codeShare: (code: string, share: number) => `${code}(占 ${pct(share)})`, + knownCodes: (n: number) => `目录里列出的 ${formatNumber(n)} 个取值之外`, + completeRevoked: (version: number | null) => + `取值观察撤销了码值项的「已列全」${version ? `(目录第 ${formatNumber(version)} 版)` : ''}:SQL 检查不再按旧的码值表提示,新取值的含义要人来填`, + completeKept: '「已列全」在观察之前已被别人改掉,没有撤销', + shift: (psi: number) => `比例的变化程度(PSI)${psi.toFixed(2)}`, + shiftTop: (code: string, from: number, to: number) => `${code}:${pct(from)} → ${pct(to)}`, + stalled: (max: string, age: number, threshold: number) => + `最新业务日期停在 ${max},已经 ${formatNumber(age)} 天(以前通常 ${formatNumber(threshold)} 天之内会前进)`, + gap: (n: number, days: string) => `窗口里有 ${formatNumber(n)} 天没有数据:${days}`, + volume: (day: string, rows: number, median: number) => + `${day} 只有 ${formatNumber(rows)} 行,前 28 天的中位数是 ${formatNumber(median)} 行`, + rows: (before: number, after: number, estimated: boolean) => + `${formatNumber(before)} 行 → ${formatNumber(after)} 行${estimated ? '(数据库统计的估算值,连续两次都这样才报)' : ''}`, + coverage: (target: string, before: number, after: number) => `指向 ${target} 的关联覆盖率 ${pct(before)} → ${pct(after)}`, + scopeRules: (entered: number, left: number) => `这次探查新进范围 ${formatNumber(entered)} 张,离开范围 ${formatNumber(left)} 张`, + reflectFailed: (error: string) => `读不到结构的原因:${error}`, + // ---- 影响面 + impactTitle: '影响面', + impactNone: '目录里没有说到它的项,已发布模板里也没有用到它的。', + impactCatalog: (n: number) => `说到这些列的目录项 ${formatNumber(n)} 项`, + impactNow: '现在', + impactGone: '已不在目录里', + impactNeeds: '人工确认过、还说着被删的列', + impactRules: '有效记录条件用到了这一列:取值有变时,按它过滤的结果可能漏掉新状态的行', + impactTemplates: (n: number) => `用到这张表的已发布模板 ${formatNumber(n)} 个`, + templateImpact: { direct: '写死的 SQL 直接引用', possible: '可能涉及' } as Record, + templateVersion: (v: number) => `v${v}`, + breaks: (cols: string[]) => (cols.length + ? `这条 SQL 现在会报列不存在(${cols.join('、')})` + : '这条 SQL 现在会报表或列不存在'), + // ---- 处理 + actionsTitle: '处理', + action: { + clean_catalog: '清理目录项', add_codes: '加入码值', redraft: '重新起草这张表', reprofile: '重新剖析这张表', + acknowledge: '知道了', dismiss: '误报', + } as Record, + actionHint: { + clean_catalog: '把影响面里说到这些列的目录项从目录里去掉(目录升一版,记下是因为这条变化)', + add_codes: '把新出现的码值加进这一列的码值表,含义可以先空着', + redraft: '建一个只含这张表的起草任务:按新的结构重写推断项,人工确认过的项不动', + reprofile: '建一个只含这张表的剖析任务:对业务库发只读查询,重新核对关系、码值和日期', + acknowledge: '记下已经看过,还留在开着的变化里', + dismiss: '不是真的变化:写明理由后不再算要处理', + } as Record, + cleanTitle: (n: number) => `清理 ${formatNumber(n)} 项目录?`, + cleanConsequences: ['这些项从数据目录里去掉,目录升一版;从已发布版本发起的正式运行会提醒「数据目录有变化」', + '人工确认过的项也会去掉:它们说的列已经不在了', '去掉之后要恢复,只能重新填写'], + cleanConfirm: '清理', + codesTitle: '加入码值', + codesHint: '含义可以先空着,之后在表详情的码值表里补;加入的项记为人工填写、已确认,依据写这条变化的编号', + codesMeaning: (code: string) => `码值 ${code} 的含义`, + codesSubmit: '加入', + noteLabel: '误报的理由', + notePlaceholder: '例如:测试库里故意删的列', + noteRequired: '标为误报要写明理由', + noteSubmit: '标为误报', + cancel: '取消', + done: { + clean_catalog: '已清理目录项', add_codes: '已加入码值', redraft: '已建起草任务', reprofile: '已建剖析任务', + acknowledge: '已记为知道了', dismiss: '已标为误报', + } as Record, + // ---- 处理过的结果 + resolutionTitle: '处理结果', + resolvedBy: (action: string, by: string | null, when: string) => `${action} · ${by ?? '未署名'} · ${when}`, + resolvedVersion: (v: number) => `目录第 ${formatNumber(v)} 版`, + resolvedVersions: (pairs: string) => `各表的目录版本:${pairs}`, + resolvedNote: (note: string) => `理由:${note}`, + resolvedJob: { draft: '起草任务', profile: '剖析任务' } as Record, + jobLink: (kind: string, id: string) => `${kind} #${id.slice(0, 6)}`, + viewProgress: '查看进度', + /** 不能做的动作的读屏说明:「重新剖析这张表:这个数据源未开启数据剖析」 */ + unavailable: (label: string, reason: string) => `${label}:${reason}`, +} + +/** 数据源卡片上的角标:要处理的变化条数(读屏写全) */ +export const SOURCE_BADGE_TEXT = { + chip: (n: number) => `${formatNumber(n)} 个变化要处理`, + aria: (name: string, n: number) => `数据源「${name}」有 ${formatNumber(n)} 个变化要处理,打开数据目录的「变化」`, + hint: '探查结构、取值观察测到的变化里要处理的条数,点开到数据目录的「变化」处理', +} + +/** 从别处链到数据源页上的一张卡片(lib/sourceAnchor 的 #source-),那个数据源已经不在了 */ +export const SOURCE_ANCHOR_TEXT = { + missing: '没找到这个数据源,可能已经删除', +} + +/** 表详情的「结构与取值变化」一节 */ +export const TABLE_CHANGES_TEXT = { + title: '结构与取值变化', + hint: '这张表的结构变化、取值变化,和最近几次取值观察', + changesHead: '这张表的变化', + changesNone: '这张表还没有测到结构变化或取值变化。', + changesMore: (n: number) => `…另有 ${formatNumber(n)} 条,在「变化」里查看`, + openChange: '去处理', + viewChange: '查看', + observeHead: '最近几次观察', + observeNone: '这张表还没有取值观察:定时维护选了「取值观察」,或者手动剖析时带上观察,才会有。', + observeKind: { rows: '行数', codes_observe: '码值分布', freshness: '最新业务日期', daily_volume: '按日行数' } as Record, + basis: (text: string) => `口径:${text}`, + basisChanged: '这几次的口径不全相同:口径不同的不互相比较', + at: (when: string) => `观察于 ${when}`, + rows: (rows: number | null, atLeast: boolean) => (rows == null ? '—' : `${atLeast ? '至少 ' : ''}${formatNumber(rows)} 行`), + codesTop: (n: number, rows: number) => `${formatNumber(n)} 个取值,共 ${formatNumber(rows)} 行`, + codesTruncated: '取值太多,只记了最多的 50 个:不判新码值、不算比例', + code: (code: string, share: number) => `${code} ${pct(share)}`, + freshness: (max: string | null) => max ?? '—', + daily: (days: number, total: number) => `有数据的 ${formatNumber(days)} 天,共 ${formatNumber(total)} 行`, + dailyRange: (since: string, until: string) => `${since} 至 ${until}`, + dailyChart: (since: string, until: string, days: number, total: number) => + `${since} 至 ${until} 每天的行数:有数据的 ${formatNumber(days)} 天,共 ${formatNumber(total)} 行`, + history: (n: number) => `前 ${formatNumber(n)} 次`, + revokedHead: '撤销过「已列全」', + revoked: (column: string, when: string, version: number | null) => + `${when}:取值观察在 ${column} 上测到目录之外的取值,撤销了「已列全」${version ? `(目录第 ${formatNumber(version)} 版)` : ''}`, +} + +/** 目录页的后台任务条(05 (g)):页头只做入口,进度只在这一条上 */ +export const JOB_STRIP_TEXT = { + label: '这个数据源的后台任务', + kind: { introspect: '探查', draft: '起草', profile: '剖析' } as Record, + running: (kind: string, done: number, total: number) => `${kind} ${formatNumber(done)}/${formatNumber(total)}`, + queued: (kind: string) => `${kind} 排队中`, + queuedBehind: (kind: string, behind: string) => `${kind} 排队中,${behind}结束后开始`, + behind: { introspect: '探查', draft: '起草', profile: '剖析', import: '导入目录', sync_draft: '起草' } as Record, + progress: (kind: string, done: number, total: number) => `${kind}:已完成 ${formatNumber(done)} / ${formatNumber(total)} 张表`, + view: '查看', + viewHint: { introspect: '到数据源卡片上看探查的进度、停止', draft: '打开起草的进度', profile: '打开剖析的进度' } as Record, + /** 页头的入口:读屏名称写状态,不写数字 */ + headRunning: (label: string) => `${label}(进行中)`, + headQueued: (label: string) => `${label}(排队中)`, +} + +/** 数据源的定时维护(数据源卡片上的「定时维护」) */ +export const MAINTENANCE_TEXT = { + open: '定时维护', + openHint: '按设定的时段自动探查结构、剖析需要的表、观察取值,缺省关闭', + title: (name: string) => `「${name}」的定时维护`, + intro: '到点按顺序探查结构 → 剖析需要的表 → 观察取值,和起草、剖析在同一条队里排队,一个接一个做。只发只读查询,每张表的查询次数和时长按剖析设置。', + enable: '开启定时维护', + enableHint: '缺省关闭:开启后会定期对业务库发只读查询,请选业务库的低峰时段(比如凌晨)', + offPeak: '请选业务库的低峰时段(比如凌晨),避开白天的业务高峰', + stepsTitle: '做哪几步', + stepsHint: '至少选一步', + step: { introspect: '探查结构', profile_needed: '剖析需要的表', observe: '取值观察' } as Record, + stepHint: { + introspect: '读取探查范围内的表结构,前后比出结构变化', + profile_needed: '剖析从没剖析过、上次没做完、目录里多出待核实项的表', + observe: '观察码值分布、最新业务日期、按日行数和行数,和同口径的历史比出取值变化', + } as Record, + stepsNone: '至少选一步', + profileOff: '这个数据源未开启数据剖析:到点时「剖析需要的表」「取值观察」两步不执行,只探查结构(照实记下)。剖析在数据源的「编辑」里开启。', + schedulerOff: '这个服务的定时调度没有开启:到点不会自动维护。', + freqTitle: '时段', + scheduleName: (name: string) => `「${name}」定时维护`, + next: (when: string) => `下一次:${when}`, + nextNone: '已暂停,不会触发', + lastTitle: '最近一次', + lastNone: '还没有触发过', + lastAt: (when: string, status: string) => `${when} · ${status}`, + fireStatus: { started: '已开始', skipped_overlap: '已跳过(上一次还没做完)', skipped_misfire: '已跳过(错过了时间)', failed_to_start: '没有执行' } as Record, + lastJob: (step: string, kind: string, id: string, tables: number | null) => + `${step}:${kind} #${id.slice(0, 6)}${tables != null ? `(${formatNumber(tables)} 张表)` : ''}`, + lastJobKind: { introspect: '探查任务', profile: '剖析任务' } as Record, + lastSkipped: (step: string, reason: string) => `${step}没有执行:${reason}`, + lastReason: (reason: string) => `原因:${reason}`, + viewJob: '查看进度', + history: '在定时任务页查看触发记录', + extra: (n: number) => `另有 ${formatNumber(n)} 条指向这个数据源的维护定时,在定时任务页管理`, + save: '保存', + cancel: '取消', + saving: '正在保存…', + saved: '已保存定时维护', + savedOff: '已保存:定时维护已关闭', + nothing: '没有改动', + notMaintained: '上传的表格、记录表的结构随版本或表单走,不需要定时维护', + /** 卡片上的标签:开着时写频率 */ + chip: (desc: string) => `定时维护 · ${desc}`, + chipPaused: '定时维护 · 已暂停', + chipHint: '到点自动探查结构、剖析需要的表、观察取值', + loadFailed: '无法载入定时维护的设置', +} diff --git a/frontend/src/pages/DataSourcesTab.tsx b/frontend/src/pages/DataSourcesTab.tsx index b8dee988..fdcd0bd5 100644 --- a/frontend/src/pages/DataSourcesTab.tsx +++ b/frontend/src/pages/DataSourcesTab.tsx @@ -1,8 +1,8 @@ import { useEffect, useId, useMemo, useRef, useState } from 'react' -import { useNavigate } from 'react-router-dom' +import { useLocation, useNavigate } from 'react-router-dom' import { - AlertTriangle, BookMarked, ChevronRight, Database, EyeOff, FileSpreadsheet, History, Info, KeyRound, Layers, Lock, Plug, Plus, - RefreshCw, ScanSearch, Search, Table2, Upload, X, + AlertOctagon, AlertTriangle, BookMarked, CalendarClock, ChevronRight, Database, EyeOff, FileSpreadsheet, History, Info, KeyRound, Layers, + Lock, Plug, Plus, RefreshCw, ScanSearch, Search, Table2, Upload, X, } from 'lucide-react' import clsx from 'clsx' import { ApiError, api, uploadDecision } from '../api/client' @@ -23,9 +23,11 @@ import { checkHealth, forgetHealth, healthFromServer, setHealth, useHealth } fro import type { HealthRecord } from '../lib/health' import { workflowList, workflowsMentioning } from '../lib/mentions' import { - CATALOG_TEXT, INTROSPECT_SCOPE_TEXT, INTROSPECT_TEXT, MASK_BOUNDARY_TEXT, PROFILE_TEXT, RAW_STATE_LABEL, RECIPE_ORIGIN_LABEL, - RECIPE_TEXT, UPLOAD_CONVERSION_LABEL, UPLOAD_SHAPE_LABEL, UPLOAD_TEXT, VERSIONS_TEXT, + CATALOG_TEXT, INTROSPECT_SCOPE_TEXT, INTROSPECT_TEXT, MAINTENANCE_TEXT, MASK_BOUNDARY_TEXT, PROFILE_TEXT, RAW_STATE_LABEL, + RECIPE_ORIGIN_LABEL, RECIPE_TEXT, SOURCE_ANCHOR_TEXT, SOURCE_BADGE_TEXT, UPLOAD_CONVERSION_LABEL, UPLOAD_SHAPE_LABEL, UPLOAD_TEXT, + VERSIONS_TEXT, } from '../lib/terms' +import { parseSourceAnchor, sourceAnchor, sourceCardId } from '../lib/sourceAnchor' import { useLoginOn } from '../store/auth' import { useRunClock } from '../run/useRunClock' import { focusProfileField, ProfileSettingsDialog, ProfileSettingsSection } from './catalog/ProfileSettings' @@ -36,6 +38,8 @@ import { import type { ProfileForm } from './catalog/profile' import { ImportWizard } from './import/ImportWizard' import { IntrospectScopeDialog } from './IntrospectScopeDialog' +import { MaintenanceChip, SourceMaintenanceDialog } from './SourceMaintenanceDialog' +import type { SourceUpkeep } from '../typesSource' import type { WizardEntry } from './import/ImportWizard' import { VersionsDialog } from './import/VersionsDialog' @@ -81,6 +85,7 @@ const syncCatalog = () => void useCatalog.getState().reload('datasources') export function DataSourcesTab({ view = 'databases' }: { view?: DataView }) { const navigate = useNavigate() + const location = useLocation() const [rows, setRows] = useState(null) const [kinds, setKinds] = useState([]) const [loadError, setLoadError] = useState(null) @@ -89,8 +94,15 @@ export function DataSourcesTab({ view = 'databases' }: { view?: DataView }) { /** 按配方导入的向导(首次、上传新一期、修改配方、继续未完成的导入) */ const [wizard, setWizard] = useState(null) const [kick, setKick] = useState<{ id: string; seq: number } | null>(null) + /** 从别处链过来要打开定时维护的那张卡片(#source-&maintenance,见 lib/sourceAnchor);seq 让同一张卡片能再开一次 */ + const [arrival, setArrival] = useState<{ id: string; seq: number } | null>(null) + /** 各源要处理的变化条数(角标)和定时维护(next-round 03 B3、B4):一次取齐;取不到只是少了角标,不挡列表 */ + const [upkeep, setUpkeep] = useState>(() => new Map()) const loadSeq = useRef(0) const hasRows = useRef(false) + const loadUpkeep = () => { + api.sourceChanges.upkeep().then((r) => setUpkeep(new Map(r.items.map((i) => [i.source_id, i]))), () => {}) + } // 首次加载才画骨架;之后的刷新保留列表、不卸载——展开的表清单、滚动位置、 // 卡片上的忙碌态都长在卡片上,以前整页换成 Spinner,这些全丢,还跳回顶部 @@ -109,8 +121,28 @@ export function DataSourcesTab({ view = 'databases' }: { view?: DataView }) { else setLoadError(e) } } - useEffect(() => { void load() }, []) - useOnReconnect(load) + useEffect(() => { void load(); loadUpkeep() }, []) + useOnReconnect(() => { void load(); loadUpkeep() }) + // 从别处链过来指名一张卡片(lib/sourceAnchor:目录页后台任务条上探查的「查看」、定时任务页上维护定时的「去数据源页设置」): + // 列表载入后滚过去;带 &maintenance 的再打开它的「定时维护」,标记用过就从地址里去掉(刷新不再弹)。指名的源已经不在了 + // (维护定时比数据源活得久)照实说一句,不然只是落在列表顶上、什么也没发生。每个地址只处理一次:列表之后每次增删改都会变 + const arrived = useRef('') + useEffect(() => { + const anchor = parseSourceAnchor(location.hash) + if (!rows || !anchor || arrived.current === location.hash) return + arrived.current = location.hash + const row = rows.find((r) => r.id === anchor.id) + if (!row) { + toast.info(SOURCE_ANCHOR_TEXT.missing) + return + } + document.getElementById(sourceCardId(row.id))?.scrollIntoView({ block: 'center' }) + if (!anchor.open) return + if (anchor.open === 'maintenance') setArrival({ id: row.id, seq: Date.now() }) + const plain = sourceAnchor(row.id) + arrived.current = plain + navigate({ pathname: location.pathname, search: location.search, hash: plain }, { replace: true }) + }, [rows, location.hash]) const upsert = (row: any) => setRows((rs) => { if (!rs) return [row] @@ -198,6 +230,9 @@ export function DataSourcesTab({ view = 'databases' }: { view?: DataView }) { // 版本页里清除原件之后服务端不返回数据源,卡片上的原件状态靠重新取列表 onReload={() => void load()} kick={kick && kick.id === row.id ? kick.seq : 0} + openMaintenance={arrival && arrival.id === row.id ? arrival.seq : 0} + upkeep={upkeep.get(row.id) ?? null} + onUpkeep={loadUpkeep} /> ))} @@ -271,8 +306,13 @@ export function DataSourcesTab({ view = 'databases' }: { view?: DataView }) { /** 结构同步超过这么久就提示可能过期:库表会变,Copilot 照旧表写 SQL 只会报错 */ const STALE_SCHEMA_MS = 7 * 24 * 3600_000 -function SourceCard({ row, meta, onChange, onRemoved, onEdit, onReupload, onRecipeReupload, onRedraft, onResume, onReload, kick }: { +function SourceCard({ row, meta, onChange, onRemoved, onEdit, onReupload, onRecipeReupload, onRedraft, onResume, onReload, kick, + openMaintenance = 0, upkeep = null, onUpkeep }: { row: any; meta?: any + /** 要处理的变化条数、定时维护(GET /api/datasources/upkeep);取不到为 null */ + upkeep?: SourceUpkeep | null + /** 定时维护存过了:重取角标和维护 */ + onUpkeep?: () => void onChange: (row: any) => void /** 重新取整个列表(版本页里的写操作没有返回数据源时用) */ onReload: () => void @@ -285,6 +325,8 @@ function SourceCard({ row, meta, onChange, onRemoved, onEdit, onReupload, onReci onResume: () => void /** 非 0 时自动点一次「探查结构」(新建后 toast 上的按钮) */ kick: number + /** 非 0 时打开一次「定时维护」(从别处链过来,#source-&maintenance);这张卡片不做定时维护时不理 */ + openMaintenance?: number }) { const healthKey = `datasource:${row.id}` // 后端记着上次测的结果:换了浏览器、清了缓存也还在。本机刚测过的更新就用本机的 @@ -324,6 +366,14 @@ function SourceCard({ row, meta, onChange, onRemoved, onEdit, onReupload, onReci const [job, setJob] = useState(row.introspect_job ?? null) const [stopping, setStopping] = useState(false) const [scoping, setScoping] = useState(false) + const [maintaining, setMaintaining] = useState(false) + /** 要处理的变化条数(数据源卡片的角标,next-round 03 B3 ③) */ + const actionChanges = upkeep?.changes.by_severity.action ?? 0 + /** 定时维护只给手工登记的库:上传表格、记录表的结构随版本或表单走 */ + const maintainable = (row.origin ?? 'manual') === 'manual' + useEffect(() => { + if (openMaintenance && maintainable && !uploaded) setMaintaining(true) + }, [openMaintenance]) const jobActive = !!job && (job.status === 'queued' || job.status === 'running') useEffect(() => { const next = row.introspect_job ?? null @@ -530,7 +580,7 @@ function SourceCard({ row, meta, onChange, onRemoved, onEdit, onReupload, onReci const failed = record && !record.ok && !checkingSince return ( -
+
{uploaded ? @@ -544,6 +594,14 @@ function SourceCard({ row, meta, onChange, onRemoved, onEdit, onReupload, onReci 可写 } {!row.enabled && 已停用} + {actionChanges > 0 && ( + // 要处理的变化:点开到数据目录的「变化」视图(读屏写全:哪个源、几条、点了去哪) + + )}
@@ -573,6 +631,12 @@ function SourceCard({ row, meta, onChange, onRemoved, onEdit, onReupload, onReci {INTROSPECT_SCOPE_TEXT.open} )} + {!uploaded && maintainable && ( + + )} {row.open_staging && ( )} + {maintainable && !uploaded && upkeep && } {!!row.tools?.length && {row.tools.join(' · ')}}
+ {maintaining && ( + setMaintaining(false)} onSaved={() => onUpkeep?.()} /> + )} {scoping && ( setScoping(false)} onSaved={onReload} onRun={() => void introspect()} /> diff --git a/frontend/src/pages/SourceMaintenanceDialog.tsx b/frontend/src/pages/SourceMaintenanceDialog.tsx new file mode 100644 index 00000000..350e8c0e --- /dev/null +++ b/frontend/src/pages/SourceMaintenanceDialog.tsx @@ -0,0 +1,297 @@ +import { useEffect, useMemo, useRef, useState } from 'react' +import { Link, useNavigate } from 'react-router-dom' +import { AlertTriangle, CalendarClock, Info } from 'lucide-react' +import { ApiError, api } from '../api/client' +import type { SchedulePatch, SchedulePreview, ScheduleTargetCreate } from '../types' +import type { MaintenanceSchedule, MaintenanceStep, SourceMaintenance } from '../typesSource' +import { FreqPicker } from '../components/FreqPicker' +import { ErrorState, Modal, Skeleton, Spinner, toast, useTicker } from '../components/ui' +import { formatDateTime, parseServerTime } from '../lib/format' +import { DEFAULT_FREQ, DEFAULT_GRACE_S, browserTimezone, cronOf, freqOf, freqProblem } from '../lib/freq' +import type { Freq } from '../lib/freq' +import { MAINTENANCE_TEXT as T } from '../lib/terms' +import { EnableSwitch, FireStatusChip, NextRun } from './schedules/parts' + +// =========================================================================== +// 数据源的定时维护(docs/design/next-round/03-数据接入与目录维护.md B4,目录-D5):按数据源开、缺省关、只在设定的时段跑。 +// +// 定时只有一套(总方案 X4):维护就是一条 target_kind=source_maintenance、target_id=数据源 id 的定时任务,这里是它的快捷入口。 +// 频率用定时任务同一个选择器(components/FreqPicker),保存成 POST /api/schedules(ScheduleTargetCreate);已经有了的改频率、 +// 开关走 PATCH(SchedulePatch,带读到的版本)。做哪几步存在数据源上(PUT /api/datasources/{id}/maintenance)。下一次什么 +// 时候、最近一次建了哪些任务、哪几步没执行和原因都照服务端给的写(触发记录的 inputs);剖析没开、调度没开时照实说到点会怎样。 +// +// 放在数据源卡片上的独立对话框(和「探查范围」一样),不塞进编辑表单:编辑表单只有一个「保存」,存的是连接配置;维护是一条 +// 单独的定时任务,存法不同,两个保存混在一起出错时说不清哪一样没存上。 +// =========================================================================== + +const STEPS: readonly MaintenanceStep[] = ['introspect', 'profile_needed', 'observe'] +/** 新开的维护缺省在凌晨三点:低峰时段 */ +const DEFAULT_MAINTENANCE_FREQ: Freq = { ...DEFAULT_FREQ, preset: 'daily', time: '03:00' } +const PREVIEW_DEBOUNCE_MS = 400 + +interface Form { + enabled: boolean + steps: MaintenanceStep[] + freq: Freq + timezone: string +} + +const formOf = (m: SourceMaintenance, s: MaintenanceSchedule | null): Form => ({ + enabled: s?.enabled ?? false, + steps: m.steps, + freq: s ? freqOf(s.cron) : DEFAULT_MAINTENANCE_FREQ, + timezone: s?.timezone ?? browserTimezone(), +}) + +export function SourceMaintenanceDialog({ source, onClose, onSaved }: { + source: { id: string; name: string } + onClose: () => void + /** 保存过了:卡片上的「定时维护」标签跟着变 */ + onSaved: () => void +}) { + const navigate = useNavigate() + const [data, setData] = useState(null) + const [loadError, setLoadError] = useState(null) + const [form, setForm] = useState
(null) + const initial = useRef('') + const [saving, setSaving] = useState(false) + const [error, setError] = useState<{ field?: string; message: string } | null>(null) + const [preview, setPreview] = useState<{ cron: string; tz: string; data?: SchedulePreview; error?: { field?: string; message: string } } | null>(null) + useTicker(30_000) + + const schedule = data?.schedules[0] ?? null + const extra = Math.max(0, (data?.schedules.length ?? 0) - 1) + + const load = async () => { + const got = await api.sourceChanges.maintenance(source.id) + setData(got) + const f = formOf(got, got.schedules[0] ?? null) + initial.current = JSON.stringify({ ...f, cron: cronOf(f.freq) }) + setForm(f) + setLoadError(null) + } + useEffect(() => { load().catch(setLoadError) }, [source.id]) + + const cron = form ? cronOf(form.freq) : '' + const freqIncomplete = form ? freqProblem(form.freq) : null + const sig = form ? JSON.stringify({ ...form, cron }) : '' + const dirty = !!form && sig !== initial.current + const noSteps = !!form && !form.steps.length + + // 频率的说法和接下来几次:服务端算(前端不解析 cron),停手 400 毫秒再问,新的一次发出时作废上一次 + useEffect(() => { + if (!form || !cron || freqIncomplete) { + setPreview(null) + return + } + const ctrl = new AbortController() + const tz = form.timezone + const timer = setTimeout(() => { + api.schedules.preview({ cron, timezone: tz, n: 3 }, { signal: ctrl.signal }) + .then((d) => setPreview({ cron, tz, data: d })) + .catch((e) => { + if ((e as { name?: string })?.name === 'AbortError') return + setPreview({ cron, tz, error: { field: e instanceof ApiError ? e.field : undefined, message: (e as Error).message } }) + }) + }, PREVIEW_DEBOUNCE_MS) + return () => { clearTimeout(timer); ctrl.abort() } + }, [cron, form?.timezone, freqIncomplete]) + + const freqError = error?.field === 'cron' ? error.message : preview?.error && preview.error.field !== 'timezone' ? preview.error.message + : form?.enabled ? freqIncomplete ?? undefined : undefined + const tzError = error?.field === 'timezone' ? error.message : preview?.error?.field === 'timezone' ? preview.error.message : undefined + + const save = async () => { + if (!form || !data || saving) return + if (noSteps || (form.enabled && (freqIncomplete || preview?.error))) return + setSaving(true) + setError(null) + try { + const was = JSON.parse(initial.current) as Form & { cron: string } + if (JSON.stringify(form.steps) !== JSON.stringify(was.steps)) await api.sourceChanges.putMaintenance(source.id, form.steps) + if (schedule) { + const body: SchedulePatch = { version: schedule.version } + if (cron && cron !== schedule.cron) body.cron = cron + if (form.timezone !== schedule.timezone) body.timezone = form.timezone + if (form.enabled !== schedule.enabled) body.enabled = form.enabled + if (Object.keys(body).length > 1) await api.schedules.update(schedule.id, body) + } else if (form.enabled) { + const body: ScheduleTargetCreate = { + name: T.scheduleName(source.name), target_kind: 'source_maintenance', target_id: source.id, cron, timezone: form.timezone, + enabled: true, overlap: 'skip', misfire_grace_s: DEFAULT_GRACE_S, + } + await api.schedules.create(body) + } + await load() + toast.ok(form.enabled ? T.saved : T.savedOff) + onSaved() + } catch (e) { + // 频率、时区不合规(422 带 field)落到那一项;版本不符(别人刚改过,409)、别的原因写在底部 + if (e instanceof ApiError && e.kind === 'http' && (e.status === 422 || e.status === 409)) setError({ field: e.field, message: e.message }) + else toast.error(e) + } finally { + setSaving(false) + } + } + + const set = (patch: Partial) => { + setError(null) + setForm((f) => f && ({ ...f, ...patch })) + } + const toggleStep = (step: MaintenanceStep) => { + if (!form) return + const on = form.steps.includes(step) + set({ steps: STEPS.filter((s) => (s === step ? !on : form.steps.includes(s))) }) + } + + const profileSteps = !!form && form.steps.some((s) => s !== 'introspect') + const blocked = !form || noSteps || (form.enabled && (!!freqIncomplete || !!preview?.error)) + + return ( + + {error && !error.field &&

{error.message}

} + + + + )}> + {!data || !form + ? (loadError ? void load().catch(setLoadError)} /> : ) + : !data.maintained + ?

{T.notMaintained}

+ : ( +
+

{T.intro}

+ +
+
+

{T.enable}

+ + + set({ enabled: v })} /> + +
+

{T.enableHint}

+ {form.enabled && ( +

+ {T.offPeak} +

+ )} + {!data.scheduler_on && ( +

+ {T.schedulerOff} +

+ )} +
+ +
+ {T.stepsTitle} +

+ {noSteps ? T.stepsNone : T.stepsHint} +

+ {STEPS.map((step) => ( + + ))} + {!data.profile_enabled && profileSteps && ( +

{T.profileOff}

+ )} +
+ +
+ set({ freq: { ...form.freq, ...patch } })} setTimezone={(tz) => set({ timezone: tz })} + freqError={freqError} tzError={tzError} /> + {preview?.data && ( +

+ {preview.data.description_text} + {preview.data.next.length > 0 && · {preview.data.next.map((n) => formatDateTime(n)).join('、')}} +

+ )} +
+ + {schedule && { + onClose() + navigate(`/data/catalog/${encodeURIComponent(source.id)}?pjob=${encodeURIComponent(id)}`) + }} />} + {extra > 0 &&

{T.extra(extra)}

} +
+ )} +
+ ) +} + +/** 已经有维护定时:下一次什么时候、最近一次建了哪些任务、哪几步没执行和原因(触发记录的 inputs),到定时任务页看全部触发 */ +function LastRun({ schedule: s, onOpenProfileJob }: { + schedule: MaintenanceSchedule; onOpenProfileJob: (id: string) => void +}) { + const fire = s.last_fire + const inputs = fire?.inputs ?? {} + const at = fire ? parseServerTime(fire.fire_at) : null + return ( +
+

+ + {s.description_text} + · + +

+

{T.lastTitle}

+ {!fire + ?

{T.lastNone}

+ : ( + <> +

+ {at ? formatDateTime(at) : fire.fire_at} + +

+ {(inputs.jobs ?? []).length > 0 && ( +
    + {(inputs.jobs ?? []).map((j) => ( +
  • + {T.lastJob(j.step.split('+').map((x) => T.step[x as MaintenanceStep] ?? x).join('、'), + T.lastJobKind[j.kind] ?? j.kind, j.id, j.tables ?? null)} + {/* 剖析任务点得进进度和结果(数据目录页的剖析对话框);探查任务进行中时进度就在这张数据源卡片上,没有另外的页面 */} + {j.kind === 'profile' && ( + + )} +
  • + ))} +
+ )} + {(inputs.skipped ?? []).map((k) => ( +

+ {T.lastSkipped(T.step[k.step] ?? k.step, k.reason)} +

+ ))} + {!(inputs.jobs ?? []).length && !(inputs.skipped ?? []).length && fire.reason && ( +

{T.lastReason(fire.reason)}

+ )} + + )} + {T.history} +
+ ) +} + +/** 卡片上的「定时维护」标签:开着时写频率、暂停时照实写;没有维护定时不画 */ +export function MaintenanceChip({ maintenance }: { maintenance: { enabled: boolean; description_text: string }[] }) { + const first = maintenance[0] + const text = useMemo(() => (first ? (first.enabled ? T.chip(first.description_text) : T.chipPaused) : ''), [first]) + if (!first) return null + return ( + + {text} + + ) +} diff --git a/frontend/src/pages/catalog/CatalogPage.tsx b/frontend/src/pages/catalog/CatalogPage.tsx index f2f58f17..cdfc38f0 100644 --- a/frontend/src/pages/catalog/CatalogPage.tsx +++ b/frontend/src/pages/catalog/CatalogPage.tsx @@ -2,7 +2,7 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react' import type { KeyboardEvent as ReactKeyboardEvent, ReactNode } from 'react' import { useNavigate, useParams, useSearchParams } from 'react-router-dom' import { - ArrowLeft, BookMarked, Database, Download, ListFilter as ListFilterIcon, ScanSearch, Sparkles, TriangleAlert, Upload, + ArrowLeft, BookMarked, Clock, Database, Download, ListFilter as ListFilterIcon, ScanSearch, Sparkles, Upload, } from 'lucide-react' import clsx from 'clsx' import { ApiError, api } from '../../api/client' @@ -13,13 +13,14 @@ import { EmptyState, ErrorState, PageHeader, Skeleton, Spinner, rovingTarget, to import { useCatalog, useDatasources, useOnReconnect } from '../../store/catalog' import { errorMessage } from '../../lib/errors' import { formatNumber } from '../../lib/format' +import { sourceAnchor } from '../../lib/sourceAnchor' import { - CATALOG_DRAFT_TEXT as DT, CATALOG_REVIEW_TEXT as RT, CATALOG_TEXT as CT, CATALOG_TRANSFER_TEXT as XT, PROFILE_TEXT as PT, - SCHEMA_PARTIAL_TEXT as SPT, + CATALOG_DRAFT_TEXT as DT, CATALOG_REVIEW_TEXT as RT, CATALOG_TEXT as CT, CATALOG_TRANSFER_TEXT as XT, JOB_STRIP_TEXT as JT, + PROFILE_TEXT as PT, SCHEMA_PARTIAL_TEXT as SPT, SOURCE_CHANGE_TEXT as GT, } from '../../lib/terms' import { StatusLegend, SystemNotesNotice } from './parts' import { DraftDialog } from './DraftDialog' -import { isJobActive, jobBlank, retryableCount, useDraftJob, waitsForProfile } from './draftJob' +import { isJobActive, jobBlank, retryableCount, useDraftJob } from './draftJob' import type { DraftPreset, DraftStartBody } from './draftJob' import { ProfileDialog } from './ProfileDialog' import { ProfileSettingsDialog } from './ProfileSettings' @@ -32,7 +33,12 @@ import { TableIndex } from './TableIndex' import { ReviewOverview, ReviewQueue } from './ReviewQueue' import { ImportDialog } from './TransferDialog' import { itemKey, reviewParamsOf, writeReviewParams } from './review' -import type { CatalogView } from './review' +import { ChangesQueue } from './ChangesQueue' +import { ChangeDetail, ChangesOverview } from './ChangeDetail' +import { changesParamsOf, clearChanges, isChangesView, writeChangesParams, writtenTables } from './changes' +import { JobStrip, stripItems, useSourceJobs } from './JobStrip' +import type { StripItem } from './JobStrip' +import type { SourceChangeCategory, SourceChangeCounts, SourceChangeResolveOut, SourceChangeStatusFilter } from '../../typesSource' import { dueCount, filterCounts, listParamsOf, progressOf, rowFromDetail, usageSummary, visibleRows, writeListParams } from './model' import type { ListFilter, ListParams, ListSort } from './model' @@ -48,13 +54,17 @@ import type { ListFilter, ListParams, ListSort } from './model' // 清单的搜索词、筛选、排序和「只看运行中查询过的表」记在地址的查询参数里(q / filter / sort / used),刷新、前进后退 // 都不丢;在清单和表详情之间跳转时带着它们走。 // -// 左栏有两个视图:表清单(逐表看)和审阅队列(view=review:跨表按优先级列出推断项,成批确认、驳回,筛选记在 kind / -// rtable 里)。从队列点开一项,右栏是所在表的详情,地址带着那一项(item),详情滚到那一项、列出用过这张表的运行。 +// 左栏有三个视图:表清单(逐表看)、审阅队列(view=review:跨表按优先级列出推断项,成批确认、驳回,筛选记在 kind / +// rtable 里)和变化(view=changes:探查、取值观察测到的结构变化、取值变化,按要处理 / 提醒 / 说明分组,筛选记在 cstatus / +// ccat 里)。从队列点开一项,右栏是所在表的详情,地址带着那一项(item),详情滚到那一项、列出用过这张表的运行;从变化里点开 +// 一条(change),右栏是它的依据、影响面和处理。 // 页头的「导出目录」「导入目录」把整理好的目录带到另一个实例(导入先预览,确认后才写)。 // -// 起草、剖析都是服务端的后台任务(draftJob.ts、profileJob.ts),状态在这一页上:关掉对话框照常进行,页头的「起草」「数据 -// 剖析」换成进度入口(「起草 3/20」「剖析 12/40」「剖析 排队中」),点开对应的对话框;结束时右下角提示,可以回到结果。 -// 同一个数据源的起草和剖析按建立顺序排队:一个在排队、进行中时,另一个建出来是排队,前一个结束后自动开始。 +// 起草、剖析都是服务端的后台任务(draftJob.ts、profileJob.ts),状态在这一页上:关掉对话框照常进行;结束时右下角提示,可以 +// 回到结果。同一个数据源的探查、起草、剖析在同一条队里排队:一个在排队、进行中时,另一个建出来是排队,前一个结束后自动开始。 +// 进度只在页头下方的后台任务条上(JobStrip,按服务端给的队列先后列出,next-round 05 (g));页头的「起草」「数据剖析」只是 +// 入口,进行中换转圈、排队中换时钟,读屏名称写状态、不写数字。 +// 地址里带 pjob / djob 时打开那个剖析、起草任务的进度(定时维护的「最近一次」从数据源页链过来)。 // =========================================================================== export function CatalogPage() { @@ -135,7 +145,12 @@ export function CatalogPage() { carriedRef.current = carried /** 审阅视图:view=review 时左栏是审阅队列;kind、rtable 是队列的筛选;item 是右栏从队列打开的那一项 */ const review = useMemo(() => reviewParamsOf(params), [params]) - const inReview = review.view === 'review' + /** 变化视图:view=changes 时左栏是变化列表;cstatus、ccat 是筛选;change 是右栏打开的那一条 */ + const inChanges = isChangesView(params) + const changesParams = useMemo(() => changesParamsOf(params), [params]) + const pageView: PageView = inChanges ? 'changes' : review.view + const inReview = pageView === 'review' + const openChangeId = inChanges ? changesParams.change || null : null const [selected, setSelected] = useState>(() => new Set()) /** * 起草对话框开着没有、打开时显示什么:setup 选范围开始;job 显示页面跟着的任务(进行中的进度、刚结束的结果)。 @@ -248,12 +263,12 @@ export function CatalogPage() { const onGuidanceSaved = useCallback((g: string | null) => { setData((cur) => cur && ({ ...cur, guidance: g })) }, []) - // 上方的进度:跟着的任务在进行就按它;跟着的就是清单说的那一个、已经结束了,就不信清单(清单还没重取回来) + // 起草任务现在的样子:跟着的任务在进行就按它;跟着的就是清单说的那一个、已经结束了,就不信清单(清单还没重取回来) const listActive = isJobActive(data?.active_job) ? data?.active_job ?? null : null - const stripJob: { done: number; total: number; status: string; waiting_for?: string | null } | null = draftJob && (draftActive || draftJob.id === listActive?.id) + const stripJob: { id: string; done: number; total: number; status: string; waiting_for?: string | null } | null = draftJob && (draftActive || draftJob.id === listActive?.id) ? (draftActive ? draftJob : null) : listActive - // 调用接连没写出来的提醒:关着对话框时也要看得到(上方的进度带着它) + // 调用接连没写出来的提醒:关着对话框时也要看得到(后台任务条上起草那一行带着它) const stripNotice = draftActive ? draftJob?.notice?.trim() || null : null const options = savedOptions?.sourceId === sourceId ? savedOptions.options : source?.options @@ -288,7 +303,7 @@ export function CatalogPage() { const profileActive = isJobActive(profileJob) // 页头的剖析进度:同起草,跟着的任务在进行就按它;跟着的就是清单说的那一个、已经结束了,就不信清单 const listProfile = isJobActive(data?.active_profile_job) ? data?.active_profile_job ?? null : null - const profileStrip: { done: number; total: number; status: string; waiting_for?: string | null } | null = + const profileStrip: { id: string; done: number; total: number; status: string; waiting_for?: string | null } | null = profileJob && (profileActive || profileJob.id === listProfile?.id) ? (profileActive ? profileJob : null) : listProfile /** 跟着的剖析任务里各表的状态(表清单的「剖析」一格据此写排队、剖析中) */ const profileLive = useMemo(() => new Map( @@ -320,6 +335,83 @@ export function CatalogPage() { toast.error(e) } }, [sourceId, profile.job?.id, profile.track]) + /** 打开起草对话框显示某个任务:页面跟着的那一个直接打开;别的(变化里「重新起草」建的)先取来 */ + const openDraftJob = useCallback(async (jobId: string) => { + if (draft.job?.id === jobId) { + setDraftOpen({ view: 'job' }) + return + } + const src = sourceId + try { + const { job } = await api.dataCatalog.draftJob(src, jobId) + if (sourceRef.current !== src) return + draft.track(job) + setDraftOpen({ view: 'job' }) + } catch (e) { + toast.error(e) + } + }, [sourceId, draft.job?.id, draft.track]) + const openJob = useCallback((kind: 'draft' | 'profile', id: string) => { + if (kind === 'draft') void openDraftJob(id) + else void openProfileJob(id) + }, [openDraftJob, openProfileJob]) + // 地址里带着 pjob / djob(定时维护的「最近一次」从数据源页链过来):打开那个任务的进度,参数用过就去掉 + const linkedJob = params.get('pjob') ? `profile:${params.get('pjob')}` : params.get('djob') ? `draft:${params.get('djob')}` : '' + const linkedSeen = useRef('') + useEffect(() => { + if (!linkedJob || linkedSeen.current === linkedJob || !data) return + linkedSeen.current = linkedJob + const [kind, id] = linkedJob.split(':') + openJob(kind as 'draft' | 'profile', id) + const next = new URLSearchParams(window.location.search) + next.delete('pjob') + next.delete('djob') + navigate({ search: next.toString() ? `?${next.toString()}` : '' }, { replace: true }) + }, [linkedJob, data, openJob, navigate]) + + /** + * 后台任务条:服务端排好的队列(探查、起草、剖析),起草、剖析换上页面跟着的任务的数和提醒。页面上的任务换了状态就重取一次 + * 队列(排在谁后面跟着变) + */ + const profileNotice = profileActive ? profileJob?.notice?.trim() || null : null + const jobsKick = `${stripJob?.id ?? ''}:${stripJob?.status ?? ''}|${profileStrip?.id ?? ''}:${profileStrip?.status ?? ''}` + const sourceJobs = useSourceJobs(sourceId, jobsKick) + const jobItems = useMemo(() => stripItems(sourceJobs.items, { + draft: stripJob ? { kind: 'draft', id: stripJob.id, status: stripJob.status, done: stripJob.done, total: stripJob.total, + waiting_for: stripJob.waiting_for ?? null, position: null, notice: stripNotice } : null, + profile: profileStrip ? { kind: 'profile', id: profileStrip.id, status: profileStrip.status, done: profileStrip.done, + total: profileStrip.total, waiting_for: profileStrip.waiting_for ?? null, position: null, notice: profileNotice } : null, + }), [sourceJobs.items, stripJob, stripNotice, profileStrip, profileNotice]) + const openStripItem = useCallback((item: StripItem) => { + if (item.kind === 'introspect') navigate(`${back}${sourceAnchor(sourceId)}`) + else openJob(item.kind, item.id) + }, [navigate, back, sourceId, openJob]) + + // 变化视图:标签上的「N 条要处理」、处理过一条之后左栏重取 + const [changeCounts, setChangeCounts] = useState<{ src: string; counts: SourceChangeCounts } | null>(null) + const [changesSeq, setChangesSeq] = useState(0) + const loadChangeCounts = useCallback(() => { + const src = sourceId + api.sourceChanges.list(src, { limit: 1 }).then((r) => setChangeCounts({ src, counts: r.counts }), () => {}) + }, [sourceId]) + useEffect(() => { loadChangeCounts() }, [loadChangeCounts]) + useOnReconnect(loadChangeCounts) + const actionCount = changeCounts?.src === sourceId ? changeCounts.counts.by_severity.action : 0 + const onChangeCounts = useCallback((counts: SourceChangeCounts) => setChangeCounts({ src: sourceId, counts }), [sourceId]) + // 队列里有任务走了(结束了):探查读到了新的结构、剖析做了取值观察,都可能记下新的变化;探查改了表结构,清单也要重取 + // (起草、剖析结束时页面跟着的任务自己会重取清单) + const queuedIds = useRef<{ src: string; ids: Map }>({ src: '', ids: new Map() }) + useEffect(() => { + const now = new Map(sourceJobs.items.map((i) => [i.id, i.kind])) + const prev = queuedIds.current.src === sourceId ? queuedIds.current.ids : new Map() + queuedIds.current = { src: sourceId, ids: now } + const gone = [...prev].filter(([id]) => !now.has(id)).map(([, kind]) => kind) + if (!gone.length) return + loadChangeCounts() + setChangesSeq((n) => n + 1) + if (gone.includes('introspect')) void load() + }, [sourceJobs.items, sourceId, loadChangeCounts, load]) + const tableProfile: TableProfileProps = useMemo(() => ({ job: profileJob, onStart: profileTable, onShowActive: profile.showActive, onSettings: () => setProfileSettingsOpen(true), onOpenJob: (id: string | null) => void openProfileJob(id), @@ -376,10 +468,29 @@ export function CatalogPage() { const next = fn(new URLSearchParams(window.location.search)).toString() navigate({ search: next ? `?${next}` : '' }, opts) }, [navigate]) - /** 换视图(表清单 / 审阅队列):记进地址(后退回到上一个视图),右栏正在看的表不动 */ - const setView = useCallback((v: CatalogView) => { - patchSearch((prev) => writeReviewParams(prev, { view: v, item: '' })) + /** 换视图(表清单 / 审阅队列 / 变化):记进地址(后退回到上一个视图),右栏正在看的表不动 */ + const setView = useCallback((v: PageView) => { + patchSearch((prev) => (v === 'changes' + ? writeChangesParams(writeReviewParams(prev, { item: '' }), { open: true }) + : clearChanges(writeReviewParams(prev, { view: v, item: '' })))) }, [patchSearch]) + /** 变化视图的筛选:替换当前这条历史 */ + const setChangesFilter = useCallback((p: { status?: SourceChangeStatusFilter; category?: SourceChangeCategory | null }) => { + patchSearch((prev) => writeChangesParams(prev, p), { replace: true }) + }, [patchSearch]) + /** 打开一条变化:右栏是它,地址带着它(表详情里点「去处理」也走这里,切到变化视图) */ + const openChange = useCallback((id: string) => { + const q = writeChangesParams(clearChanges(carriedRef.current), { open: true, change: id, status: changesParams.status, + category: changesParams.category }).toString() + navigate(`${base}${q ? `?${q}` : ''}`) + }, [navigate, base, changesParams.status, changesParams.category]) + /** 窄屏:从一条变化回到变化列表 */ + const closeChange = useCallback(() => patchSearch((prev) => writeChangesParams(prev, { change: '' })), [patchSearch]) + /** 从一条变化去看它说的那张表:切回表清单 */ + const openChangeTable = useCallback((t: string) => { + const q = clearChanges(writeReviewParams(carriedRef.current, { view: 'list', item: '' })).toString() + navigate(`${base}/${encodeURIComponent(t)}${q ? `?${q}` : ''}`) + }, [navigate, base]) /** 审阅队列的筛选:替换当前这条历史(筛选不该占满后退键) */ const setReviewParams = useCallback((p: { kind?: CatalogReviewKind; table?: string }) => { patchSearch((prev) => writeReviewParams(prev, p), { replace: true }) @@ -394,6 +505,17 @@ export function CatalogPage() { setDrafted((cur) => ({ seq: cur.seq + 1, tables: new Set(tables) })) void load() }, [load]) + /** 处理过一条变化:左栏重取、计数跟着变;动了目录的重取清单、重新载入那几张表;建了任务的跟上(清单带着进行中的任务) */ + const onChangeResolved = useCallback((out: SourceChangeResolveOut) => { + setChangesSeq((n) => n + 1) + loadChangeCounts() + const tables = writtenTables(out.change, out.catalog_versions) + if (tables.length) onTablesWritten(tables) + else if (out.job) void load() + if (out.job) sourceJobs.refresh() + }, [loadChangeCounts, onTablesWritten, load, sourceJobs.refresh]) + /** 右栏有内容(打开了一张表、一条变化):窄屏一次只显示一栏,这时显示右栏 */ + const rightOpen = !!table || !!openChangeId const activeReviewKey = inReview && table && review.item ? itemKey({ table_name: table, path: review.item }) : null const proposedTotal = useMemo(() => rows.reduce((n, r) => n + (r.in_schema ? r.counts.proposed : 0), 0), [rows]) @@ -438,36 +560,28 @@ export function CatalogPage() { {CT.draft} ) - // 页头的「数据剖析」「起草」:任务在排队、进行中时换成进度入口(「剖析 12/40」「剖析 排队中」),点开对应的对话框 - // (关掉对话框任务照常进行);排在另一种任务后面时悬停写明。窄屏只留图标,标题才放得下(进度在读屏的名称里) - const profileQueued = profileStrip?.status === 'queued' - const profileLabel = profileStrip ? (profileQueued ? PT.headerQueued : PT.headerRunning(profileStrip.done, profileStrip.total)) : PT.action - const profileTitle = profileStrip - ? (profileQueued ? (profileStrip.waiting_for === 'draft' ? PT.headerQueuedDraft - : profileStrip.waiting_for === 'introspect' ? PT.headerQueuedIntrospect : PT.headerQueuedHint) - : PT.headerRunningHint(profileStrip.done, profileStrip.total)) - : PT.actionHint - // 最近查询多数超时的提醒:对话框关着时也要看得到(页头的入口换成提醒色,悬停写全文) - const profileNotice = profileActive ? profileJob?.notice?.trim() || null : null + // 页头的「数据剖析」「起草」只是入口(进度在下方的后台任务条上,05 (g)):进行中换转圈图标、排队中换时钟,读屏名称写状态、 + // 不写数字;点开对应的对话框(关掉对话框任务照常进行)。窄屏只留图标,标题才放得下 + const headState = (job: { status: string } | null) => (job?.status === 'queued' ? 'queued' : job?.status === 'running' ? 'running' : null) + const headLabel = (label: string, state: 'queued' | 'running' | null) => + (state === 'running' ? JT.headRunning(label) : state === 'queued' ? JT.headQueued(label) : label) + const headIcon = (state: 'queued' | 'running' | null, idle: ReactNode) => + (state === 'running' ? : state === 'queued' ? : idle) + const profileState = headState(profileStrip) + const profileLabel = headLabel(PT.action, profileState) const headerProfile = ( - ) - const draftQueued = stripJob?.status === 'queued' - const draftLabel = stripJob ? (draftQueued ? DT.headerQueued : DT.headerRunning(stripJob.done, stripJob.total)) : CT.draft - const draftTitle = stripJob - ? (draftQueued ? (waitsForProfile(stripJob) ? DT.headerQueuedProfile - : stripJob.waiting_for === 'introspect' ? DT.headerQueuedIntrospect : DT.headerQueuedHint) - : DT.headerRunningHint(stripJob.done, stripJob.total)) - : CT.draft + const draftState = headState(stripJob) + const draftLabel = headLabel(CT.draft, draftState) const headerDraft = ( - ) @@ -495,10 +609,15 @@ export function CatalogPage() { } else { body = (
-