From 238330b9c1b4aa6b8e9f96b32006c1e9944ed27c Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Fri, 16 Oct 2026 19:31:21 +0800 Subject: [PATCH] =?UTF-8?q?W4-d=20=E5=90=8E=E7=AB=AF=EF=BC=9A=E7=BB=99?= =?UTF-8?q?=E7=9B=AE=E5=BD=95=E5=89=8D=E7=AB=AF=E7=9A=84=E5=8F=98=E5=8C=96?= =?UTF-8?q?=E3=80=81=E8=A7=82=E5=AF=9F=E3=80=81=E5=90=8E=E5=8F=B0=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1=E3=80=81=E5=AE=9A=E6=97=B6=E7=BB=B4=E6=8A=A4=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3=E8=A1=A5=E9=BD=90=E7=95=8C=E9=9D=A2=E8=A6=81=E7=9A=84?= =?UTF-8?q?=E5=87=A0=E6=A0=B7=EF=BC=9B=E6=89=93=E5=BC=80=20source.changed?= =?UTF-8?q?=20=E7=9A=84=E8=AE=A2=E9=98=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 第 3 波的变化接口只给了 agentlab-change/1 的原样,界面要的几样(能不能做、依据什么、现在怎样)如果让前端自己判,就违背 「界面层不放逻辑」。这一提交由服务端给出,界面只照写: - 变化列表、单条、处理的回答里每条另带 category(结构 / 取值)、basis_text(依据的口径,照观察时记下的写整表、窗口、块采样、 按存储顺序的前 N 行)、actions(现在能做哪几个处理动作,做不了的写原因:剖析没开、表结构里没有了、影响面里没有目录项)、 job(处理时建的任务,界面能点进进度)、影响面里每个目录项现在的状态(清理过、改过的照实)。召回拉的 list_changes 不带这些。 - 列表能按 category 筛(和 kind 一起给时取交集,一种都对不上就是空的,不是不筛),能按 status=handled 看处理过的(最近测到的 在前:处理过的不再按要不要先处理排)。 - GET /api/datasources/upkeep:数据源列表一次取齐各源要处理的变化条数(卡片角标)和指向它的维护定时(频率、开没开、下一次)。 - GET /{id}/observations:表详情「结构与取值变化」一节的最近几次观察,各带口径;码值的占比、按日行数的合计由服务端算;现在 被遮罩的列(观察时还没遮的)整组不给,遮罩在读的那一刻再核。 - GET /{id}/background-jobs:这个源进行中和排队的探查、起草、剖析,按同源队的先后排好(05 (g):排队顺序取服务端的,界面 不自己排),各带 waiting_for。 - GET /{id}/maintenance:维护定时换成定时任务页同一个形状(改频率、开关照样带 version 走 PATCH),最近一次触发带 inputs (建了哪些任务、哪几步没执行和原因),另说调度开没开、这个源要不要维护(上传表格、记录表不要)。 - notify:source.changed 打开订阅(只推全局渠道,按源汇总不归哪个人),链接落到目录页的「变化」标签页。 test_notify_events 里 W4-d 那一行加上;test_source_changes 里原先假设订阅没开的那句改成「没有渠道订阅时推不出去」。 data/source_changes.py(W4-a 的文件)只在末尾加了 available_actions、counts_by_source、list_handled 和 HANDLED; data/catalog_profile_observe.py 末尾加了 key_text、recent。新测试 tests/test_source_upkeep.py。 --- backend/app/api/source_changes.py | 230 ++++++++++++- backend/app/data/catalog_profile_observe.py | 75 +++++ backend/app/data/source_changes.py | 81 +++++ backend/app/notify/service.py | 6 +- backend/tests/test_notify_events.py | 3 +- backend/tests/test_source_changes.py | 5 +- backend/tests/test_source_upkeep.py | 343 ++++++++++++++++++++ 7 files changed, 718 insertions(+), 25 deletions(-) create mode 100644 backend/tests/test_source_upkeep.py diff --git a/backend/app/api/source_changes.py b/backend/app/api/source_changes.py index cc95b76b..904fc00b 100644 --- a/backend/app/api/source_changes.py +++ b/backend/app/api/source_changes.py @@ -8,12 +8,26 @@ - GET /{id}/changes/{change_id} 一条变化 - POST /{id}/changes/{change_id}/resolve 处理:{action, note?, codes?}。动作落到目录已有的写入接口并记目录版本 (data/source_changes.handle);误报必须写理由(422);已经处理过的 409 -- GET /{id}/maintenance 定时维护要做哪几步、剖析开没开、指向这个源的维护定时 +- GET /{id}/maintenance 定时维护要做哪几步、剖析开没开、指向这个源的维护定时(定时任务页同一个形状,最近一次触发 + 带 inputs:建了哪些任务、哪几步没执行和原因)、调度开没开、这个源要不要维护 - PUT /{id}/maintenance 改要做哪几步:{steps: [introspect | profile_needed | observe]}。定时本身(cron、时区、开关) 在 /api/schedules 里建(target_kind=source_maintenance),这里只管步骤 -访问密钥默认不放(auth/keys.py 的 KEY_ROUTES 没有登记这几条)。界面(审阅页「变化」标签页、数据源角标、定时维护设置)在 -第 4 波。 +第 4 波(W4-d)给界面加的: + +- GET /upkeep 数据源列表一次取齐:各源要处理的变化条数(卡片角标)、指向它的维护定时(频率、开没开、 + 下一次) +- GET /{id}/observations?table= 表详情「结构与取值变化」一节:这张表最近几次观察(行数、码值分布、最新业务日期、按日 + 行数),各带口径;遮罩在读的那一刻再核 +- GET /{id}/background-jobs 这个源进行中和排队的后台任务(探查、起草、剖析),按同源队的先后排好(目录页的后台任务条, + 界面不自己排) + +变化列表、单条、处理的回答里,每条变化在 agentlab-change/1 之外另带几项只给界面看的(_views;召回拉的 list_changes 不带): +category(struct 结构 / value 取值)、basis_text(依据的口径)、actions(现在能做哪几个处理动作,做不了的写原因)、job(处理时 +建的任务,能点进进度)、影响面里每个目录项现在的状态(affects.catalog[*].now:清理过、改过的照实)。列表另能按 category 筛、 +按 status=handled 看处理过的。 + +访问密钥默认不放(auth/keys.py 的 KEY_ROUTES 没有登记这几条)。 """ from __future__ import annotations @@ -21,10 +35,11 @@ from typing import Any from fastapi import APIRouter, Depends, Header, HTTPException, Query from pydantic import BaseModel, Field +from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from app.api.deps import actor_of -from app.data import catalog, source_changes as sc, source_maintenance +from app.data import catalog, schema_diff as sd, source_changes as sc, source_maintenance from app.db.base import get_session from app.db.models import DataSource @@ -32,6 +47,14 @@ router = APIRouter(prefix="/api/datasources", tags=["source-changes"]) #: 一页最多几条 LIMIT_MAX = 200 +#: 列表的状态筛选:pending 开着的和知道了的(缺省)、handled 处理过的(解决了的、标为误报的)、all、或者某一个状态 +STATUS_FILTERS = ("pending", "handled", "all", *sc.STATUSES) +#: 变化的两类:结构变化、取值变化(叫法分开,和已有的「目录有变化」也分开,总方案 X10) +CATEGORIES = {"struct": tuple(sd.STRUCT_KINDS), "value": tuple(sc.VALUE_KINDS)} +#: 处理动作建的任务是哪一种 +_JOB_OF = {"redraft": "draft", "reprofile": "profile"} +#: 关联覆盖率下降的依据(剖析核对关系时抽样) +RELATION_BASIS = "关联关系抽样核对" async def _source(session: AsyncSession, source_id: str) -> DataSource: @@ -41,19 +64,128 @@ async def _source(session: AsyncSession, source_id: str) -> DataSource: return row +# ---- 给界面的几样(要登记在 /{source_id}/… 前面:/upkeep 只有一段) + + +@router.get("/upkeep") +async def upkeep(session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """数据源列表一次取齐:{items: [{source_id, changes: {pending, open, by_severity}, maintenance: [{id, enabled, cron, + timezone, description_text, next_run_at}]}]}。卡片角标「N 个变化要处理」按 changes.by_severity.action;没有变化、没有 + 维护定时的源也列出来(计数为 0、maintenance 为空)。""" + from app.db.models import Schedule + from app.scheduler import service + + ids = list((await session.execute(select(DataSource.id))).scalars()) + counts = await sc.counts_by_source(session, ids) + schedules: dict[str, list[dict[str, Any]]] = {} + for r in (await session.execute(select(Schedule).where(Schedule.target_kind == "source_maintenance") + .order_by(Schedule.created_at))).scalars(): + schedules.setdefault(r.target_id or "", []).append({ + "id": r.id, "enabled": r.enabled, "cron": r.cron, "timezone": r.timezone, + "description_text": _describe(r.cron, r.timezone), + "next_run_at": service.as_utc(r.next_run_at).isoformat() if r.next_run_at else None}) + empty = {"pending": 0, "open": 0, "by_severity": dict.fromkeys(sc.SEVERITIES, 0)} + return {"items": [{"source_id": i, "changes": counts.get(i, empty), "maintenance": schedules.get(i, [])} + for i in ids]} + + +def _describe(cron_expr: str, tz: str) -> str: + from app.scheduler import cron + + try: + return cron.describe(cron_expr, tz) + except cron.CronError: + return f"按 cron 表达式 {cron_expr}({tz})" + + +def _basis_text(row: Any) -> str | None: + """依据的口径(取值变化):整表、业务日期最近 N 天的窗口、块采样、按存储顺序的前 N 行、统计估算……照观察时记下的写。 + 结构变化的依据是探查读到的结构版本(前后各一版),界面照 struct_version_before / after 写,这里为 None。""" + from app.data import catalog_profile_observe as observe + + if row.kind in sd.STRUCT_KINDS: + return None + basis = row.basis if isinstance(row.basis, dict) else {} + if basis.get("method") == "relation_sample": + return RELATION_BASIS + return observe.key_text(basis.get("key")) or (observe.basis_text(basis) if basis.get("method") else None) + + +async def _views(session: AsyncSession, source: DataSource, rows: list[Any]) -> list[dict[str, Any]]: + """接口给界面的变化:change_out 之外另带 category、basis_text、actions、job、影响面里每个目录项现在的状态(见模块说明)。""" + from app.data import catalog_profile, table_versions + + if not rows: + return [] + try: + view = await table_versions.resolve_source(session, source) + except table_versions.SnapshotMissing: + view = source + tables = set(((getattr(view, "schema_cache", None) or {}).get("tables") or {})) + settings = catalog_profile.profile_settings(source.options) + profile_reason = None + if not settings.enabled: + try: + catalog_profile.ensure_enabled(source) + except catalog_profile.ProfileDisabled as e: + profile_reason = str(e) + entries = await catalog.read_catalog(session, source.id) if any( + (r.affects or {}).get("catalog") for r in rows) else {} + slots = {t: catalog._slots(e.notes) for t, e in entries.items()} + + def now_of(ref: dict[str, Any]) -> dict[str, Any] | None: + try: + item = slots.get(str(ref.get("table") or ""), {}).get(catalog.parse_path(str(ref.get("path") or ""))) + except catalog.CatalogPathError: + return None + return {"status": item.get("status"), "source": item.get("source")} if item else None + + out: list[dict[str, Any]] = [] + for r in rows: + v = sc.change_out(r) + v["category"] = "struct" if r.kind in sd.STRUCT_KINDS else "value" + v["basis_text"] = _basis_text(r) + v["actions"] = sc.available_actions(r, in_schema=bool(r.table_name) and r.table_name in tables, + profile_enabled=settings.enabled, profile_reason=profile_reason) + resolution = r.resolution or {} + kind = _JOB_OF.get(str(resolution.get("action") or "")) + v["job"] = {"kind": kind, "id": resolution["job_id"]} if kind and resolution.get("job_id") else None + affects = dict(v["affects"]) + affects["catalog"] = [{**ref, "now": now_of(ref)} for ref in affects.get("catalog") or [] + if isinstance(ref, dict)] + v["affects"] = affects + out.append(v) + return out + + @router.get("/{source_id}/changes") async def list_changes(source_id: str, status: str = Query("pending"), kind: list[str] | None = Query(None), - table: str | None = Query(None), limit: int = Query(100, ge=1, le=LIMIT_MAX), - offset: int = Query(0, ge=0), session: AsyncSession = Depends(get_session)) -> dict[str, Any]: - """{items: [变化(agentlab-change/1)], total, counts: {pending, open, by_severity}}。""" - if status not in ("pending", "all", *sc.STATUSES): - raise HTTPException(422, f"不认识的状态:{status}。可选:pending、all、{'、'.join(sc.STATUSES)}") + category: str | None = Query(None), table: str | None = Query(None), + limit: int = Query(100, ge=1, le=LIMIT_MAX), offset: int = Query(0, ge=0), + session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """{items: [变化(agentlab-change/1,另带给界面的几项)], total, counts: {pending, open, by_severity}}。 + + status=pending(缺省)按「使用次数权重 × 严重程度」排,同分新的在前;status=handled 是处理过的,最近测到的在前。 + category=struct / value 只看结构变化 / 取值变化(和 kind 一起给时取两者都对得上的;一种都对不上就是空的)。""" + if status not in STATUS_FILTERS: + raise HTTPException(422, f"不认识的状态:{status}。可选:{'、'.join(STATUS_FILTERS)}") if kind and (bad := [k for k in kind if k not in sc.KINDS]): raise HTTPException(422, f"不认识的变化种类:{'、'.join(bad)}") + if category is not None and category not in CATEGORIES: + raise HTTPException(422, f"不认识的变化类别:{category}。可选:struct(结构变化)、value(取值变化)") row = await _source(session, source_id) - rows, total = await sc.list_for_source(session, row, status=status, kinds=kind, table=table, limit=limit, - offset=offset) - return {"items": [sc.change_out(r) for r in rows], "total": total, "counts": await sc.counts(session, row.id)} + kinds = list(kind) if kind else None + if category is not None: + kinds = [k for k in (kinds or CATEGORIES[category]) if k in CATEGORIES[category]] + counts = await sc.counts(session, row.id) + if kinds == []: + return {"items": [], "total": 0, "counts": counts} + if status == "handled": + rows, total = await sc.list_handled(session, row.id, kinds=kinds, table=table, limit=limit, offset=offset) + else: + rows, total = await sc.list_for_source(session, row, status=status, kinds=kinds, table=table, limit=limit, + offset=offset) + return {"items": await _views(session, row, rows), "total": total, "counts": counts} @router.get("/{source_id}/changes/{change_id}") @@ -62,7 +194,7 @@ async def get_change(source_id: str, change_id: str, session: AsyncSession = Dep found = await sc.load(session, row.id, change_id) if found is None: raise HTTPException(404, "这条变化不存在,可能已随数据源删除") - return {"change": sc.change_out(found)} + return {"change": (await _views(session, row, [found]))[0]} class ResolveIn(BaseModel): @@ -83,8 +215,8 @@ async def resolve_change(source_id: str, change_id: str, payload: ResolveIn, x_a if found is None: raise HTTPException(404, "这条变化不存在,可能已随数据源删除") try: - return await sc.handle(session, row, found, payload.action, actor=actor_of(x_actor), note=payload.note, - codes=payload.codes) + out = await sc.handle(session, row, found, payload.action, actor=actor_of(x_actor), note=payload.note, + codes=payload.codes) except sc.ChangeInvalid as e: await session.rollback() raise HTTPException(422, str(e)) from e @@ -94,18 +226,42 @@ async def resolve_change(source_id: str, change_id: str, payload: ResolveIn, x_a except catalog.CatalogConflict as e: await session.rollback() raise HTTPException(409, "这张表的数据目录刚被别人改过,请刷新后再处理") from e + # 处理完照实给这条变化现在的样子(另带给界面的几项):目录项清理过的、动作做完的都跟着变 + await session.refresh(row) + fresh = await sc.load(session, row.id, change_id) + if fresh is not None: + out["change"] = (await _views(session, row, [fresh]))[0] + return out @router.get("/{source_id}/maintenance") async def get_maintenance(source_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: - """{steps, profile_enabled, schedules}:要做哪几步、剖析开没开(没开时剖析、观察两步到点照实记没执行)、指向这个源的 - 维护定时。""" + """{steps, profile_enabled, maintained, scheduler_on, schedules}:要做哪几步、剖析开没开(没开时剖析、观察两步到点照实记 + 没执行)、这个源要不要维护(上传表格、记录表不要)、调度开没开(关着时到点不会触发)、指向这个源的维护定时。 + + 定时是定时任务页同一个形状(api/schedules 的 Schedule:改频率、开关照样带 version 走 PATCH /api/schedules/{id}),最近 + 一次触发另带 inputs({steps, jobs: [{step, kind, id, tables}], skipped: [{step, reason}]}:建了哪些任务、哪几步没执行和原因)。""" + from app import scheduler + from app.api.schedules import _schedule_outs from app.data import catalog_profile + from app.db.models import Schedule, ScheduleFire row = await _source(session, source_id) + rows = list((await session.execute(select(Schedule).where( + Schedule.target_kind == "source_maintenance", Schedule.target_id == row.id) + .order_by(Schedule.created_at))).scalars()) + outs = await _schedule_outs(session, rows) + fire_ids = [o["last_fire"]["id"] for o in outs if o.get("last_fire")] + inputs = dict((await session.execute(select(ScheduleFire.id, ScheduleFire.inputs).where( + ScheduleFire.id.in_(fire_ids)))).all()) if fire_ids else {} + for o in outs: + if o.get("last_fire"): + o["last_fire"]["inputs"] = dict(inputs.get(o["last_fire"]["id"]) or {}) return {"steps": source_maintenance.steps_of(row.options), "profile_enabled": catalog_profile.profile_settings(row.options).enabled, - "schedules": await source_maintenance.schedules_for(session, row.id)} + "maintained": (row.origin or "manual") not in ("upload", "records"), + "scheduler_on": scheduler.scheduler_enabled(), + "schedules": outs} class MaintenanceIn(BaseModel): @@ -123,3 +279,41 @@ async def put_maintenance(source_id: str, payload: MaintenanceIn, x_actor: str | row.options = source_maintenance.with_steps(row.options, [str(s) for s in payload.steps], actor=actor_of(x_actor)) await session.commit() return await get_maintenance(source_id, session) + + +@router.get("/{source_id}/observations") +async def table_observations(source_id: str, table: str = Query(..., min_length=1, max_length=500), + session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """表详情「结构与取值变化」一节:{table, series: [{kind, column, points: [{at, basis_key, basis_text, data}], + basis_changed}]}(catalog_profile_observe.recent)。现在被遮罩的列不给;没观察过是空的。""" + from app.data import catalog_profile, catalog_profile_observe as observe + + row = await _source(session, source_id) + masked = catalog_profile.masked_of(row.options) + return {"table": table, "series": await observe.recent(session, row.id, table, masked=masked)} + + +@router.get("/{source_id}/background-jobs") +async def background_jobs(source_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """这个源进行中和排队的后台任务(目录页的后台任务条,05 (g)):{items: [{kind: introspect | draft | profile, id, status, + total, done, waiting_for, position}]}。 + + 按同源队(core/job_runtime.lanes)的先后排:position 是在队里的位置(0 是正在做的);刚建、后台还没排进队的(position 为 + null)排在队里的后面、按建的先后。waiting_for 是各任务自己的说法(排在谁后面),界面照写,不自己排。""" + from app.core import job_runtime as runtime + from app.data import catalog_jobs, catalog_profile_jobs, introspect_jobs + + row = await _source(session, source_id) + found: list[tuple[int | None, Any, dict[str, Any]]] = [] + for kind, module in (("introspect", introspect_jobs), ("draft", catalog_jobs), ("profile", catalog_profile_jobs)): + job = await module.active_job(session, row.id) + brief = await module.job_brief(session, job) if job is not None else None + if job is None or brief is None: + continue + ahead = runtime.lanes.ahead(row.id, job.id) + position = len(ahead) if ahead is not None else None + found.append((position, job.created_at, { + "kind": kind, "id": job.id, "status": brief["status"], "total": brief.get("total", 0), + "done": brief.get("done", 0), "waiting_for": brief.get("waiting_for"), "position": position})) + found.sort(key=lambda f: (f[0] is None, f[0] if f[0] is not None else 0, f[1])) + return {"items": [f[2] for f in found]} diff --git a/backend/app/data/catalog_profile_observe.py b/backend/app/data/catalog_profile_observe.py index 044355e1..60cee5ef 100644 --- a/backend/app/data/catalog_profile_observe.py +++ b/backend/app/data/catalog_profile_observe.py @@ -516,3 +516,78 @@ async def observed_facts(session: AsyncSession, source_id: str, __all__ = ["KIND_LABEL", "OBSERVE_CODES_MAX", "OBSERVE_EVERY_H", "OBSERVE_KINDS", "OBSERVER", "SAMPLE_LADDER", "WINDOW_DAYS", "basis_key", "basis_text", "code_columns", "coverage_drops", "execute", "last_observed", "observe_reason", "observed_facts", "plan_observations", "record_coverage_drops", "sample_percent", "save"] + + +# ========================================================================== +# 给人看的(第 4 波 W4-d:表详情「结构与取值变化」一节、变化的依据) +# ========================================================================== + +#: 表详情里每种观察给最近几次 +RECENT_SHOWN = 6 + + +def key_text(key: str | None) -> str | None: + """口径的规范写法(basis_key)→ 说法:变化记录里只存了规范写法,依据的口径照它写。认不出的为 None。""" + text = str(key or "") + head, _, rest = text.partition(":") + if head == "window": + column, _, days = rest.rpartition(":") + return basis_text({"method": "window", "column": column, "days": days}) if column and days else None + if head == "sample": + try: + return basis_text({"method": "sample", "percent": float(rest)}) + except ValueError: + return None + if head == "first_rows" and rest: + return basis_text({"method": "first_rows", "rows": rest}) + if text in ("full", "stats", "count"): + return basis_text({"method": text}) + return None + + +def _shown(kind: str, data: Mapping[str, Any]) -> dict[str, Any]: + """一次观察给界面的样子:码值按行数从多到少、带占比;按日行数按日期排成列表;其余原样。数都由这里算,界面只照写。""" + if kind == "codes_observe": + pairs = [(str(p[0]), int(p[1])) for p in data.get("values") or [] + if isinstance(p, (list, tuple)) and len(p) == 2 and isinstance(p[1], (int, float))] + rows = int(data.get("rows") or sum(n for _, n in pairs)) + pairs.sort(key=lambda p: (-p[1], p[0])) + return {"values": [[c, n, round(n / rows, 4) if rows else 0.0] for c, n in pairs], "rows": rows, + "truncated": bool(data.get("truncated"))} + if kind == "daily_volume": + days = data.get("days") if isinstance(data.get("days"), Mapping) else {} + series = sorted((str(d), int(n)) for d, n in days.items() if isinstance(n, (int, float))) + return {"since": data.get("since"), "until": data.get("until"), "days": [list(p) for p in series], + "total": sum(n for _, n in series), "days_with_data": sum(1 for _, n in series if n > 0)} + if kind == "freshness": + return {"max": data.get("max")} + return {"rows": data.get("rows"), "at_least": bool(data.get("at_least")), "method": data.get("method")} + + +async def recent(session: AsyncSession, source_id: str, table: str, *, masked: set[str], + per: int = RECENT_SHOWN) -> list[dict[str, Any]]: + """这张表最近几次观察:按(种类、列)分组,种类按 OBSERVE_KINDS 的顺序、同种按列名;每组最近 per 次,新的在前: + [{kind, column, points: [{at, basis_key, basis_text, data}], basis_changed}]。basis_changed:这几次的口径不全相同(口径不同 + 不比,界面照实说)。遮罩在读的这一刻再核(look D6 的同一个道理):观察时还没遮、现在被遮罩的列整组不给。""" + rows = (await session.execute(select(TableObservation).where( + TableObservation.source_id == source_id, TableObservation.table_name == table) + .order_by(TableObservation.at.desc(), TableObservation.id.desc()))).scalars() + groups: dict[tuple[str, str], list[TableObservation]] = {} + for r in rows: + if r.kind not in OBSERVE_KINDS or (r.column_name and r.column_name.lower() in masked): + continue + group = groups.setdefault((r.kind, r.column_name or ""), []) + if len(group) < per: + group.append(r) + order = {k: i for i, k in enumerate(OBSERVE_KINDS)} + out: list[dict[str, Any]] = [] + for (kind, column), group in sorted(groups.items(), key=lambda kv: (order[kv[0][0]], kv[0][1])): + out.append({"kind": kind, "column": column, + "points": [{"at": r.at.isoformat() if r.at else None, "basis_key": r.basis_key, + "basis_text": basis_text(r.basis or {}), "data": _shown(kind, r.data or {})} + for r in group], + "basis_changed": len({r.basis_key for r in group}) > 1}) + return out + + +__all__ += ["RECENT_SHOWN", "key_text", "recent"] diff --git a/backend/app/data/source_changes.py b/backend/app/data/source_changes.py index 876bcdf2..e6c6bd2f 100644 --- a/backend/app/data/source_changes.py +++ b/backend/app/data/source_changes.py @@ -524,3 +524,84 @@ __all__ = [ "record", "record_all", "resolve", "summary", "triggers_recall", "without_paths", ] + + +# ========================================================================== +# 给界面的(第 4 波 W4-d:审阅页「变化」标签页、数据源列表的角标) +# ========================================================================== + +#: 处理过的:解决了的、标为误报的(「变化」标签页的「处理过的」) +HANDLED = ("resolved", "dismissed") +#: 重新起草有用的结构变化:新表、列和键变了(起草按新的结构重写这张表的推断项;人工确认过的项照旧不动) +_REDRAFT_KINDS = frozenset({sd.TABLE_ADDED, sd.COLUMN_ADDED, sd.COLUMN_REMOVED, sd.COLUMN_TYPE, sd.COLUMN_NULLABLE, + sd.KEY, sd.FK, sd.COMMENT}) +#: 这张表现在不在表结构里、或者说的是整个源的:起草、剖析都无从做起 +_NO_TABLE_KINDS = frozenset({sd.TABLE_REMOVED, sd.TABLE_LEFT_SCOPE, sd.TABLE_REFLECT_FAILED, sd.SCOPE}) +NOT_IN_SCHEMA = "表结构里已经没有这张表,先重新探查结构" +NO_CATALOG_ITEMS = "影响面里没有目录项,不需要清理" + + +def available_actions(row: Any, *, in_schema: bool, profile_enabled: bool, profile_reason: str | None = None, + ) -> list[dict[str, Any]]: + """这条变化现在能做哪几个处理动作(「变化」标签页照它画按钮,界面不自己判):[{action, ok, reason}],按 ACTIONS 的顺序。 + + 已经处理过的(resolved / dismissed)一个都不给。只列对这种变化有意义的动作:加入码值只给「出现了新码值」,重新起草只给 + 新表和列、键的变化,重新剖析给说的是一张还在的表的变化(注释变了除外);清理目录项只在影响面里有目录项时能点。前置核对 + 和 handle 一致;建任务时才知道的(起草模型、同源正忙)不在这里判,点了照实 422。ok 为假时 reason 写为什么做不了。""" + if row.status in HANDLED: + return [] + kind = row.kind + has_table = bool(row.table_name) and kind not in _NO_TABLE_KINDS + out: list[dict[str, Any]] = [] + + def put(action: str, reason: str | None = None) -> None: + out.append({"action": action, "ok": reason is None, "reason": reason}) + + if (row.affects or {}).get("catalog"): + put("clean_catalog", None if _catalog_paths(row) else NO_CATALOG_ITEMS) + if kind == CODES_NEW and row.column_name: + put("add_codes") + if has_table and kind in _REDRAFT_KINDS: + put("redraft", None if in_schema else NOT_IN_SCHEMA) + if has_table and kind != sd.COMMENT: + reason = None if in_schema else NOT_IN_SCHEMA + if reason is None and not profile_enabled: + reason = profile_reason or "这个数据源未开启数据剖析" + put("reprofile", reason) + if row.status == "open": + put("acknowledge") + put("dismiss") + return out + + +async def counts_by_source(session: AsyncSession, source_ids: Iterable[str] | None = None) -> dict[str, dict[str, Any]]: + """各源还要人看的变化(同 counts 的形状),一条查询取齐(数据源列表的角标)。没有变化的源不在里面。""" + query = select(SourceChange.source_id, SourceChange.status, SourceChange.severity, func.count()).where( + SourceChange.status.in_(PENDING)).group_by(SourceChange.source_id, SourceChange.status, SourceChange.severity) + if source_ids is not None: + query = query.where(SourceChange.source_id.in_(list(source_ids))) + out: dict[str, dict[str, Any]] = {} + for source_id, status, severity, n in (await session.execute(query)).all(): + one = out.setdefault(source_id, {"pending": 0, "open": 0, "by_severity": dict.fromkeys(SEVERITIES, 0)}) + one["by_severity"][severity] = one["by_severity"].get(severity, 0) + n + one["pending"] += n + if status == "open": + one["open"] += n + return out + + +async def list_handled(session: AsyncSession, source_id: str, *, kinds: Iterable[str] | None = None, + table: str | None = None, limit: int = 100, offset: int = 0) -> tuple[list[SourceChange], int]: + """处理过的变化(解决了的、标为误报的),最近测到的在前:处理过的不再按要不要先处理排。返回 (这一页, 一共几条)。""" + filt = [SourceChange.source_id == source_id, SourceChange.status.in_(HANDLED)] + if kinds: + filt.append(SourceChange.kind.in_(list(kinds))) + if table: + filt.append(SourceChange.table_name == table) + total = int((await session.execute(select(func.count()).select_from(SourceChange).where(*filt))).scalar() or 0) + rows = list((await session.execute(select(SourceChange).where(*filt).order_by(SourceChange.seq.desc()) + .offset(offset).limit(limit))).scalars()) + return rows, total + + +__all__ += ["HANDLED", "available_actions", "counts_by_source", "list_handled"] diff --git a/backend/app/notify/service.py b/backend/app/notify/service.py index 8d0fea86..f231f031 100644 --- a/backend/app/notify/service.py +++ b/backend/app/notify/service.py @@ -68,7 +68,7 @@ EVENTS: dict[str, Event] = { "review.done": Event("我提交审核的应用有结果", ("global", "user")), # ---- 本轮登记的(总方案 X6):scopes 为空的还没有推送;实现推送的那一块只改自己这一行的 scopes,消息照模块说明的规矩 ---- # 数据源有要处理的结构 / 取值变化(03 B3):只推 severity 为 action 的,按源汇总;源名、各级别条数、链接 - "source.changed": Event("数据源有变化要处理", ()), + "source.changed": Event("数据源有变化要处理", ("global",)), # 口径升版后召回重算,数真变了(02 第 7 项):答案的主人有 recall 待办;口径名、版本、链接,不放新旧值和差值 "ledger.recall": Event("口径升版,答案的数变了", ()), # 定时评测有题从通过变成不通过 / 评测没跑完(01 第 15 项):题集名、通过数、退步题数、链接;不放期望值和答案里的数。 @@ -434,7 +434,7 @@ async def notify_source_changed(source_id: str) -> int: """数据源有要处理的变化(source.changed,03 B3 ④、目录-D9):只推还要人看(开着、知道了还没处理)的「要处理」级变化, 按源汇总;消息里只有源名、各级别条数和链接,不放表名、列名、取值。探查、剖析任务结束时有新的「要处理」变化才调。 - 这一行的 scopes 第 4 波有了「变化」标签页才打开:现在 _channels 推不出去,这里返回 0(照实:没有界面看不了的通知不发)。""" + 只推全局渠道(按源汇总,不归哪个人);链接落到数据目录页的「变化」标签页(第 4 波 W4-d 有了这个标签页才打开订阅)。""" from app.data import source_changes from app.db.models import DataSource @@ -454,7 +454,7 @@ async def notify_source_changed(source_id: str) -> int: lines = [("数据源", f"「{row.name}」"), ("要处理", f"{by['action']} 条"), ("提醒", f"{by.get('warning', 0)} 条"), ("说明", f"{by.get('info', 0)} 条")] message = {"event": event, "status": "action", "title": f"数据源「{row.name[:80]}」有 {by['action']} 个变化要处理", - "lines": lines, "links": [("去看看", f"{base}/data/catalog/{source_id}")] if base else [], + "lines": lines, "links": [("去看看", f"{base}/data/catalog/{source_id}?view=changes")] if base else [], "subject": {"source_id": source_id}} return await _push(channels, message, subject_kind="source", subject_id=source_id, event=event, run_id=None, task_id=None) diff --git a/backend/tests/test_notify_events.py b/backend/tests/test_notify_events.py index 4957bfb5..5d79fcc6 100644 --- a/backend/tests/test_notify_events.py +++ b/backend/tests/test_notify_events.py @@ -39,7 +39,8 @@ PUSHED_THIS_ROUND = { "eval.failed": {"global"}, "task.digest": {"user"}, "key.expiring": {"global", "user"}, - # 第 4 波:W4-d 打开 source.changed 时在这里加它推给的几条路 + # 第 4 波 W4-d:数据源有变化要处理,按源汇总、只推全局渠道(notify_source_changed);链接落到「变化」标签页 + "source.changed": {"global"}, # (两行之间隔一行不动的,两个包各改各的、合并不撞) # 第 4 波:W4-a 打开 ledger.recall 时在这里加它推给的几条路 } diff --git a/backend/tests/test_source_changes.py b/backend/tests/test_source_changes.py index 1eb0cf60..95ed16ff 100644 --- a/backend/tests/test_source_changes.py +++ b/backend/tests/test_source_changes.py @@ -8,7 +8,7 @@ source_changes.record 记一条。守的是: apply_patch(含义待填写、依据写变化编号)、记目录版本;知道了只改状态;剖析没开时「重新剖析」照实 422; - 同一个变化再次测到只更新 last_seen_at、seen_count; - list_changes 按 seq 游标拉、affecting 按表反查、triggers_recall 只认结构类和触及规则的新码值; -- source.changed 的推送:订阅还没打开时推不出去;打开后(测试里临时打开)消息里只有源名、条数和链接。 +- source.changed 的推送:没有渠道订阅时推不出去;全局渠道订阅了,消息里只有源名、条数和链接(第 4 波打开了订阅)。 """ from __future__ import annotations @@ -194,7 +194,7 @@ async def test_cursor_pull_affecting_and_recall_triggers(shop): async def test_source_changed_message_has_only_name_counts_and_link(shop, monkeypatch): sid, name = shop["id"], shop["name"] - assert await service.notify_source_changed(sid) == 0, "订阅还没打开(第 4 波):推不出去" + assert await service.notify_source_changed(sid) == 0, "没有渠道订阅它:推不出去" got: list[httpx.Request] = [] def handler(request: httpx.Request) -> httpx.Response: @@ -203,7 +203,6 @@ async def test_source_changed_message_has_only_name_counts_and_link(shop, monkey monkeypatch.setattr(core, "_transport", httpx.MockTransport(handler)) monkeypatch.setattr(core, "RETRY_DELAYS", (0, 0, 0)) - monkeypatch.setitem(service.EVENTS, "source.changed", service.Event("数据源有变化要处理", ("global",))) async with SessionLocal() as session: from app.core.crypto import encrypt from app.db.models import NotifyChannel diff --git a/backend/tests/test_source_upkeep.py b/backend/tests/test_source_upkeep.py new file mode 100644 index 00000000..95a4acc7 --- /dev/null +++ b/backend/tests/test_source_upkeep.py @@ -0,0 +1,343 @@ +"""数据源变化、定时维护、后台任务条的界面接口(next-round 03 B2–B4、05 (g),第 4 波 W4-d)。 + +第 3 波的变化接口只给了 agentlab-change/1 的原样;界面要的几样由服务端给,界面不自己判(界面层不放逻辑): + +- 变化列表每条另带 category(结构 / 取值)、basis_text(依据的口径,照实写整表 / 窗口 / 块采样 / 前 N 行)、actions + (现在能做哪几个处理动作、做不了的写原因)、job(处理时建的任务)、影响面里每个目录项现在的状态(清理过的照实为空); +- 列表能按 category 筛、能看「处理过的」(解决了的、标为误报的,最近的在前); +- GET /upkeep:数据源列表一次取齐各源要处理的变化条数(角标)和定时维护的频率; +- GET /{id}/observations:表详情「结构与取值变化」一节的最近几次观察,各带口径;遮罩在读的那一刻再核; +- GET /{id}/background-jobs:这个源进行中和排队的后台任务(探查、起草、剖析),按同源队的先后排好; +- GET /maintenance:定时维护的定时任务是定时任务页同一个形状,最近一次触发带上 inputs(建了哪些任务、哪几步没执行和原因), + 另说调度开没开; +- source.changed 打开订阅(只推全局渠道),链接落到目录页的「变化」标签页。 + +SQLite 夹具是虚构的店铺库;名字、数字都是假的。 +""" +from __future__ import annotations + +import json +import sqlite3 +import uuid +from datetime import timedelta + +import httpx +import pytest +from httpx import ASGITransport, AsyncClient +from sqlalchemy import delete, select + +from app.core import job_runtime as runtime +from app.data import catalog, source_changes as sc +from app.db.base import SessionLocal, utcnow +from app.db.models import CatalogDraftJob, CatalogProfileJob, DataSource, NotifyChannel, Schedule, ScheduleFire +from app.db.models_sources import IntrospectJob, SourceChange, TableObservation +from app.main import app +from app.notify import core, service +from app.scheduler import service as scheduler_service +from tests.fixtures.sources import drop_source, introspect_now + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +@pytest.fixture +async def shop(client, tmp_path): + """探查建基线、写目录(status 列上有人工确认的码值和中文名),删掉 status 列再探查:得到一条要处理的 column_removed。""" + path = tmp_path / "shop.db" + db = sqlite3.connect(path) + db.executescript(""" + CREATE TABLE orders (id INTEGER PRIMARY KEY, status VARCHAR(10), channel VARCHAR(8), amount NUMERIC(10, 2)); + INSERT INTO orders VALUES (1, 'paid', 'web', 3), (2, 'void', 'shop', 4); + """) + db.commit() + db.close() + name = f"upk_{uuid.uuid4().hex[:8]}" + r = await client.post("/api/datasources", json={"name": name, "kind": "sqlite", "database": str(path), + "readonly": True}) + sid = r.json()["id"] + await introspect_now(client, sid) + async with SessionLocal() as session: + await catalog.write_entry(session, sid, "orders", { + "label": catalog.make_item("订单", "human"), + "columns": {"status": {"codes": catalog.make_item({"paid": "已支付", "void": "作废"}, "human"), + "label": catalog.make_item("状态", "human")}, + "channel": {"codes": catalog.make_item({"web": "线上"}, "profile", "proposed")}}, + }, if_version=0, actor="测试员") + db = sqlite3.connect(path) + db.execute("ALTER TABLE orders DROP COLUMN status") + db.commit() + db.close() + await introspect_now(client, sid) + made: list[str] = [] + yield {"id": sid, "name": name, "schedules": made} + async with SessionLocal() as session: + await session.execute(delete(Schedule).where(Schedule.id.in_(made))) + await session.commit() + await drop_source(sid) + + +async def _codes_new(sid: str) -> SourceChange: + async with SessionLocal() as session: + row = await session.get(DataSource, sid) + change, _new = await sc.record(session, row, sc.NewChange( + kind=sc.CODES_NEW, table="orders", column="channel", severity="warning", + after={"codes": ["app"], "shares": {"app": 0.4}}, basis={"method": "first_rows", "key": "first_rows:100000"}, + affects={"catalog": [], "templates": [], "rules_touched": False}, key=("first_rows:100000", ("app",))), + sc.Detected(job_kind="profile", job_id="job-upk", trigger="schedule")) + await session.commit() + return change + + +def _by_kind(items: list[dict]) -> dict[str, dict]: + return {c["kind"]: c for c in items} + + +# -------------------------------------------------------------------------- +# 变化列表:界面要的几样由服务端给 +# -------------------------------------------------------------------------- + + +async def test_changes_carry_category_basis_actions_and_current_status(client, shop): + sid = shop["id"] + await _codes_new(sid) + body = (await client.get(f"/api/datasources/{sid}/changes")).json() + by = _by_kind(body["items"]) + removed, fresh = by["column_removed"], by["codes_new"] + assert removed["category"] == "struct" and fresh["category"] == "value" + assert fresh["basis_text"] == "按存储顺序取前 100000 行", "依据的口径照实写" + assert removed["basis_text"] is None, "结构变化的依据是探查读到的结构版本(界面照 struct_version 写)" + + acts = {a["action"]: a for a in removed["actions"]} + assert list(acts) == ["clean_catalog", "redraft", "reprofile", "acknowledge", "dismiss"], acts + assert acts["clean_catalog"]["ok"] and acts["redraft"]["ok"] and acts["dismiss"]["ok"] + assert not acts["reprofile"]["ok"] and "剖析" in acts["reprofile"]["reason"], "剖析没开:照实写做不了的原因" + assert "add_codes" not in acts, "只有出现了新码值才给「加入码值」" + assert [a["action"] for a in fresh["actions"]][:1] == ["add_codes"] + + refs = removed["affects"]["catalog"] + assert refs and all(r["now"]["status"] == "confirmed" for r in refs if r["path"].startswith("columns.status")) + assert removed["job"] is None + + +async def test_category_and_handled_filters(client, shop): + sid = shop["id"] + change = await _codes_new(sid) + url = f"/api/datasources/{sid}/changes" + value = (await client.get(url, params={"category": "value"})).json() + assert [c["kind"] for c in value["items"]] == ["codes_new"] and value["total"] == 1 + struct = (await client.get(url, params={"category": "struct"})).json() + assert struct["items"] and all(c["category"] == "struct" for c in struct["items"]) + none = (await client.get(url, params={"category": "value", "kind": ["column_removed"]})).json() + assert none["items"] == [] and none["total"] == 0, "种类和类别对不上:一条都没有,不是不筛" + assert (await client.get(url, params={"category": "nope"})).status_code == 422 + + assert (await client.get(url, params={"status": "handled"})).json()["items"] == [] + r = await client.post(f"{url}/{change.id}/resolve", json={"action": "dismiss", "note": "测试库造的取值"}) + assert r.status_code == 200, r.text + got = r.json()["change"] + assert got["status"] == "dismissed" and got["actions"] == [], "处理过的不再给动作" + handled = (await client.get(url, params={"status": "handled"})).json() + assert [c["id"] for c in handled["items"]] == [change.id] and handled["total"] == 1 + assert change.id not in {c["id"] for c in (await client.get(url)).json()["items"]} + + +async def test_acknowledged_change_offers_no_second_acknowledge_and_clean_shows_items_gone(client, shop): + sid = shop["id"] + url = f"/api/datasources/{sid}/changes" + removed = _by_kind((await client.get(url)).json()["items"])["column_removed"] + r = await client.post(f"{url}/{removed['id']}/resolve", json={"action": "acknowledge"}) + assert r.status_code == 200 + acts = [a["action"] for a in r.json()["change"]["actions"]] + assert "acknowledge" not in acts and "clean_catalog" in acts, "知道了还没处理:还能清理、不能再点知道了" + r = await client.post(f"{url}/{removed['id']}/resolve", json={"action": "clean_catalog"}, headers={"X-Actor": "reviewer"}) + assert r.status_code == 200, r.text + got = r.json() + assert got["catalog_version"] >= 2 and got["change"]["resolution"]["catalog_version"] == got["catalog_version"] + gone = [ref for ref in got["change"]["affects"]["catalog"] if ref["path"].startswith("columns.status")] + assert gone and all(ref["now"] is None for ref in gone), "清理过的项现在不在目录里了:照实为空" + + +# -------------------------------------------------------------------------- +# 数据源列表:角标和定时维护一次取齐 +# -------------------------------------------------------------------------- + + +async def test_upkeep_gives_badge_counts_and_maintenance_per_source(client, shop): + sid = shop["id"] + r = await client.post("/api/schedules", json={ + "name": "店铺库的定时维护", "target_kind": "source_maintenance", "target_id": sid, "cron": "0 3 * * *", + "timezone": "Asia/Shanghai", "enabled": True, "overlap": "skip", "misfire_grace_s": 3600}) + assert r.status_code in (200, 201), r.text + shop["schedules"].append(r.json()["id"]) + items = {i["source_id"]: i for i in (await client.get("/api/datasources/upkeep")).json()["items"]} + mine = items[sid] + assert mine["changes"]["by_severity"]["action"] >= 1 and mine["changes"]["pending"] >= 1 + [m] = mine["maintenance"] + assert m["enabled"] is True and m["timezone"] == "Asia/Shanghai" and "03:00" in m["description_text"] + assert m["next_run_at"], "开着的维护写下一次什么时候" + + +# -------------------------------------------------------------------------- +# 表详情:最近几次观察,各带口径;遮罩在读的那一刻再核 +# -------------------------------------------------------------------------- + + +async def test_observations_grouped_with_basis_and_masked_columns_hidden(client, shop): + sid = shop["id"] + now = utcnow() + async with SessionLocal() as session: + for i, rows in enumerate((120, 118)): + session.add(TableObservation(source_id=sid, table_name="orders", column_name="", kind="rows", + basis={"method": "count"}, basis_key="count", + data={"rows": rows, "at_least": False, "method": "count"}, + at=now - timedelta(days=i))) + session.add(TableObservation(source_id=sid, table_name="orders", column_name="channel", kind="codes_observe", + basis={"method": "sample", "percent": 1.0}, basis_key="sample:1", + data={"values": [["web", 30], ["shop", 10]], "rows": 40, "truncated": False}, at=now)) + session.add(TableObservation(source_id=sid, table_name="orders", column_name="channel", kind="codes_observe", + basis={"method": "first_rows", "rows": 1000}, basis_key="first_rows:1000", + data={"values": [["web", 5]], "rows": 5, "truncated": False}, + at=now - timedelta(days=2))) + session.add(TableObservation(source_id=sid, table_name="orders", column_name="amount", kind="codes_observe", + basis={"method": "full"}, basis_key="full", + data={"values": [["3", 1]], "rows": 1, "truncated": False}, at=now)) + await session.commit() + url = f"/api/datasources/{sid}/observations" + body = (await client.get(url, params={"table": "orders"})).json() + series = {(s["kind"], s["column"]): s for s in body["series"]} + rows = series[("rows", "")] + assert [p["data"]["rows"] for p in rows["points"]] == [120, 118], "新的在前" + assert rows["points"][0]["basis_text"] == "逐行数出" and rows["basis_changed"] is False + codes = series[("codes_observe", "channel")] + assert codes["points"][0]["basis_text"] == "块采样约 1%" + assert codes["points"][1]["basis_text"] == "按存储顺序取前 1000 行" + assert codes["basis_changed"] is True, "口径换过:照实说,不跨口径比" + assert codes["points"][0]["data"]["values"] == [["web", 30, 0.75], ["shop", 10, 0.25]], "占比由服务端算" + assert [s["kind"] for s in body["series"]][0] == "rows", "按观察种类的固定顺序" + + async with SessionLocal() as session: + row = await session.get(DataSource, sid) + row.options = {**(row.options or {}), "mask_columns": ["amount"]} + await session.commit() + body = (await client.get(url, params={"table": "orders"})).json() + assert ("codes_observe", "amount") not in {(s["kind"], s["column"]) for s in body["series"]}, \ + "现在被遮罩的列:观察时还没遮的也不给" + assert (await client.get(url, params={"table": "nope"})).json()["series"] == [] + + +# -------------------------------------------------------------------------- +# 后台任务条:按同源队的先后 +# -------------------------------------------------------------------------- + + +async def test_background_jobs_follow_the_lane_order(client, shop): + sid = shop["id"] + async with SessionLocal() as session: + intro = IntrospectJob(source_id=sid, status="queued", created_at=utcnow() - timedelta(seconds=30)) + draft = CatalogDraftJob(source_id=sid, status="running") + profile = CatalogProfileJob(source_id=sid, status="queued", scope="selected", settings={}) + session.add_all([intro, draft, profile]) + await session.commit() + ids = {"introspect": intro.id, "draft": draft.id, "profile": profile.id} + # 队里是 起草(在做)→ 剖析(排着);探查刚建、还没进队:排在队里的后面 + tickets = [runtime.lanes.enter(sid, "draft", ids["draft"]), runtime.lanes.enter(sid, "profile", ids["profile"])] + try: + items = (await client.get(f"/api/datasources/{sid}/background-jobs")).json()["items"] + assert [i["kind"] for i in items] == ["draft", "profile", "introspect"], items + assert [i["id"] for i in items] == [ids["draft"], ids["profile"], ids["introspect"]] + assert items[0]["position"] == 0 and items[1]["position"] == 1 and items[2]["position"] is None + assert items[1]["waiting_for"] == "draft", "排队的写等的是什么(服务端的 waiting_for)" + assert {"total", "done", "status"} <= set(items[0]) + finally: + for t in tickets: + t.release() + async with SessionLocal() as session: + for model, key in ((IntrospectJob, "introspect"), (CatalogDraftJob, "draft"), (CatalogProfileJob, "profile")): + await session.execute(delete(model).where(model.id == ids[key])) + await session.commit() + assert (await client.get(f"/api/datasources/{sid}/background-jobs")).json()["items"] == [] + + +# -------------------------------------------------------------------------- +# 定时维护:定时任务同一个形状,最近一次触发带 inputs +# -------------------------------------------------------------------------- + + +async def test_maintenance_returns_schedule_shape_and_last_fire_inputs(client, shop): + sid = shop["id"] + url = f"/api/datasources/{sid}/maintenance" + body = (await client.get(url)).json() + assert body["maintained"] is True and body["schedules"] == [] and isinstance(body["scheduler_on"], bool) + r = await client.post("/api/schedules", json={ + "name": "店铺库的定时维护", "target_kind": "source_maintenance", "target_id": sid, "cron": "30 2 * * *", + "timezone": "Asia/Shanghai", "enabled": True, "overlap": "skip", "misfire_grace_s": 3600}) + schedule_id = r.json()["id"] + shop["schedules"].append(schedule_id) + inputs = {"steps": ["introspect", "profile_needed", "observe"], + "jobs": [{"step": "introspect", "kind": "introspect", "id": "job-i", "tables": 3}], + "skipped": [{"step": "profile_needed", "reason": "数据源「x」未开启数据剖析"}]} + async with SessionLocal() as session: + session.add(ScheduleFire(schedule_id=schedule_id, fire_at=utcnow(), kind="scheduled", status="started", + reason="剖析需要的表没有执行:未开启", inputs=inputs)) + await session.commit() + [s] = (await client.get(url)).json()["schedules"] + assert s["id"] == schedule_id and s["version"] >= 1 and s["target_kind"] == "source_maintenance" + assert "02:30" in s["description_text"] and s["next_run_at"] + assert s["last_fire"]["status"] == "started" and s["last_fire"]["inputs"] == inputs + + async with SessionLocal() as session: + row = await session.get(DataSource, sid) + row.origin = "upload" + await session.commit() + assert (await client.get(url)).json()["maintained"] is False, "上传表格、记录表不给这一节" + + +# -------------------------------------------------------------------------- +# 通知:source.changed 打开订阅,链接落到「变化」标签页 +# -------------------------------------------------------------------------- + + +async def test_source_changed_is_subscribable_and_links_to_the_changes_tab(client, shop, monkeypatch): + assert service.EVENTS["source.changed"].scopes == ("global",), "只推全局渠道(按源汇总,不归哪个人)" + assert service.subscribe_problem("source.changed", "global") is None + assert service.subscribe_problem("source.changed", "user") is not None + got: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + got.append(request) + return httpx.Response(200, json={"ok": True}) + + monkeypatch.setattr(core, "_transport", httpx.MockTransport(handler)) + monkeypatch.setattr(core, "RETRY_DELAYS", (0, 0, 0)) + monkeypatch.setattr(service, "_base_url", _fake_base) + from app.core.crypto import encrypt + + async with SessionLocal() as session: + channel = NotifyChannel(name=f"变化-{uuid.uuid4().hex[:6]}", scope="global", format="generic", + url=encrypt("https://hooks.example.com/agentlab/upkeep-3b9e"), events=["source.changed"]) + session.add(channel) + await session.commit() + channel_id = channel.id + try: + assert await service.notify_source_changed(shop["id"]) == 1 + [request] = got + body = json.loads(request.content) + links = json.dumps(body, ensure_ascii=False) + assert f"/data/catalog/{shop['id']}?view=changes" in links, links + finally: + async with SessionLocal() as session: + await session.execute(delete(NotifyChannel).where(NotifyChannel.id == channel_id)) + await session.commit() + + +async def _fake_base(_session) -> str: + return "https://agentlab.example.com" + + +def test_schedule_target_columns_are_the_ones_the_page_uses(): + """数据源设置里建的维护定时:target_kind / target_id 写法和调度那一套同一处封装(X4)。""" + cols = scheduler_service.target_columns("source_maintenance", "src-x") + assert cols["target_kind"] == "source_maintenance" and cols["target_id"] == "src-x" and cols["workflow_id"] == ""