From 4819b2126920ba868edadd08986ae3bf12dbf0e2 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Fri, 16 Oct 2026 09:43:08 +0800 Subject: [PATCH 01/39] =?UTF-8?q?=E4=B8=8B=E4=B8=80=E8=BD=AE=E7=AC=AC=201?= =?UTF-8?q?=20=E6=B3=A2=E7=9A=84=E5=A5=91=E7=BA=A6=E9=AA=A8=E6=9E=B6?= =?UTF-8?q?=EF=BC=9A=E8=BF=90=E8=A1=8C=E5=8A=A0=20end=5Freason=EF=BC=88?= =?UTF-8?q?=E4=BA=BA=E5=B7=A5=E9=A9=B3=E5=9B=9E=20/=20=E6=89=8B=E5=8A=A8?= =?UTF-8?q?=E5=81=9C=E6=AD=A2=EF=BC=89=EF=BC=8C=E5=AE=9A=E6=97=B6=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1=E5=8A=A0=E7=9B=AE=E6=A0=87=E7=A7=8D=E7=B1=BB=E5=92=8C?= =?UTF-8?q?=20id=E3=80=81=E8=A7=A6=E5=8F=91=E8=AE=B0=E5=BD=95=E5=8A=A0?= =?UTF-8?q?=E8=AF=84=E6=B5=8B=20id=EF=BC=88=E5=AE=9A=E6=97=B6=E5=8F=AA?= =?UTF-8?q?=E6=9C=89=E4=B8=80=E5=A5=97=EF=BC=89=EF=BC=8C=E8=BF=90=E8=A1=8C?= =?UTF-8?q?=E6=9D=A5=E6=BA=90=E7=99=BB=E8=AE=B0=20eval=E3=80=81ledger=5Fis?= =?UTF-8?q?sue=E3=80=81reconcile=E3=80=81recall=EF=BC=88=E9=83=BD=E5=8F=AA?= =?UTF-8?q?=E8=83=BD=E7=94=B1=E6=9C=8D=E5=8A=A1=E7=AB=AF=E5=86=99=EF=BC=89?= =?UTF-8?q?=EF=BC=9B=E8=BF=81=E7=A7=BB=E5=8F=AA=E5=8A=A0=E4=B8=8D=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/db/base.py | 6 ++++++ backend/app/db/models.py | 11 +++++++++++ backend/app/engine/origin.py | 5 +++++ frontend/src/lib/terms.ts | 4 ++++ frontend/src/typesApp.ts | 1 + 5 files changed, 27 insertions(+) diff --git a/backend/app/db/base.py b/backend/app/db/base.py index 98e28bc1..40204684 100644 --- a/backend/app/db/base.py +++ b/backend/app/db/base.py @@ -239,6 +239,12 @@ _COLUMN_MIGRATIONS: list[tuple[str, str, str]] = [ ("apps", "scope", "VARCHAR(20) DEFAULT 'shared'"), ("apps", "owner", "VARCHAR(100)"), ("apps", "blueprint_id", "VARCHAR(32)"), + # 下一轮(docs/design/next-round/00-总方案.md X4、X9):运行为什么结束在 cancelled;定时的目标种类和 id; + # 定时触发建的那次评测。存量行:end_reason 为空,target_kind 是 workflow(target_id 由 scheduler 按 workflow_id 读) + ("runs", "end_reason", "VARCHAR(20)"), + ("schedules", "target_kind", "VARCHAR(20) DEFAULT 'workflow'"), + ("schedules", "target_id", "VARCHAR(32)"), + ("schedule_fires", "eval_run_id", "VARCHAR(32)"), ] #: 迁移补上的列要建的索引:(索引名, 表, 列或多列)。create_all 不碰已有的表,表上新列的索引也就不会建。 diff --git a/backend/app/db/models.py b/backend/app/db/models.py index 7e3ad023..6c11c6bf 100644 --- a/backend/app/db/models.py +++ b/backend/app/db/models.py @@ -654,6 +654,9 @@ class Run(Base, TimestampMixin): input: Mapped[dict[str, Any]] = mapped_column(default=dict) output: Mapped[dict[str, Any]] = mapped_column(default=dict) error: Mapped[str | None] = mapped_column(Text, default=None) + # 运行为什么结束在 cancelled(docs/design/next-round/00-总方案.md X9):rejected 人工驳回、stopped 手动停止。 + # 其余终态、加这一列之前的运行为空 + end_reason: Mapped[str | None] = mapped_column(String(20), default=None) usage: Mapped[dict[str, Any]] = mapped_column(default=dict) # tokens / cost / 各节点耗时 started_at: Mapped[datetime | None] = mapped_column(default=None) finished_at: Mapped[datetime | None] = mapped_column(default=None) @@ -975,6 +978,12 @@ class Schedule(Base, TimestampMixin): id: Mapped[str] = mapped_column(String(32), primary_key=True, default=new_id) name: Mapped[str] = mapped_column(String(200), default="") + # 定时的目标(next-round X4,定时只有一套):workflow 发起工作流运行;eval_suite 跑一次评测题集; + # source_maintenance 维护一个数据源(探查、剖析、取值观察)。加这一列之前的任务都是 workflow + target_kind: Mapped[str] = mapped_column(String(20), default="workflow") + # 目标的 id:workflow 时同 workflow_id;其余两种是题集 id、数据源 id + target_id: Mapped[str | None] = mapped_column(String(32), default=None, index=True) + # 非空(老表的约束,只加不改的迁移放宽不了):目标不是工作流时存空串,读写都经 scheduler/service 里的一处封装 workflow_id: Mapped[str] = mapped_column(String(32), index=True) # 保存时的工作流名。工作流还在时接口给它现在的名字,删掉之后才用这一份 workflow_name: Mapped[str] = mapped_column(String(200), default="") @@ -1020,6 +1029,8 @@ class ScheduleFire(Base): # started | skipped_overlap | skipped_misfire | failed_to_start status: Mapped[str] = mapped_column(String(30), default="started") run_id: Mapped[str | None] = mapped_column(String(32), default=None, index=True) + # 目标是评测题集时,这一次触发建的那次评测(target_kind=eval_suite,next-round X4) + eval_run_id: Mapped[str | None] = mapped_column(String(32), default=None) reason: Mapped[str | None] = mapped_column(Text, default=None) # 渲染后的输入,就是交给运行的那一份 inputs: Mapped[dict[str, Any]] = mapped_column(default=dict) diff --git a/backend/app/engine/origin.py b/backend/app/engine/origin.py index dd50ce0a..6e418db8 100644 --- a/backend/app/engine/origin.py +++ b/backend/app/engine/origin.py @@ -24,6 +24,11 @@ ORIGIN_KINDS = ( "api", # 直接调接口,没说是哪 "trial", # 蓝图的试运行(探索运行,docs/design/normal-user-apps.md N4) "personal_app", # 个人应用的刷新(探索运行,N7) + # 下一轮(docs/design/next-round/00-总方案.md X1):都只能由服务端写 + "eval", # 评测:作答和算期望的运行(探索运行) + "ledger_issue", # 问数据的答案按已治理口径正式出具(正式运行) + "reconcile", # 口径对账:这次的取数和口径在同一份数据上各算一次(探索运行) + "recall", # 口径召回:新旧两版口径在同一份数据上各算一次(探索运行) ) #: 客户端在 POST /api/runs 里能声明的来源 diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index 6e4a5e4d..51fb6dba 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -5114,6 +5114,10 @@ export const ORIGIN_LABEL: Record = { api: '接口', trial: '蓝图试运行', personal_app: '个人应用刷新', + eval: '评测', + ledger_issue: '按口径正式出具', + reconcile: '口径对账', + recall: '口径召回', } export const ORIGIN_TEXT = { diff --git a/frontend/src/typesApp.ts b/frontend/src/typesApp.ts index 744c8beb..df2b6c2a 100644 --- a/frontend/src/typesApp.ts +++ b/frontend/src/typesApp.ts @@ -703,6 +703,7 @@ export interface NotifyChannelRow { export type OriginKind = | 'chat' | 'canvas' | 'launcher' | 'schedule' | 'app_entry' | 'app_view' | 'app_backfill' | 'hook' | 'api' | 'trial' | 'personal_app' + | 'eval' | 'ledger_issue' | 'reconcile' | 'recall' /** 查看的对象。caliber 是已发布口径卡上的指标(蓝图助手复用口径时看,服务端 look/calibers.py) */ export interface LookObject { From 3565227748549af398daf3beb3ba4ba38173c5bf Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Fri, 16 Oct 2026 09:59:44 +0800 Subject: [PATCH 02/39] =?UTF-8?q?=E5=90=8E=E5=8F=B0=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E8=BF=90=E8=A1=8C=E6=97=B6=E4=B8=8A=E7=A7=BB=E5=88=B0=20core/j?= =?UTF-8?q?ob=5Fruntime.py=EF=BC=88=E6=80=BB=E6=96=B9=E6=A1=88=20X5?= =?UTF-8?q?=EF=BC=89=EF=BC=9A=E8=AF=84=E6=B5=8B=E3=80=81=E5=90=8E=E5=8F=B0?= =?UTF-8?q?=E6=8E=A2=E6=9F=A5=E9=83=BD=E8=A6=81=E7=94=A8=E5=AE=83=EF=BC=8C?= =?UTF-8?q?=E4=BB=8E=20data=20=E5=8C=85=E9=87=8C=E5=AF=BC=E5=85=A5?= =?UTF-8?q?=E4=B8=80=E4=B8=AA=E3=80=8C=E6=95=B0=E6=8D=AE=E7=9B=AE=E5=BD=95?= =?UTF-8?q?=E7=9A=84=E3=80=8D=E8=BF=90=E8=A1=8C=E6=97=B6=E8=AF=B4=E4=B8=8D?= =?UTF-8?q?=E9=80=9A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - data/catalog_job_runtime.py 只留一个老路径:sys.modules 指向 core.job_runtime,同一个模块对象、同一条队,起草、剖析和测试里的 import 一个字不用改 - 同源队里的事做成开放的:运行时不再认具体几种,各模块 register_lane_kind 登记(起草这边 draft / sync_draft / import,剖析 profile),进队只核对登记过;第 2 波的后台探查登记一行就排进同一条队,拼错的 kind 当场报错 - JobRun 多一个 lane_of 钩子(缺省 job.source_id):以后不占源的任务(评测)返回 None 就不进队,现有起草、剖析行为不变 - 新增 tests/test_job_runtime.py --- backend/app/core/job_runtime.py | 439 +++++++++++++++++++++++ backend/app/data/catalog_job_runtime.py | 401 +-------------------- backend/app/data/catalog_jobs.py | 5 + backend/app/data/catalog_profile_jobs.py | 3 + backend/tests/test_job_runtime.py | 79 ++++ 5 files changed, 533 insertions(+), 394 deletions(-) create mode 100644 backend/app/core/job_runtime.py create mode 100644 backend/tests/test_job_runtime.py diff --git a/backend/app/core/job_runtime.py b/backend/app/core/job_runtime.py new file mode 100644 index 00000000..98b641a9 --- /dev/null +++ b/backend/app/core/job_runtime.py @@ -0,0 +1,439 @@ +"""后台任务的共用运行时:数据目录的起草(data/catalog_jobs.py)和剖析(data/catalog_profile_jobs.py)各自建表、各自的步骤, +生命周期是同一套;之后的评测、后台探查也用它(docs/design/next-round/00-总方案.md X5)。 + +**为什么在 core。** 最早只服务数据目录,放在 data/catalog_job_runtime.py;评测(一次评测一个任务、每题一项)和后台探查 +(同源排队)要的是同一样东西,从 data 包里导入一个「数据目录的」运行时说不通。原路径留着再导出(同一个模块对象里的 +名字,lanes 也是同一条队),已有的 import 不用改,正在改起草、剖析的工作也不会因此冲突。 + +**为什么抽出来。** 剖析以前是一次同步长请求,和起草改成后台任务之前是同一类问题:关掉页面结果就没了、不能停、不留痕。 +两边要的是同一样东西——进程内跑、能停、重启标中断、写库不被停止打断、同源排队——各写一份,迟早一边修了另一边没修。 +共用的是生命周期代码,不是表:两种任务的进度完全不同(起草按段、块、模型用量,剖析按检查项、查询),合成一张 +「kind + JSON」的通用任务表要丢掉起草那些强类型列,还要迁移刚稳定的起草数据。 + +- **状态词**(ACTIVE、FINISHED、TASK_STATUSES、TASK_DONE、RETRYABLE、OPEN):任务 queued → running → succeeded / partial + / failed / cancelled / interrupted;逐表 pending → running → ok / partial / failed / skipped / cancelled。两种任务、 + 接口、界面都用这一套词。 +- **SerialWriter**:同一个任务的库写入一条一条来,shield 起来——停止任务时不会打断写到一半的库操作;收尾前 settle 等它们 + 写完。 +- **JobRun / JobManager**:进程内一个 asyncio 任务一件事(照 app/scheduler 的写法):start / stop(grace) / wait / running + / shutdown。JobRun.run 把「进队、等轮到自己、执行、按停止的原因收尾、出队」写一次,各种任务只写执行和收尾的那几步。 +- **SourceLanes**:每个数据源一条先来先到的队。起草、剖析都打业务库或模型、都写同一个源的目录:同源串行(先建的先跑), + 不同源并行。任务阻塞地占:建出来是 queued,轮到了才 running。同步起草、导入目录这类同步写操作不阻塞地试占:占不到 + 说明源上正有别的事,接口回 409;占到了,执行期间新建的任务排在它后面。**队里的事是开放的**:运行时不认具体哪几种, + 每种在自己的模块里 register_lane_kind 登记一次(kind → 叫法),进队时只核对登记过——第 2 波的后台探查(introspect) + 登记一行就排进同一条队,不用改这里;拼错的 kind 当场报错,不会悄悄排成一种谁都不认识的事。 +- **close_job / interrupt_on_startup**:收尾(还没结束的表记成某个状态,任务记成某个状态),和服务启动时的整理:排队、 + 进行中的标 interrupted,没结束的表记 cancelled。**不自动续跑**:复制出来的数据目录(检查脚本、验收副本)启动时不能 + 自己去调模型、查业务库,界面让人点「重试」接着做。 + +进程内登记、进程内的队:服务端是单进程部署,多进程部署时要换成库里的锁。 +""" +from __future__ import annotations + +import asyncio +import contextlib +import logging +from collections.abc import AsyncIterator, Awaitable, Callable, Iterator +from typing import Any + +from sqlalchemy import select, update +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.base import SessionLocal, utcnow + +logger = logging.getLogger(__name__) + +#: 任务还没结束的状态(同一个源同一种任务只能有一个)、结束了的状态 +ACTIVE = ("queued", "running") +FINISHED = ("succeeded", "partial", "failed", "cancelled", "interrupted") +#: 逐表的状态,其中算「已结束」的、重试时要带上的、还没结束的 +TASK_STATUSES = ("pending", "running", "ok", "partial", "failed", "skipped", "cancelled") +TASK_DONE = ("ok", "partial", "failed", "skipped", "cancelled") +RETRYABLE = ("failed", "partial", "cancelled") +OPEN = ("pending", "running") + +#: 停止进行中的任务时最多等它收尾多久(秒);关停服务时另给(shutdown 的 grace) +STOP_GRACE_S = 10.0 +SHUTDOWN_GRACE_S = 3.0 + + +class JobFailed(Exception): + """任务整体出错(数据源没了、快照没了、模型用不了……):message 就是任务的 error。""" + + +# ========================================================================== +# 库写入 +# ========================================================================== + + +class SerialWriter: + """一个任务的库写入:一条一条来,每条开一个会话;shield 起来,停止任务时不会打断写到一半的库操作。""" + + def __init__(self) -> None: + self._writes: set[asyncio.Task[Any]] = set() + self._lock = asyncio.Lock() + + async def __call__(self, fn: Callable[..., Awaitable[Any]], *args: Any) -> Any: + async def go() -> Any: + async with self._lock: + async with SessionLocal() as session: + return await fn(session, *args) + + task = asyncio.ensure_future(go()) + self._writes.add(task) + task.add_done_callback(self._writes.discard) + return await asyncio.shield(task) + + async def settle(self) -> None: + """等手上的库写入都写完(停止、出错收尾之前)。""" + while self._writes: + await asyncio.gather(*list(self._writes), return_exceptions=True) + + +# ========================================================================== +# 每个源一条队 +# ========================================================================== + + +#: 同源队里能排的事:{kind: 叫法}。开放的,各种事在自己的模块里登记(register_lane_kind):数据目录登记了 draft 起草任务、 +#: profile 剖析任务、sync_draft 同步起草、import 导入目录(data/catalog_jobs.py) +_LANE_KINDS: dict[str, str] = {} + + +def register_lane_kind(kind: str, label: str) -> str: + """登记一种同源队里的事(重复登记同一个叫法无害,换叫法报错:同一个 kind 只能有一种说法)。返回 kind。""" + if not kind or not label: + raise ValueError("同源队里的事要有 kind 和叫法") + known = _LANE_KINDS.get(kind) + if known is not None and known != label: + raise ValueError(f"同源队里的「{kind}」已经登记成「{known}」,不能再登记成「{label}」") + _LANE_KINDS[kind] = label + return kind + + +def lane_kinds() -> dict[str, str]: + """登记过的各种事:{kind: 叫法}。""" + return dict(_LANE_KINDS) + + +def lane_label(kind: str) -> str: + """一种事的叫法(「起草」「剖析」……);没登记的照原样给 kind。""" + return _LANE_KINDS.get(kind, kind) + + +class LaneTicket: + """在一个源的队里的一个位置。kind 是占着它的是什么(登记过的一种,见 register_lane_kind);job_id 是任务的(同步写 + 操作没有)。轮到它时 granted 置位。""" + + def __init__(self, lanes: SourceLanes, source_id: str, kind: str, job_id: str | None, + loop: asyncio.AbstractEventLoop) -> None: + self._lanes = lanes + self.source_id = source_id + self.kind = kind + self.job_id = job_id + self.loop = loop + self.granted = asyncio.Event() + self.released = False + + async def wait(self) -> None: + """等轮到自己。""" + await self.granted.wait() + + def release(self) -> None: + """出队(轮到了、没轮到都行,重复调用无害):排在后面的第一个接着开始。""" + self._lanes._release(self) + + +class LaneBusy(Exception): + """不阻塞地试占时这个源的队里已经有人:holder 是排在最前面(正在做)的那一个。""" + + def __init__(self, holder: LaneTicket) -> None: + super().__init__(holder.kind) + self.holder = holder + + +class SourceLanes: + """每个数据源一条先来先到的队。""" + + def __init__(self) -> None: + self._queues: dict[str, list[LaneTicket]] = {} + + def _queue(self, source_id: str) -> list[LaneTicket]: + """这个源的队。别的事件循环留下的(测试每个用例一个循环)去掉:那个循环里没人会再出队了。""" + queue = self._queues.get(source_id) or [] + loop = asyncio.get_running_loop() + alive = [t for t in queue if t.loop is loop and not t.released] + if alive: + self._queues[source_id] = alive + alive[0].granted.set() + else: + self._queues.pop(source_id, None) + return alive + + def enter(self, source_id: str, kind: str, job_id: str | None = None) -> LaneTicket: + """排进队尾,马上返回(不等)。排在最前面的立刻 granted;调用方 await ticket.wait() 等轮到自己。kind 要登记过。""" + if kind not in _LANE_KINDS: + raise ValueError(f"同源队里没有登记「{kind}」这种事,先 register_lane_kind") + queue = self._queue(source_id) + ticket = LaneTicket(self, source_id, kind, job_id, asyncio.get_running_loop()) + queue.append(ticket) + self._queues[source_id] = queue + if len(queue) == 1: + ticket.granted.set() + return ticket + + def try_enter(self, source_id: str, kind: str) -> LaneTicket: + """队空着才占(马上就轮到);有人在队里抛 LaneBusy。""" + queue = self._queue(source_id) + if queue: + raise LaneBusy(queue[0]) + return self.enter(source_id, kind) + + def holders(self, source_id: str) -> list[LaneTicket]: + """队里的各位,排在最前面(正在做)的在前。""" + return list(self._queue(source_id)) + + def ahead(self, source_id: str, job_id: str) -> list[str] | None: + """排在这个任务前面的各位是什么(kind,正在做的在前);这个任务不在队里(刚建、后台还没排进来)返回 None。""" + queue = self._queue(source_id) + for i, ticket in enumerate(queue): + if ticket.job_id == job_id: + return [t.kind for t in queue[:i]] + return None + + def _release(self, ticket: LaneTicket) -> None: + if ticket.released: + return + ticket.released = True + queue = self._queues.get(ticket.source_id) + if not queue or ticket not in queue: + return + head = queue[0] is ticket + queue.remove(ticket) + if not queue: + self._queues.pop(ticket.source_id, None) + elif head: + queue[0].granted.set() + + @contextlib.asynccontextmanager + async def hold(self, source_id: str, kind: str, job_id: str | None = None) -> AsyncIterator[LaneTicket]: + """阻塞地占:排队,轮到了再进去,出来时出队。""" + ticket = self.enter(source_id, kind, job_id) + try: + await ticket.wait() + yield ticket + finally: + ticket.release() + + @contextlib.contextmanager + def try_hold(self, source_id: str, kind: str) -> Iterator[LaneTicket]: + """不阻塞地试占(同步起草、导入目录):占不到抛 LaneBusy;占到了,执行期间新建的任务排在后面。""" + ticket = self.try_enter(source_id, kind) + try: + yield ticket + finally: + ticket.release() + + +lanes = SourceLanes() + + +# ========================================================================== +# 收尾、启动时的整理 +# ========================================================================== + + +async def close_job(session: AsyncSession, job_model: Any, task_model: Any, job_id: str, job_status: str, + job_error: str | None, task_status: str, task_error: str | None) -> None: + """收尾:还没结束的表记 task_status(原因 task_error),任务(还没结束的话)记 job_status。提交。""" + now = utcnow() + await session.execute( + update(task_model).where(task_model.job_id == job_id, task_model.status.in_(OPEN)) + .values(status=task_status, error=task_error, finished_at=now).execution_options(synchronize_session=False)) + await session.execute( + update(job_model).where(job_model.id == job_id, job_model.status.in_(ACTIVE)) + .values(status=job_status, error=job_error, finished_at=now).execution_options(synchronize_session=False)) + await session.commit() + + +async def cancel_queued(session: AsyncSession, job_model: Any, task_model: Any, job_id: str) -> bool: + """停止一个排队中的任务:任务和没结束的表直接记 cancelled。已经被后台认领(开始了)返回 False,交给调用方按进行中 + 的停。""" + now = utcnow() + claimed = await session.execute( + update(job_model).where(job_model.id == job_id, job_model.status == "queued") + .values(status="cancelled", finished_at=now).execution_options(synchronize_session=False)) + if claimed.rowcount != 1: + await session.rollback() + return False + await session.execute( + update(task_model).where(task_model.job_id == job_id, task_model.status.in_(OPEN)) + .values(status="cancelled", finished_at=now).execution_options(synchronize_session=False)) + await session.commit() + return True + + +async def claim(session: AsyncSession, job_model: Any, job_id: str) -> bool: + """认领:排队中 → 进行中。开始之前就被停止了返回 False。""" + result = await session.execute( + update(job_model).where(job_model.id == job_id, job_model.status == "queued") + .values(status="running", started_at=utcnow()).execution_options(synchronize_session=False)) + await session.commit() + return result.rowcount == 1 + + +async def interrupt_on_startup(job_model: Any, close: Callable[[AsyncSession, str], Awaitable[None]], *, + running: Callable[[str], bool]) -> int: + """服务启动时:排队、进行中、而且不在这个进程里跑的任务逐个 close(各自记成 interrupted、没结束的表记 cancelled)。 + 不自动续跑。返回整理了几个。""" + async with SessionLocal() as session: + ids = [job_id for job_id in (await session.execute( + select(job_model.id).where(job_model.status.in_(ACTIVE)))).scalars() if not running(job_id)] + for job_id in ids: + await close(session, job_id) + return len(ids) + + +# ========================================================================== +# 在后台执行 +# ========================================================================== + + +class JobRun: + """一个任务在后台的一次执行。子类给 kind(进队时的身份)、model(任务表,进队前读源和状态),写 _run(执行)和 + _stopped / _failed / _crashed(停止、整体出错、意外出错时的收尾)。""" + + kind = "" + model: Any = None + + def __init__(self, job_id: str) -> None: + self.job_id = job_id + self.task: asyncio.Task[None] | None = None + #: 为什么停:user 使用者点了停止,shutdown 服务关停(标中断) + self.stop_reason: str | None = None + self.writer = SerialWriter() + self.ticket: LaneTicket | None = None + + async def _write(self, fn: Callable[..., Awaitable[Any]], *args: Any) -> Any: + """一条库写入(SerialWriter:一条一条来,停止时不被打断)。""" + return await self.writer(fn, *args) + + async def _settle(self) -> None: + await self.writer.settle() + + async def run(self) -> None: + try: + try: + await self._queue() + await self._run() + except asyncio.CancelledError: + current = asyncio.current_task() + if current is not None: + current.uncancel() + await self._stopped() + except JobFailed as e: + await self._failed(str(e)) + except Exception as e: # noqa: BLE001 - 任务整体出错:照实记下,不留在「进行中」 + await self._crashed(e) + finally: + # 收尾写完才出队:排在后面的任务开始时,这个任务已经记成结束了 + if self.ticket is not None: + self.ticket.release() + + def lane_of(self, job: Any) -> str | None: + """这个任务排在哪条队里:缺省是任务的数据源(job.source_id)。不占源的任务(以后的评测:开始时拍目录快照,不挡 + 起草、剖析)覆盖成返回 None,不进队、直接执行。""" + return job.source_id + + async def _queue(self) -> None: + """进这个源的队,等轮到自己。任务不在了、已经不是排队中(开始之前就被停止了)的不进队:_run 认领不到,直接结束。""" + async with SessionLocal() as session: + job = await session.get(self.model, self.job_id) + source_id = self.lane_of(job) if job is not None and job.status == "queued" else None + if source_id is None: + return + self.ticket = lanes.enter(source_id, self.kind, self.job_id) + await self.ticket.wait() + + async def _run(self) -> None: + raise NotImplementedError + + async def _stopped(self) -> None: + raise NotImplementedError + + async def _failed(self, reason: str) -> None: + raise NotImplementedError + + async def _crashed(self, e: Exception) -> None: + raise NotImplementedError + + +class JobManager: + """进程内的一种后台任务:一个 asyncio 任务一件事,停的时候给宽限。factory 按任务编号造一次执行(JobRun);label 是 + 日志里的叫法(「数据目录起草任务」),name 是 asyncio 任务名的前缀。""" + + def __init__(self, factory: Callable[[str], JobRun], *, label: str, name: str) -> None: + self._runs: dict[str, JobRun] = {} + self._factory = factory + self.label = label + self.name = name + + def _alive(self, job_id: str) -> JobRun | None: + run = self._runs.get(job_id) + if run is None or run.task is None or run.task.done(): + return None + try: + if run.task.get_loop() is not asyncio.get_running_loop(): + self._runs.pop(job_id, None) # 别的事件循环留下的(测试每个用例一个循环) + return None + except RuntimeError: + return None + return run + + def start(self, job_id: str) -> None: + """开始在后台执行这个(排队中的)任务:先进这个源的队,轮到了才开始。""" + run = self._factory(job_id) + run.task = asyncio.get_running_loop().create_task(run.run(), name=f"{self.name}-{job_id}") + self._runs[job_id] = run + + def forget(_: asyncio.Task[None]) -> None: + if self._runs.get(job_id) is run: + self._runs.pop(job_id, None) + + run.task.add_done_callback(forget) + + def running(self, job_id: str) -> bool: + return self._alive(job_id) is not None + + async def stop(self, job_id: str, reason: str, *, grace: float = STOP_GRACE_S) -> bool: + """停掉这个进程里在跑(或在排队等)的任务,等它收尾(最多 grace 秒)。不在这个进程里返回 False。""" + run = self._alive(job_id) + if run is None: + return False + if run.stop_reason is None: + run.stop_reason = reason + run.task.cancel() # type: ignore[union-attr] + await asyncio.wait([run.task], timeout=grace) # type: ignore[list-item] + return True + + async def wait(self, job_id: str, timeout: float | None = None) -> None: + """等这个任务做完(测试、调试用)。不在跑直接返回。""" + run = self._alive(job_id) + if run is not None: + await asyncio.wait([run.task], timeout=timeout) # type: ignore[list-item] + + async def shutdown(self, *, grace: float = SHUTDOWN_GRACE_S) -> None: + """关停:停掉进行中、排队等着的任务(标中断),最多等 grace 秒收尾。""" + runs = [run for job_id in list(self._runs) if (run := self._alive(job_id)) is not None] + for run in runs: + if run.stop_reason is None: + run.stop_reason = "shutdown" + run.task.cancel() # type: ignore[union-attr] + if runs: + await asyncio.wait([run.task for run in runs], timeout=grace) # type: ignore[misc] + logger.info("已停止 %d 个%s", len(runs), self.label) + + +__all__ = ["ACTIVE", "FINISHED", "JobFailed", "JobManager", "JobRun", "LaneBusy", "LaneTicket", "OPEN", "RETRYABLE", + "SHUTDOWN_GRACE_S", "STOP_GRACE_S", "SerialWriter", "SourceLanes", "TASK_DONE", "TASK_STATUSES", "cancel_queued", + "claim", "close_job", "interrupt_on_startup", "lane_kinds", "lane_label", "lanes", "register_lane_kind"] diff --git a/backend/app/data/catalog_job_runtime.py b/backend/app/data/catalog_job_runtime.py index 970159a4..664c5c51 100644 --- a/backend/app/data/catalog_job_runtime.py +++ b/backend/app/data/catalog_job_runtime.py @@ -1,400 +1,13 @@ -"""数据目录后台任务的共用运行时:起草(data/catalog_jobs.py)和剖析(data/catalog_profile_jobs.py)各自建表、各自的步骤, -生命周期是同一套。 +"""后台任务运行时的老路径:已上移到 app/core/job_runtime.py(docs/design/next-round/00-总方案.md X5)。 -**为什么抽出来。** 剖析以前是一次同步长请求,和起草改成后台任务之前是同一类问题:关掉页面结果就没了、不能停、不留痕。 -两边要的是同一样东西——进程内跑、能停、重启标中断、写库不被停止打断、同源排队——各写一份,迟早一边修了另一边没修。 -共用的是生命周期代码,不是表:两种任务的进度完全不同(起草按段、块、模型用量,剖析按检查项、查询),合成一张 -「kind + JSON」的通用任务表要丢掉起草那些强类型列,还要迁移刚稳定的起草数据。 - -- **状态词**(ACTIVE、FINISHED、TASK_STATUSES、TASK_DONE、RETRYABLE、OPEN):任务 queued → running → succeeded / partial - / failed / cancelled / interrupted;逐表 pending → running → ok / partial / failed / skipped / cancelled。两种任务、 - 接口、界面都用这一套词。 -- **SerialWriter**:同一个任务的库写入一条一条来,shield 起来——停止任务时不会打断写到一半的库操作;收尾前 settle 等它们 - 写完。 -- **JobRun / JobManager**:进程内一个 asyncio 任务一件事(照 app/scheduler 的写法):start / stop(grace) / wait / running - / shutdown。JobRun.run 把「进队、等轮到自己、执行、按停止的原因收尾、出队」写一次,各种任务只写执行和收尾的那几步。 -- **SourceLanes**:每个数据源一条先来先到的队。起草、剖析都打业务库或模型、都写同一个源的目录:同源串行(先建的先跑), - 不同源并行。任务阻塞地占:建出来是 queued,轮到了才 running。同步起草、导入目录这类同步写操作不阻塞地试占:占不到 - 说明源上正有别的事,接口回 409;占到了,执行期间新建的任务排在它后面。 -- **close_job / interrupt_on_startup**:收尾(还没结束的表记成某个状态,任务记成某个状态),和服务启动时的整理:排队、 - 进行中的标 interrupted,没结束的表记 cancelled。**不自动续跑**:复制出来的数据目录(检查脚本、验收副本)启动时不能 - 自己去调模型、查业务库,界面让人点「重试」接着做。 - -进程内登记、进程内的队:服务端是单进程部署,多进程部署时要换成库里的锁。 +这里把那个模块原样放在老名字下(sys.modules 指向同一个模块对象):起草、剖析和别处的 `from app.data import +catalog_job_runtime as runtime`、`from app.data.catalog_job_runtime import …` 拿到的是同一批名字、同一条队(lanes), +测试里改它的属性也就是改那个模块。新代码从 app.core.job_runtime 导入。 """ from __future__ import annotations -import asyncio -import contextlib -import logging -from collections.abc import AsyncIterator, Awaitable, Callable, Iterator -from typing import Any +import sys -from sqlalchemy import select, update -from sqlalchemy.ext.asyncio import AsyncSession +from app.core import job_runtime -from app.db.base import SessionLocal, utcnow - -logger = logging.getLogger(__name__) - -#: 任务还没结束的状态(同一个源同一种任务只能有一个)、结束了的状态 -ACTIVE = ("queued", "running") -FINISHED = ("succeeded", "partial", "failed", "cancelled", "interrupted") -#: 逐表的状态,其中算「已结束」的、重试时要带上的、还没结束的 -TASK_STATUSES = ("pending", "running", "ok", "partial", "failed", "skipped", "cancelled") -TASK_DONE = ("ok", "partial", "failed", "skipped", "cancelled") -RETRYABLE = ("failed", "partial", "cancelled") -OPEN = ("pending", "running") - -#: 停止进行中的任务时最多等它收尾多久(秒);关停服务时另给(shutdown 的 grace) -STOP_GRACE_S = 10.0 -SHUTDOWN_GRACE_S = 3.0 - - -class JobFailed(Exception): - """任务整体出错(数据源没了、快照没了、模型用不了……):message 就是任务的 error。""" - - -# ========================================================================== -# 库写入 -# ========================================================================== - - -class SerialWriter: - """一个任务的库写入:一条一条来,每条开一个会话;shield 起来,停止任务时不会打断写到一半的库操作。""" - - def __init__(self) -> None: - self._writes: set[asyncio.Task[Any]] = set() - self._lock = asyncio.Lock() - - async def __call__(self, fn: Callable[..., Awaitable[Any]], *args: Any) -> Any: - async def go() -> Any: - async with self._lock: - async with SessionLocal() as session: - return await fn(session, *args) - - task = asyncio.ensure_future(go()) - self._writes.add(task) - task.add_done_callback(self._writes.discard) - return await asyncio.shield(task) - - async def settle(self) -> None: - """等手上的库写入都写完(停止、出错收尾之前)。""" - while self._writes: - await asyncio.gather(*list(self._writes), return_exceptions=True) - - -# ========================================================================== -# 每个源一条队 -# ========================================================================== - - -class LaneTicket: - """在一个源的队里的一个位置。kind 是占着它的是什么:draft 起草任务、profile 剖析任务、sync_draft 同步起草、 - import 导入目录;job_id 是任务的(同步写操作没有)。轮到它时 granted 置位。""" - - def __init__(self, lanes: SourceLanes, source_id: str, kind: str, job_id: str | None, - loop: asyncio.AbstractEventLoop) -> None: - self._lanes = lanes - self.source_id = source_id - self.kind = kind - self.job_id = job_id - self.loop = loop - self.granted = asyncio.Event() - self.released = False - - async def wait(self) -> None: - """等轮到自己。""" - await self.granted.wait() - - def release(self) -> None: - """出队(轮到了、没轮到都行,重复调用无害):排在后面的第一个接着开始。""" - self._lanes._release(self) - - -class LaneBusy(Exception): - """不阻塞地试占时这个源的队里已经有人:holder 是排在最前面(正在做)的那一个。""" - - def __init__(self, holder: LaneTicket) -> None: - super().__init__(holder.kind) - self.holder = holder - - -class SourceLanes: - """每个数据源一条先来先到的队。""" - - def __init__(self) -> None: - self._queues: dict[str, list[LaneTicket]] = {} - - def _queue(self, source_id: str) -> list[LaneTicket]: - """这个源的队。别的事件循环留下的(测试每个用例一个循环)去掉:那个循环里没人会再出队了。""" - queue = self._queues.get(source_id) or [] - loop = asyncio.get_running_loop() - alive = [t for t in queue if t.loop is loop and not t.released] - if alive: - self._queues[source_id] = alive - alive[0].granted.set() - else: - self._queues.pop(source_id, None) - return alive - - def enter(self, source_id: str, kind: str, job_id: str | None = None) -> LaneTicket: - """排进队尾,马上返回(不等)。排在最前面的立刻 granted;调用方 await ticket.wait() 等轮到自己。""" - queue = self._queue(source_id) - ticket = LaneTicket(self, source_id, kind, job_id, asyncio.get_running_loop()) - queue.append(ticket) - self._queues[source_id] = queue - if len(queue) == 1: - ticket.granted.set() - return ticket - - def try_enter(self, source_id: str, kind: str) -> LaneTicket: - """队空着才占(马上就轮到);有人在队里抛 LaneBusy。""" - queue = self._queue(source_id) - if queue: - raise LaneBusy(queue[0]) - return self.enter(source_id, kind) - - def holders(self, source_id: str) -> list[LaneTicket]: - """队里的各位,排在最前面(正在做)的在前。""" - return list(self._queue(source_id)) - - def ahead(self, source_id: str, job_id: str) -> list[str] | None: - """排在这个任务前面的各位是什么(kind,正在做的在前);这个任务不在队里(刚建、后台还没排进来)返回 None。""" - queue = self._queue(source_id) - for i, ticket in enumerate(queue): - if ticket.job_id == job_id: - return [t.kind for t in queue[:i]] - return None - - def _release(self, ticket: LaneTicket) -> None: - if ticket.released: - return - ticket.released = True - queue = self._queues.get(ticket.source_id) - if not queue or ticket not in queue: - return - head = queue[0] is ticket - queue.remove(ticket) - if not queue: - self._queues.pop(ticket.source_id, None) - elif head: - queue[0].granted.set() - - @contextlib.asynccontextmanager - async def hold(self, source_id: str, kind: str, job_id: str | None = None) -> AsyncIterator[LaneTicket]: - """阻塞地占:排队,轮到了再进去,出来时出队。""" - ticket = self.enter(source_id, kind, job_id) - try: - await ticket.wait() - yield ticket - finally: - ticket.release() - - @contextlib.contextmanager - def try_hold(self, source_id: str, kind: str) -> Iterator[LaneTicket]: - """不阻塞地试占(同步起草、导入目录):占不到抛 LaneBusy;占到了,执行期间新建的任务排在后面。""" - ticket = self.try_enter(source_id, kind) - try: - yield ticket - finally: - ticket.release() - - -lanes = SourceLanes() - - -# ========================================================================== -# 收尾、启动时的整理 -# ========================================================================== - - -async def close_job(session: AsyncSession, job_model: Any, task_model: Any, job_id: str, job_status: str, - job_error: str | None, task_status: str, task_error: str | None) -> None: - """收尾:还没结束的表记 task_status(原因 task_error),任务(还没结束的话)记 job_status。提交。""" - now = utcnow() - await session.execute( - update(task_model).where(task_model.job_id == job_id, task_model.status.in_(OPEN)) - .values(status=task_status, error=task_error, finished_at=now).execution_options(synchronize_session=False)) - await session.execute( - update(job_model).where(job_model.id == job_id, job_model.status.in_(ACTIVE)) - .values(status=job_status, error=job_error, finished_at=now).execution_options(synchronize_session=False)) - await session.commit() - - -async def cancel_queued(session: AsyncSession, job_model: Any, task_model: Any, job_id: str) -> bool: - """停止一个排队中的任务:任务和没结束的表直接记 cancelled。已经被后台认领(开始了)返回 False,交给调用方按进行中 - 的停。""" - now = utcnow() - claimed = await session.execute( - update(job_model).where(job_model.id == job_id, job_model.status == "queued") - .values(status="cancelled", finished_at=now).execution_options(synchronize_session=False)) - if claimed.rowcount != 1: - await session.rollback() - return False - await session.execute( - update(task_model).where(task_model.job_id == job_id, task_model.status.in_(OPEN)) - .values(status="cancelled", finished_at=now).execution_options(synchronize_session=False)) - await session.commit() - return True - - -async def claim(session: AsyncSession, job_model: Any, job_id: str) -> bool: - """认领:排队中 → 进行中。开始之前就被停止了返回 False。""" - result = await session.execute( - update(job_model).where(job_model.id == job_id, job_model.status == "queued") - .values(status="running", started_at=utcnow()).execution_options(synchronize_session=False)) - await session.commit() - return result.rowcount == 1 - - -async def interrupt_on_startup(job_model: Any, close: Callable[[AsyncSession, str], Awaitable[None]], *, - running: Callable[[str], bool]) -> int: - """服务启动时:排队、进行中、而且不在这个进程里跑的任务逐个 close(各自记成 interrupted、没结束的表记 cancelled)。 - 不自动续跑。返回整理了几个。""" - async with SessionLocal() as session: - ids = [job_id for job_id in (await session.execute( - select(job_model.id).where(job_model.status.in_(ACTIVE)))).scalars() if not running(job_id)] - for job_id in ids: - await close(session, job_id) - return len(ids) - - -# ========================================================================== -# 在后台执行 -# ========================================================================== - - -class JobRun: - """一个任务在后台的一次执行。子类给 kind(进队时的身份)、model(任务表,进队前读源和状态),写 _run(执行)和 - _stopped / _failed / _crashed(停止、整体出错、意外出错时的收尾)。""" - - kind = "" - model: Any = None - - def __init__(self, job_id: str) -> None: - self.job_id = job_id - self.task: asyncio.Task[None] | None = None - #: 为什么停:user 使用者点了停止,shutdown 服务关停(标中断) - self.stop_reason: str | None = None - self.writer = SerialWriter() - self.ticket: LaneTicket | None = None - - async def _write(self, fn: Callable[..., Awaitable[Any]], *args: Any) -> Any: - """一条库写入(SerialWriter:一条一条来,停止时不被打断)。""" - return await self.writer(fn, *args) - - async def _settle(self) -> None: - await self.writer.settle() - - async def run(self) -> None: - try: - try: - await self._queue() - await self._run() - except asyncio.CancelledError: - current = asyncio.current_task() - if current is not None: - current.uncancel() - await self._stopped() - except JobFailed as e: - await self._failed(str(e)) - except Exception as e: # noqa: BLE001 - 任务整体出错:照实记下,不留在「进行中」 - await self._crashed(e) - finally: - # 收尾写完才出队:排在后面的任务开始时,这个任务已经记成结束了 - if self.ticket is not None: - self.ticket.release() - - async def _queue(self) -> None: - """进这个源的队,等轮到自己。任务不在了、已经不是排队中(开始之前就被停止了)的不进队:_run 认领不到,直接结束。""" - async with SessionLocal() as session: - job = await session.get(self.model, self.job_id) - source_id = job.source_id if job is not None and job.status == "queued" else None - if source_id is None: - return - self.ticket = lanes.enter(source_id, self.kind, self.job_id) - await self.ticket.wait() - - async def _run(self) -> None: - raise NotImplementedError - - async def _stopped(self) -> None: - raise NotImplementedError - - async def _failed(self, reason: str) -> None: - raise NotImplementedError - - async def _crashed(self, e: Exception) -> None: - raise NotImplementedError - - -class JobManager: - """进程内的一种后台任务:一个 asyncio 任务一件事,停的时候给宽限。factory 按任务编号造一次执行(JobRun);label 是 - 日志里的叫法(「数据目录起草任务」),name 是 asyncio 任务名的前缀。""" - - def __init__(self, factory: Callable[[str], JobRun], *, label: str, name: str) -> None: - self._runs: dict[str, JobRun] = {} - self._factory = factory - self.label = label - self.name = name - - def _alive(self, job_id: str) -> JobRun | None: - run = self._runs.get(job_id) - if run is None or run.task is None or run.task.done(): - return None - try: - if run.task.get_loop() is not asyncio.get_running_loop(): - self._runs.pop(job_id, None) # 别的事件循环留下的(测试每个用例一个循环) - return None - except RuntimeError: - return None - return run - - def start(self, job_id: str) -> None: - """开始在后台执行这个(排队中的)任务:先进这个源的队,轮到了才开始。""" - run = self._factory(job_id) - run.task = asyncio.get_running_loop().create_task(run.run(), name=f"{self.name}-{job_id}") - self._runs[job_id] = run - - def forget(_: asyncio.Task[None]) -> None: - if self._runs.get(job_id) is run: - self._runs.pop(job_id, None) - - run.task.add_done_callback(forget) - - def running(self, job_id: str) -> bool: - return self._alive(job_id) is not None - - async def stop(self, job_id: str, reason: str, *, grace: float = STOP_GRACE_S) -> bool: - """停掉这个进程里在跑(或在排队等)的任务,等它收尾(最多 grace 秒)。不在这个进程里返回 False。""" - run = self._alive(job_id) - if run is None: - return False - if run.stop_reason is None: - run.stop_reason = reason - run.task.cancel() # type: ignore[union-attr] - await asyncio.wait([run.task], timeout=grace) # type: ignore[list-item] - return True - - async def wait(self, job_id: str, timeout: float | None = None) -> None: - """等这个任务做完(测试、调试用)。不在跑直接返回。""" - run = self._alive(job_id) - if run is not None: - await asyncio.wait([run.task], timeout=timeout) # type: ignore[list-item] - - async def shutdown(self, *, grace: float = SHUTDOWN_GRACE_S) -> None: - """关停:停掉进行中、排队等着的任务(标中断),最多等 grace 秒收尾。""" - runs = [run for job_id in list(self._runs) if (run := self._alive(job_id)) is not None] - for run in runs: - if run.stop_reason is None: - run.stop_reason = "shutdown" - run.task.cancel() # type: ignore[union-attr] - if runs: - await asyncio.wait([run.task for run in runs], timeout=grace) # type: ignore[misc] - logger.info("已停止 %d 个%s", len(runs), self.label) - - -__all__ = ["ACTIVE", "FINISHED", "JobFailed", "JobManager", "JobRun", "LaneBusy", "LaneTicket", "OPEN", "RETRYABLE", - "SHUTDOWN_GRACE_S", "STOP_GRACE_S", "SerialWriter", "SourceLanes", "TASK_DONE", "TASK_STATUSES", "cancel_queued", - "claim", "close_job", "interrupt_on_startup", "lanes"] +sys.modules[__name__] = job_runtime diff --git a/backend/app/data/catalog_jobs.py b/backend/app/data/catalog_jobs.py index 19c009d9..557978c4 100644 --- a/backend/app/data/catalog_jobs.py +++ b/backend/app/data/catalog_jobs.py @@ -55,6 +55,11 @@ logger = logging.getLogger(__name__) _OPEN = runtime.OPEN +#: 起草这边占同源队的几种事(叫法登记在运行时里,见 core/job_runtime.register_lane_kind):后台起草任务、同步起草、导入目录 +runtime.register_lane_kind("draft", "起草") +runtime.register_lane_kind("sync_draft", "同步起草") +runtime.register_lane_kind("import", "导入目录") + #: 一个任务最多几张表;起草要求、本次说明最多几个字 MAX_TABLES = 500 GUIDANCE_MAX = 2000 diff --git a/backend/app/data/catalog_profile_jobs.py b/backend/app/data/catalog_profile_jobs.py index 726e0417..74f9c058 100644 --- a/backend/app/data/catalog_profile_jobs.py +++ b/backend/app/data/catalog_profile_jobs.py @@ -58,6 +58,9 @@ from app.db.models import CatalogDraftJob, CatalogProfileJob, CatalogProfileTask logger = logging.getLogger(__name__) +#: 剖析任务占同源队(叫法登记在运行时里,见 core/job_runtime.register_lane_kind) +runtime.register_lane_kind("profile", "剖析") + #: 一个任务最多几张表 MAX_TABLES = 500 #: 每个源保留几个结束了的剖析任务(某张表最近一次剖析所在的任务另外保留) diff --git a/backend/tests/test_job_runtime.py b/backend/tests/test_job_runtime.py new file mode 100644 index 00000000..ffda8c90 --- /dev/null +++ b/backend/tests/test_job_runtime.py @@ -0,0 +1,79 @@ +"""后台任务运行时上移到 core/job_runtime.py(docs/design/next-round/00-总方案.md X5)。 + +守三件事:老路径 data/catalog_job_runtime 拿到的是同一个模块(同一条队、同一批名字,起草、剖析的 import 不用改); +同源队里的事是开放的——新的一种登记一行就和起草、剖析排进同一条队、同源串行;没登记的 kind 当场报错,不会悄悄排成 +一种谁都不认识的事。 +""" +from __future__ import annotations + +import asyncio + +import pytest + +from app.core import job_runtime + + +def test_the_old_path_is_the_same_module(): + from app.data import catalog_job_runtime as old + from app.data.catalog_job_runtime import lanes + + assert old is job_runtime + assert lanes is job_runtime.lanes + # 起草、剖析照旧从老路径导入,登记的几种事都在 + import app.data.catalog_jobs # noqa: F401 + import app.data.catalog_profile_jobs # noqa: F401 + + assert {"draft", "profile", "sync_draft", "import"} <= set(job_runtime.lane_kinds()) + assert job_runtime.lane_label("profile") == "剖析" + + +def test_an_unregistered_kind_is_refused(): + lanes = job_runtime.SourceLanes() + + async def go() -> None: + with pytest.raises(ValueError, match="没有登记"): + lanes.enter("src", "nonsense-kind") + + asyncio.run(go()) + + +def test_a_registered_label_cannot_change(): + job_runtime.register_lane_kind("test-open-kind", "测试用") + job_runtime.register_lane_kind("test-open-kind", "测试用") # 同一个叫法再登记无害 + with pytest.raises(ValueError, match="已经登记"): + job_runtime.register_lane_kind("test-open-kind", "别的叫法") + + +def test_a_new_kind_queues_behind_the_existing_ones_on_the_same_source(): + """新登记的一种(第 2 波的后台探查就是这样加进来的)和起草排进同一条队:同源串行,不同源并行。""" + import app.data.catalog_jobs # noqa: F401 - 登记 draft + + job_runtime.register_lane_kind("test-introspect", "测试探查") + lanes = job_runtime.SourceLanes() + + async def go() -> None: + first = lanes.enter("src-a", "draft", "job-1") + second = lanes.enter("src-a", "test-introspect", "job-2") + other = lanes.enter("src-b", "test-introspect", "job-3") + assert first.granted.is_set() and not second.granted.is_set() + assert other.granted.is_set(), "不同源并行" + assert lanes.ahead("src-a", "job-2") == ["draft"] + first.release() + await asyncio.wait_for(second.wait(), timeout=1) + second.release() + other.release() + + asyncio.run(go()) + + +def test_a_job_without_a_lane_skips_the_queue(): + """lane_of 返回 None 的任务(以后不占源的评测)不进队、直接执行。""" + + class _NoLane(job_runtime.JobRun): + kind = "draft" + + def lane_of(self, job): # noqa: ANN001 + return None + + run = _NoLane("whatever") + assert run.lane_of(object()) is None From 0ce272e94b56e428b4dedb2a98b0261b61ef53cc Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Fri, 16 Oct 2026 10:02:53 +0800 Subject: [PATCH 03/39] =?UTF-8?q?=E6=AF=8F=E6=97=A5=E8=B4=A6=E6=B3=9B?= =?UTF-8?q?=E5=8C=96=E6=88=90=20Ledger(key)=EF=BC=88=E7=BB=86=E6=A1=88=200?= =?UTF-8?q?1=20=E5=81=9A=E6=B3=95=2017=EF=BC=89=EF=BC=9A=E6=A0=B8=E6=9F=A5?= =?UTF-8?q?=E3=80=81=E8=A3=81=E5=88=A4=E4=BB=A5=E5=89=8D=E5=90=84=E5=86=99?= =?UTF-8?q?=E4=B8=80=E4=BB=BD=E4=B8=80=E6=A8=A1=E4=B8=80=E6=A0=B7=E7=9A=84?= =?UTF-8?q?=E9=A2=84=E7=95=99=E3=80=81=E6=89=BE=E8=A1=A5=E3=80=81=E6=8C=89?= =?UTF-8?q?=E6=97=A5=E6=BB=9A=E5=8A=A8=EF=BC=8C=E8=AF=84=E6=B5=8B=E5=86=8D?= =?UTF-8?q?=E6=8A=84=E5=B0=B1=E6=98=AF=E7=AC=AC=E4=B8=89=E4=BB=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - engine/spend.py:Ledger(key) 一套代码,三个实例各记各的键——verify_spend(核查)、judge_spend(裁判)、eval_spend(评测,预留,设置和接口第 3 波加);锁按(事件循环, 键)各一把,一本触顶不挤占另外两本 - 核查的模块函数(SPEND_KEY、daily_spend、charge、settle)照旧,委托给 verify_spend - engine/judge.py:_charge / _settle 和它自己那份 _today、_spend_of、模块级锁去掉,改用 spend.judge_spend;SPEND_KEY、daily_spend 照旧,裁判的预留、找补、上限行为不变 - 三本账按同一个本地日期滚动(spend._today):test_evidence_judge 里模拟跨天改成换 spend._today(以前换的是裁判自己那份时钟,已经不在了) - 新增 tests/test_spend_ledger.py(三本账三个键、预留找补与上限、跨天、互不挤占、同一个时钟) --- backend/app/engine/judge.py | 64 ++--------- backend/app/engine/spend.py | 160 +++++++++++++++++---------- backend/tests/test_evidence_judge.py | 6 +- backend/tests/test_spend_ledger.py | 78 +++++++++++++ 4 files changed, 190 insertions(+), 118 deletions(-) create mode 100644 backend/tests/test_spend_ledger.py diff --git a/backend/app/engine/judge.py b/backend/app/engine/judge.py index 53c6dff4..c9f6bc90 100644 --- a/backend/app/engine/judge.py +++ b/backend/app/engine/judge.py @@ -37,7 +37,6 @@ import re import time from collections.abc import Callable, Iterable from dataclasses import asdict, dataclass -from datetime import datetime from typing import Any from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, SystemMessage @@ -68,6 +67,7 @@ from app.engine.evidence import ( ) from app.engine.expressions import CellError, column_kind, locate_cell from app.engine.merge_query import MERGE_SOURCE +from app.engine import spend from app.engine.nodes.llm import _split_structured, _usage_of from app.engine.state import message_text from app.providers import catalog as pricing @@ -855,8 +855,8 @@ JUDGE_DEFAULTS: dict[str, Any] = { "click_max_cost_usd": 0.01, # 探索运行里每点一次「请模型判断」的金额上限 "daily_max_usd": 2.0, # 全局每日裁判金额上限(按本地日期) } -#: 每日计数存在设置表的这一行:{date: 本地日期, usd, calls, unpriced_calls}。不从设置接口进出 -SPEND_KEY = "judge_spend" +#: 每日计数存在设置表的这一行:{date: 本地日期, usd, calls, unpriced_calls}。不从设置接口进出(账本见 engine/spend.py) +SPEND_KEY = spend.judge_spend.key _COUNTS = frozenset({"report_max_claims"}) _SECONDS = frozenset({"report_timeout_s"}) @@ -1024,61 +1024,13 @@ async def source_masks(catalog: dict[str, Any], *, loader: Callable[[str], Any] # 每日计数 # -------------------------------------------------------------------------- -_spend_lock = asyncio.Lock() - - -def _today() -> str: - """本地日期:每日上限按它滚动。""" - return datetime.now().astimezone().date().isoformat() - - -def _spend_of(row: Any) -> dict[str, Any]: - value = row.value if row is not None and isinstance(row.value, dict) else {} - today = _today() - if value.get("date") != today: - return {"date": today, "usd": 0.0, "calls": 0, "unpriced_calls": 0} - return {"date": today, "usd": float(value.get("usd") or 0.0), "calls": int(value.get("calls") or 0), - "unpriced_calls": int(value.get("unpriced_calls") or 0)} +#: 裁判这本每日账(engine/spend.py 的 Ledger:预留、找补、按本地日期滚动和核查、评测是同一套代码,各记各的键) +_LEDGER = spend.judge_spend async def daily_spend() -> dict[str, Any]: """今天(本地日期)裁判已经花了多少:{date, usd, calls, unpriced_calls}。估不出金额的调用只计次数。""" - async with SessionLocal() as session: - return _spend_of(await session.get(Setting, SPEND_KEY)) - - -async def _charge(amount: float, *, limit: float | None, priced: bool) -> str | None: - """记一次调用、先按估算预留金额。加上这笔会超过每日上限就不记,返回 None;记下了返回记在哪天。""" - async with _spend_lock: - async with SessionLocal() as session: - row = await session.get(Setting, SPEND_KEY) - spend = _spend_of(row) - if limit is not None and spend["usd"] + amount > limit + 1e-12: - return None - spend["usd"] = round(spend["usd"] + amount, 8) - spend["calls"] += 1 - spend["unpriced_calls"] += 0 if priced else 1 - if row is None: - session.add(Setting(key=SPEND_KEY, value=spend)) - else: - row.value = spend - await session.commit() - return spend["date"] - - -async def _settle(date: str, delta: float) -> None: - """调用结束后按实际用量找补预留的差额。跨了天就不找补:预留记在前一天,今天从零算。""" - if not delta: - return - async with _spend_lock: - async with SessionLocal() as session: - row = await session.get(Setting, SPEND_KEY) - spend = _spend_of(row) - if row is None or spend["date"] != date: - return - spend["usd"] = max(0.0, round(spend["usd"] + delta, 8)) - row.value = spend - await session.commit() + return await _LEDGER.daily() # -------------------------------------------------------------------------- @@ -1318,7 +1270,7 @@ async def run_request(request: JudgeRequest, *, spec: ModelSpec, budget: Budget, pending = pending[:len(batch)] pending = pending[len(batch):] charged = cost or 0.0 - day = await _charge(charged, limit=budget.daily_max_usd if priced else None, priced=priced) + day = await _LEDGER.charge(charged, limit=budget.daily_max_usd if priced else None, priced=priced) if day is None: # 估算之后、记账之前别的裁判先花掉了今天剩下的钱 stop([*batch, *pending], "daily_max_usd") pending = [] @@ -1353,7 +1305,7 @@ async def run_request(request: JudgeRequest, *, spec: ModelSpec, budget: Budget, inp, out = sum(t["input_tokens"] for t in tokens), sum(t["output_tokens"] for t in tokens) # 回复没带用量(有的网关不给、调用超时)就按估算记:宁可多记,不能让每日上限漏掉 actual = (pricing.estimate_cost(model_id, inp, out) if inp or out else charged) if priced else 0.0 - await _settle(day, actual - charged) + await _LEDGER.settle(day, actual - charged) spent += actual usage["input_tokens"] += inp usage["output_tokens"] += out diff --git a/backend/app/engine/spend.py b/backend/app/engine/spend.py index 0efa52a6..900324fd 100644 --- a/backend/app/engine/spend.py +++ b/backend/app/engine/spend.py @@ -1,11 +1,18 @@ -"""核查花费账:双盲复算、反方审稿、审批质询里的模型调用,按本地日期记一本每日账。 +"""模型调用的每日花费账:一本账一个键,按本地日期记,先预留、后找补(docs/design/next-round/01-效果度量与回归评测.md 做法 17)。 -照结论句裁判的每日计数做(judge.py 的 SPEND_KEY、_charge、_settle):调用前按目录价估算、先预留,加上这笔会超过 -每日上限就不记;调用后按实际用量找补差额。账落在设置表里,按本地日期滚动。 +调用前按目录价估算、先预留,加上这笔会超过每日上限就不记;调用后按实际用量找补差额。账落在设置表里,按本地日期 +滚动,不从设置接口进出(api/settings.py 只给只读的「今天花了多少」)。 -**为什么和裁判分开记。** 核查是一个 agent 查库、来回好几轮,一次花的钱比裁判判一份报告多得多。记在同一本账上, -核查一多,当天的裁判就全因为「每日上限」判不了——报告的出具会因此降档,而降档的原因和这份报告毫无关系。两本账 -各管各的上限,互不挤占。 +**一套代码,几本账。** 以前核查(这个模块)和结论句裁判(judge.py 的 _charge、_settle)各写一份一模一样的读改写,评测 +要是再抄一份就是第三份。现在都是 Ledger(key) 的实例,各记各的键: + +- verify_spend:核查——双盲复算、反方审稿、审批质询里的模型调用; +- judge_spend:结论句裁判(engine/judge.py); +- eval_spend:评测(预留,第 3 波的评测执行接上;设置和接口在那时加)。 + +**为什么各记各的。** 核查是一个 agent 查库、来回好几轮,一次花的钱比裁判判一份报告多得多;评测一次几十道题。记在同一本 +账上,一边一多,另一边当天就全因为「每日上限」做不了——报告的出具会因此降档,而降档的原因和这份报告毫无关系。几本账 +各管各的上限,互不挤占。三本按同一个本地日期滚动(_today)。 目录里没有价格的模型估不出金额:只计次数(unpriced_calls),金额上限对它不起作用,由调用方照实说明。 """ @@ -13,27 +20,26 @@ from __future__ import annotations import asyncio import weakref +from collections.abc import Callable from datetime import datetime from typing import Any from app.db.base import SessionLocal from app.db.models import Setting -#: 每日计数存在设置表的这一行:{date: 本地日期, usd, calls, unpriced_calls}。不从设置接口进出(api/settings.py) -SPEND_KEY = "verify_spend" - -_locks: "weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, asyncio.Lock]" = weakref.WeakKeyDictionary() +#: 每本账的读改写锁:{事件循环: {键: 锁}}。asyncio.Lock 不能跨循环用(测试每个用例一个循环),按循环各一组;同一个键 +#: 同一把——几个核查并行时各自预留,不串行的话两边读到同一个余额,会一起越过上限 +_locks: "weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, dict[str, asyncio.Lock]]" = weakref.WeakKeyDictionary() -def _lock() -> asyncio.Lock: - """读改写这一行时持的锁,按事件循环各一把(asyncio.Lock 不能跨循环用)。 - - 几个核查并行时各自预留:不串行的话两边读到同一个余额,会一起越过上限。 - """ +def _lock(key: str) -> asyncio.Lock: loop = asyncio.get_running_loop() - lock = _locks.get(loop) + group = _locks.get(loop) + if group is None: + group = _locks[loop] = {} + lock = group.get(key) if lock is None: - lock = _locks[loop] = asyncio.Lock() + lock = group[key] = asyncio.Lock() return lock @@ -42,57 +48,91 @@ def _today() -> str: return datetime.now().astimezone().date().isoformat() -def _spend_of(row: Any) -> dict[str, Any]: - value = row.value if row is not None and isinstance(row.value, dict) else {} - today = _today() - if value.get("date") != today: - return {"date": today, "usd": 0.0, "calls": 0, "unpriced_calls": 0} - return {"date": today, "usd": float(value.get("usd") or 0.0), "calls": int(value.get("calls") or 0), - "unpriced_calls": int(value.get("unpriced_calls") or 0)} +class Ledger: + """一本每日账,存在设置表 key 这一行:{date: 本地日期, usd, calls, unpriced_calls}。 + + today 是这本账的日历(测试换掉它来模拟跨天);不给就按本模块的 _today,调用时才取,换掉 _today 对没给 today 的几本 + 账都生效。""" + + def __init__(self, key: str, *, today: Callable[[], str] | None = None) -> None: + self.key = key + self._clock = today + + def _date(self) -> str: + return self._clock() if self._clock is not None else _today() + + def _spend_of(self, row: Any) -> dict[str, Any]: + value = row.value if row is not None and isinstance(row.value, dict) else {} + today = self._date() + if value.get("date") != today: + return {"date": today, "usd": 0.0, "calls": 0, "unpriced_calls": 0} + return {"date": today, "usd": float(value.get("usd") or 0.0), "calls": int(value.get("calls") or 0), + "unpriced_calls": int(value.get("unpriced_calls") or 0)} + + async def daily(self) -> dict[str, Any]: + """今天(本地日期)这本账已经花了多少:{date, usd, calls, unpriced_calls}。估不出金额的调用只计次数。""" + async with SessionLocal() as session: + return self._spend_of(await session.get(Setting, self.key)) + + async def charge(self, amount: float, *, limit: float | None, priced: bool) -> str | None: + """记一次调用、先按估算预留金额。加上这笔会超过每日上限就不记,返回 None;记下了返回记在哪天(找补时要用)。 + + limit 为 None 表示不限。估不出金额的调用(priced=False)照记次数,金额记 0,调用方应传 limit=None: + 拿 0 去比上限,等于说它不花钱。 + """ + async with _lock(self.key): + async with SessionLocal() as session: + row = await session.get(Setting, self.key) + spend = self._spend_of(row) + if limit is not None and spend["usd"] + amount > limit + 1e-12: + return None + spend["usd"] = round(spend["usd"] + amount, 8) + spend["calls"] += 1 + spend["unpriced_calls"] += 0 if priced else 1 + if row is None: + session.add(Setting(key=self.key, value=spend)) + else: + row.value = spend + await session.commit() + return spend["date"] + + async def settle(self, date: str, delta: float) -> None: + """调用结束后按实际用量找补预留的差额。跨了天就不找补:预留记在前一天,今天从零算。""" + if not delta: + return + async with _lock(self.key): + async with SessionLocal() as session: + row = await session.get(Setting, self.key) + spend = self._spend_of(row) + if row is None or spend["date"] != date: + return + spend["usd"] = max(0.0, round(spend["usd"] + delta, 8)) + row.value = spend + await session.commit() + + +#: 核查、裁判、评测各一本 +verify_spend = Ledger("verify_spend") +judge_spend = Ledger("judge_spend") +eval_spend = Ledger("eval_spend") + +#: 核查那本账的老名字(verify_agent、设置接口、已有测试从这里取) +SPEND_KEY = verify_spend.key async def daily_spend() -> dict[str, Any]: - """今天(本地日期)核查已经花了多少:{date, usd, calls, unpriced_calls}。估不出金额的调用只计次数。""" - async with SessionLocal() as session: - return _spend_of(await session.get(Setting, SPEND_KEY)) + """今天核查已经花了多少(verify_spend.daily)。""" + return await verify_spend.daily() async def charge(amount: float, *, limit: float | None, priced: bool) -> str | None: - """记一次调用、先按估算预留金额。加上这笔会超过每日上限就不记,返回 None;记下了返回记在哪天(找补时要用)。 - - limit 为 None 表示不限。估不出金额的调用(priced=False)照记次数,金额记 0,调用方应传 limit=None: - 拿 0 去比上限,等于说它不花钱。 - """ - async with _lock(): - async with SessionLocal() as session: - row = await session.get(Setting, SPEND_KEY) - spend = _spend_of(row) - if limit is not None and spend["usd"] + amount > limit + 1e-12: - return None - spend["usd"] = round(spend["usd"] + amount, 8) - spend["calls"] += 1 - spend["unpriced_calls"] += 0 if priced else 1 - if row is None: - session.add(Setting(key=SPEND_KEY, value=spend)) - else: - row.value = spend - await session.commit() - return spend["date"] + """核查记一次调用(verify_spend.charge)。""" + return await verify_spend.charge(amount, limit=limit, priced=priced) async def settle(date: str, delta: float) -> None: - """调用结束后按实际用量找补预留的差额。跨了天就不找补:预留记在前一天,今天从零算。""" - if not delta: - return - async with _lock(): - async with SessionLocal() as session: - row = await session.get(Setting, SPEND_KEY) - spend = _spend_of(row) - if row is None or spend["date"] != date: - return - spend["usd"] = max(0.0, round(spend["usd"] + delta, 8)) - row.value = spend - await session.commit() + """核查找补(verify_spend.settle)。""" + await verify_spend.settle(date, delta) -__all__ = ["SPEND_KEY", "charge", "daily_spend", "settle"] +__all__ = ["Ledger", "SPEND_KEY", "charge", "daily_spend", "eval_spend", "judge_spend", "settle", "verify_spend"] diff --git a/backend/tests/test_evidence_judge.py b/backend/tests/test_evidence_judge.py index f46d6a62..c742ccf1 100644 --- a/backend/tests/test_evidence_judge.py +++ b/backend/tests/test_evidence_judge.py @@ -30,6 +30,7 @@ from app.core.events import EventType from app.db.base import SessionLocal from app.db.models import Setting from app.engine import judge as judge_mod +from app.engine import spend from app.engine.evidence import build_catalog, compose_doc, iter_units from app.engine.judge import ( BATCH_SIZE, @@ -503,7 +504,7 @@ async def test_unlimited_never_truncates(monkeypatch, stored): raw = "".join(f"第 {i} 个区域的销售额增长来自新客。[[see:m:gmv]]" for i in range(BATCH_SIZE + 5)) doc = compose_doc(raw, catalog) async with SessionLocal() as session: # 今天已经花掉很多:不限就是不限 - session.add(Setting(key=SPEND_KEY, value={"date": judge_mod._today(), "usd": 999.0, "calls": 9})) + session.add(Setting(key=SPEND_KEY, value={"date": spend._today(), "usd": 999.0, "calls": 9})) await session.commit() outcome = await judge_doc(doc, catalog, spec=SPEC, budget=UNLIMITED) assert [len(units) for units in judge.judged] == [BATCH_SIZE, 5] @@ -517,7 +518,8 @@ async def test_the_daily_cap_resets_the_next_day(monkeypatch, stored): cost = prepare(doc, catalog).cost(PRICED, prepare(doc, catalog).cands) budget = Budget(max_claims=None, max_cost_usd=None, timeout_s=None, daily_max_usd=cost * 1.05) day = {"today": "2026-09-28"} - monkeypatch.setattr(judge_mod, "_today", lambda: day["today"]) + # 裁判的账和核查、评测按同一个本地日期滚动(engine/spend.py 的 Ledger) + monkeypatch.setattr(spend, "_today", lambda: day["today"]) judge = Judge(monkeypatch) first = await judge_doc(doc, catalog, spec=SPEC, budget=budget) diff --git a/backend/tests/test_spend_ledger.py b/backend/tests/test_spend_ledger.py new file mode 100644 index 00000000..a847ed7e --- /dev/null +++ b/backend/tests/test_spend_ledger.py @@ -0,0 +1,78 @@ +"""每日账泛化成 Ledger(key)(docs/design/next-round/01-效果度量与回归评测.md 做法 17)。 + +核查、裁判以前各写一份一模一样的「预留 → 找补 → 按本地日期滚动」,评测要是再抄一份就是第三份。现在三本账是同一个 +Ledger 的三个实例,各记各的键、各管各的上限:一本触顶不挤占另外两本。裁判、核查原有的行为(模块级的 SPEND_KEY、 +daily_spend、charge / settle)不变,已有的测试照旧守着;这里守账本本身和互不挤占。 +""" +from __future__ import annotations + +import pytest +from sqlalchemy import delete + +from app.db.base import SessionLocal +from app.db.models import Setting +from app.engine import judge, spend +from app.engine.spend import Ledger, eval_spend, judge_spend, verify_spend + + +@pytest.fixture(autouse=True) +async def _clean(): + keys = [verify_spend.key, judge_spend.key, eval_spend.key, "test_ledger"] + async with SessionLocal() as session: + await session.execute(delete(Setting).where(Setting.key.in_(keys))) + await session.commit() + yield + async with SessionLocal() as session: + await session.execute(delete(Setting).where(Setting.key.in_(keys))) + await session.commit() + + +def test_three_books_three_keys(): + assert (verify_spend.key, judge_spend.key, eval_spend.key) == ("verify_spend", "judge_spend", "eval_spend") + # 老名字照旧指向各自那本:核查的模块函数、裁判的 SPEND_KEY / daily_spend + assert spend.SPEND_KEY == verify_spend.key + assert judge.SPEND_KEY == judge_spend.key + + +async def test_reserve_then_settle_and_the_limit(): + book = Ledger("test_ledger") + day = await book.charge(0.4, limit=1.0, priced=True) + assert day is not None + assert await book.charge(0.7, limit=1.0, priced=True) is None, "加上这笔超过上限就不记" + await book.settle(day, -0.1) + got = await book.daily() + assert got["usd"] == pytest.approx(0.3) and got["calls"] == 1 and got["unpriced_calls"] == 0 + # 估不出金额的照记次数,不比上限 + assert await book.charge(0.0, limit=None, priced=False) is not None + assert (await book.daily())["unpriced_calls"] == 1 + + +async def test_a_new_day_starts_from_zero(): + day = {"today": "2026-09-28"} + book = Ledger("test_ledger", today=lambda: day["today"]) + first = await book.charge(0.9, limit=1.0, priced=True) + assert first == "2026-09-28" + assert await book.charge(0.2, limit=1.0, priced=True) is None + day["today"] = "2026-09-29" + assert await book.charge(0.2, limit=1.0, priced=True) == "2026-09-29" + await book.settle(first, -0.5) # 跨了天的预留不找补:今天从零算 + assert (await book.daily())["usd"] == pytest.approx(0.2) + + +async def test_one_book_hitting_its_cap_does_not_squeeze_the_others(): + """核查花光了当天的额度,裁判、评测照常记:三本账互不挤占。""" + assert await verify_spend.charge(2.0, limit=2.0, priced=True) is not None + assert await verify_spend.charge(0.01, limit=2.0, priced=True) is None + assert await judge_spend.charge(1.0, limit=2.0, priced=True) is not None + assert await eval_spend.charge(1.5, limit=2.0, priced=True) is not None + assert (await judge.daily_spend())["usd"] == pytest.approx(1.0) + assert (await spend.daily_spend())["usd"] == pytest.approx(2.0) + assert (await eval_spend.daily())["usd"] == pytest.approx(1.5) + + +async def test_all_books_roll_over_on_the_same_clock(monkeypatch): + """三本账按同一个本地日期滚动(spend._today):裁判不再自己另有一份时钟。""" + monkeypatch.setattr(spend, "_today", lambda: "2026-01-02") + assert await judge_spend.charge(0.1, limit=None, priced=True) == "2026-01-02" + assert (await judge.daily_spend())["date"] == "2026-01-02" + assert (await eval_spend.daily())["date"] == "2026-01-02" From e3d0fe618cc2d059179d424cf02bfa671e6775f7 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Fri, 16 Oct 2026 10:07:15 +0800 Subject: [PATCH 04/39] =?UTF-8?q?=E5=AE=9A=E6=97=B6=E7=9B=AE=E6=A0=87?= =?UTF-8?q?=E7=9A=84=E5=BA=95=E5=BA=A7=EF=BC=88=E6=80=BB=E6=96=B9=E6=A1=88?= =?UTF-8?q?=20X4=EF=BC=8C=E7=BB=86=E6=A1=88=2001=20=E5=81=9A=E6=B3=95=2015?= =?UTF-8?q?=EF=BC=89=EF=BC=9A=E5=AE=9A=E6=97=B6=E5=8F=AA=E6=9C=89=E4=B8=80?= =?UTF-8?q?=E5=A5=97=EF=BC=8C=E7=9B=AE=E6=A0=87=E7=A7=8D=E7=B1=BB=E5=9C=A8?= =?UTF-8?q?=20scheduler/service=20=E9=87=8C=E4=B8=80=E5=A4=84=E5=B0=81?= =?UTF-8?q?=E8=A3=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - service.target_of / target_columns:读写目标只经这里。workflow 目标 target_id 同 workflow_id;非工作流目标 workflow_id 存空串(老表的非空约束,只加不改的迁移放宽不了),这条规矩不散到别处 - fire_once 按目标种类分派:工作流那段原样搬进 _fire_workflow(重叠跳过、渲染输入、记触发、发起一字不变);这一波别的种类照实记「未能发起」并写明是哪类目标,不发起运行、不把空串当工作流去跑 - 建任务、PATCH 改工作流经 target_columns 一起写两列;存量任务(target_id 为空)照旧按 workflow_id 发起 - 接口和界面这一波不暴露新种类(第 3 波开放),返回的形状不变 - 新增 tests/test_schedule_target.py --- backend/app/api/schedules.py | 5 +- backend/app/scheduler/service.py | 109 +++++++++++++++------ backend/tests/test_schedule_target.py | 135 ++++++++++++++++++++++++++ 3 files changed, 217 insertions(+), 32 deletions(-) create mode 100644 backend/tests/test_schedule_target.py diff --git a/backend/app/api/schedules.py b/backend/app/api/schedules.py index 35e8cca1..d7f5b0c3 100644 --- a/backend/app/api/schedules.py +++ b/backend/app/api/schedules.py @@ -491,6 +491,8 @@ async def create_schedule_row(session: AsyncSession, body: dict[str, Any], *, ac values = schedule_values(body) hook = values.pop("webhook") workflow = await _check_workflow(session, values["workflow_id"], values["run_class"], values["input_template"]) + # 目标的几列只经 scheduler/service 的一处封装写(工作流目标:target_id 同 workflow_id) + values |= service.target_columns("workflow", values["workflow_id"]) row = Schedule(**values, workflow_name=workflow.name, next_run_at=service.next_run(values["cron"], values["timezone"], values["enabled"]), created_by=actor, **_webhook_columns(hook)) @@ -528,7 +530,8 @@ async def update_schedule(schedule_id: str, request: Request, if "name" in body: values["name"] = _name(body["name"]) if "workflow_id" in body: - values["workflow_id"] = _workflow_id(body["workflow_id"]) + # 目标的几列一起改(scheduler/service.target_columns):只改 workflow_id 的话 target_id 还指着原来的工作流 + values |= service.target_columns("workflow", _workflow_id(body["workflow_id"])) if "cron" in body: values["cron"] = _cron(body["cron"]) if "timezone" in body: diff --git a/backend/app/scheduler/service.py b/backend/app/scheduler/service.py index 14c25443..87d0ff18 100644 --- a/backend/app/scheduler/service.py +++ b/backend/app/scheduler/service.py @@ -3,6 +3,11 @@ 调度循环(loop.py)到点调 fire_once,接口的「立即运行」也调它:重叠跳过、渲染输入、记触发、发起运行这一套 只有一份。发起本身走 engine/launch.py 的 launch_run,和 HTTP 发起是同一套规则——正式运行只能从当前发布版本 发起,这一条不会在定时任务这边被绕过。 + +**定时的目标**(docs/design/next-round/00-总方案.md X4:定时只有一套):一个定时任务指向一个目标——工作流、评测题集 +(第 3 波)或数据源维护(第 3 波)。认领、补跑、重叠跳过、触发记录、通知都只有这一套,fire_once 按目标种类分派。 +老表的 workflow_id 是非空约束,只加不改的迁移放宽不了:目标不是工作流时它存空串。这条规矩只在 target_of / +target_columns 这一处,别处读写目标都经它们,不直接碰这两列。 """ from __future__ import annotations @@ -64,6 +69,33 @@ INTERRUPTED_REASON = "服务在发起运行前停止,这次触发没有发起 GENERIC_REASON = "发起运行时出错,详细信息见服务日志" +#: 定时的目标种类(Schedule.target_kind),顺序即以后界面上的顺序。这一波 fire_once 只会发起 workflow +TARGET_KINDS = ("workflow", "eval_suite", "source_maintenance") +TARGET_LABEL = {"workflow": "工作流", "eval_suite": "评测题集", "source_maintenance": "数据源维护"} +#: 目标种类这个版本还发起不了(库里有别的版本建的任务、手改过库):照实记「未能发起」,不当成工作流去跑 +TARGET_UNSUPPORTED = "这个定时任务的目标是「{kind}」,当前版本还不能按时发起这类目标,这次触发没有发起" + + +def target_of(schedule: Any) -> tuple[str, str]: + """一个定时任务的目标:(种类, id)。加列之前建的任务 target_kind 缺省 workflow、target_id 为空,目标就是 workflow_id。 + + workflow 时 id 以 workflow_id 为准(两列由 target_columns 一起写,对得上;老任务只有这一列)。""" + kind = getattr(schedule, "target_kind", None) or "workflow" + if kind == "workflow": + return kind, schedule.workflow_id or getattr(schedule, "target_id", None) or "" + return kind, getattr(schedule, "target_id", None) or "" + + +def target_columns(kind: str, target_id: str) -> dict[str, str]: + """写目标的几列:{target_kind, target_id, workflow_id}。workflow 时 target_id 同 workflow_id;别的种类 workflow_id 存空串 + (老表的非空约束)。种类不认识、id 为空抛 ValueError。""" + if kind not in TARGET_KINDS: + raise ValueError(f"不认识的定时目标种类:{kind}") + if not target_id: + raise ValueError("定时任务的目标 id 不能为空") + return {"target_kind": kind, "target_id": target_id, "workflow_id": target_id if kind == "workflow" else ""} + + class TemplateError(ValueError): """输入模板写错了。str() 是给人看的中文说明,指出是哪个输入。""" @@ -282,46 +314,61 @@ async def fire_once(schedule_id: str, fire_at: datetime, kind: str, *, actor: st """处理一次触发,返回记下的触发记录。同一 (任务, fire_at, kind) 已经有记录(两拍撞在一起、立即运行连点两下) 时不再发起,返回已有的那一条;任务不存在返回 None。 - 上一次运行还没结束就记 skipped_overlap;输入渲染不了、launch_run 拒绝或出错记 failed_to_start(拒绝写原话, - 其他异常写通用说明、详细信息进日志);发起成功记 started 和运行 id。 + 按目标种类分派(target_of):这一波只有工作流(_fire_workflow),别的种类记 failed_to_start、写明原因,不发起。 """ async with launch_lock(): async with SessionLocal() as session: schedule = await session.get(Schedule, schedule_id) if schedule is None: return None - name, workflow_id, run_class = schedule.name, schedule.workflow_id, schedule.run_class + target_kind, target_id = target_of(schedule) + name, run_class = schedule.name, schedule.run_class tz, cron_expr, template = schedule.timezone, schedule.cron, schedule.input_template - try: - inputs = render_inputs(check_template(template), template_values(fire_at, tz, cron_expr)) - except (TemplateError, cron.CronError, cron.TimezoneError) as e: - # 存进库之后才坏掉的(手改过库、时区库升级删了时区):照实说是哪一项,不发起 - return await _record(schedule_id, fire_at, kind, "failed_to_start", f"无法生成这次运行的输入:{e}", {}) - busy = await _active_run(schedule_id) - if busy is not None: - return await _record(schedule_id, fire_at, kind, "skipped_overlap", overlap_reason(*busy), inputs) - fire = await _record(schedule_id, fire_at, kind, "failed_to_start", INTERRUPTED_REASON, inputs, fresh=True) - if fire is None: - return await _existing(schedule_id, fire_at, kind) - from app.engine.launch import LaunchRefused, launch_run + if target_kind != "workflow": + reason = TARGET_UNSUPPORTED.format(kind=TARGET_LABEL.get(target_kind, target_kind)) + return await _record(schedule_id, fire_at, kind, "failed_to_start", reason, {}) + return await _fire_workflow(schedule_id, fire_at, kind, actor=actor, name=name, workflow_id=target_id, + run_class=run_class, tz=tz, cron_expr=cron_expr, template=template) - _LAUNCHING.add(fire.id) + +async def _fire_workflow(schedule_id: str, fire_at: datetime, kind: str, *, actor: str | None, name: str, + workflow_id: str, run_class: str, tz: str, cron_expr: str, template: Any, + ) -> ScheduleFire | None: + """目标是工作流的一次触发(持着 launch_lock)。 + + 上一次运行还没结束就记 skipped_overlap;输入渲染不了、launch_run 拒绝或出错记 failed_to_start(拒绝写原话, + 其他异常写通用说明、详细信息进日志);发起成功记 started 和运行 id。 + """ + try: + inputs = render_inputs(check_template(template), template_values(fire_at, tz, cron_expr)) + except (TemplateError, cron.CronError, cron.TimezoneError) as e: + # 存进库之后才坏掉的(手改过库、时区库升级删了时区):照实说是哪一项,不发起 + return await _record(schedule_id, fire_at, kind, "failed_to_start", f"无法生成这次运行的输入:{e}", {}) + busy = await _active_run(schedule_id) + if busy is not None: + return await _record(schedule_id, fire_at, kind, "skipped_overlap", overlap_reason(*busy), inputs) + fire = await _record(schedule_id, fire_at, kind, "failed_to_start", INTERRUPTED_REASON, inputs, fresh=True) + if fire is None: + return await _existing(schedule_id, fire_at, kind) + from app.engine.launch import LaunchRefused, launch_run + + _LAUNCHING.add(fire.id) + try: try: - try: - async with SessionLocal() as session: - run = await launch_run(session, workflow_id=workflow_id, run_class=run_class, - input_payload=inputs, actor=actor or actor_label(name), - schedule_id=schedule_id, origin_kind="schedule", - origin={"schedule_id": schedule_id, "schedule": name, - "fire_id": fire.id, "fire_kind": kind}) - except LaunchRefused as e: - return await _finish(fire.id, status="failed_to_start", reason=e.message) - except Exception: # noqa: BLE001 - 发起失败不能让调度循环停下;详细信息进日志,记录上写通用说明 - logger.exception("定时任务「%s」(%s)发起运行失败", name, schedule_id) - return await _finish(fire.id, status="failed_to_start", reason=GENERIC_REASON) - return await _finish(fire.id, status="started", reason=None, run_id=run.id) - finally: - _LAUNCHING.discard(fire.id) + async with SessionLocal() as session: + run = await launch_run(session, workflow_id=workflow_id, run_class=run_class, + input_payload=inputs, actor=actor or actor_label(name), + schedule_id=schedule_id, origin_kind="schedule", + origin={"schedule_id": schedule_id, "schedule": name, + "fire_id": fire.id, "fire_kind": kind}) + except LaunchRefused as e: + return await _finish(fire.id, status="failed_to_start", reason=e.message) + except Exception: # noqa: BLE001 - 发起失败不能让调度循环停下;详细信息进日志,记录上写通用说明 + logger.exception("定时任务「%s」(%s)发起运行失败", name, schedule_id) + return await _finish(fire.id, status="failed_to_start", reason=GENERIC_REASON) + return await _finish(fire.id, status="started", reason=None, run_id=run.id) + finally: + _LAUNCHING.discard(fire.id) async def record_skip(schedule_id: str, fire_at: datetime, reason: str) -> ScheduleFire | None: diff --git a/backend/tests/test_schedule_target.py b/backend/tests/test_schedule_target.py new file mode 100644 index 00000000..ad717c8d --- /dev/null +++ b/backend/tests/test_schedule_target.py @@ -0,0 +1,135 @@ +"""定时任务的目标(docs/design/next-round/00-总方案.md X4:定时只有一套)。 + +骨架给 Schedule 加了 target_kind / target_id;workflow_id 是老表上的非空约束,只加不改的迁移放宽不了。规矩只在 +scheduler/service.py 的一处封装里:workflow 目标的 target_id 同 workflow_id;不是工作流的目标 workflow_id 存空串。 +这里守: + +- 存量任务(target_kind 是缺省的 workflow、target_id 为空)照旧按 workflow_id 发起,行为一字不变; +- 建任务、改工作流都经同一处封装写目标,两列对得上; +- fire_once 按目标种类分派。这一波只实现 workflow,别的种类照实记「未能发起」并写明原因,不发起运行、不悄悄当成 + 工作流去跑空串; +- 接口这一波不暴露新种类(接口返回的形状不变,测试见 test_schedules_api.py 的 SCHEDULE_KEYS)。 + +工作流、任务名都是虚构的。 +""" +from __future__ import annotations + +import uuid +from datetime import datetime, timezone + +import pytest +from httpx import ASGITransport, AsyncClient +from sqlalchemy import delete, func, select + +from app.db.base import SessionLocal +from app.db.models import Run, Schedule, ScheduleFire, Workflow, WorkflowVersion +from app.engine.runner import run_manager +from app.main import app +from app.scheduler import service + +UTC = timezone.utc +NINE = datetime(2026, 10, 5, 1, 0, tzinfo=UTC) + + +@pytest.fixture(autouse=True) +async def engine_up(): + await run_manager.setup() + async with SessionLocal() as session: + await session.execute(delete(Schedule)) + await session.commit() + yield + await run_manager.shutdown() + + +def _node(nid, ntype, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}} + + +def _graph() -> dict: + return {"nodes": [_node("start", "input", fields=[{"name": "stat_date", "required": True}]), + _node("out", "output", fields=[{"name": "r", "value": "完成 {{ input.stat_date }}"}])], + "edges": [{"source": "start", "target": "out"}]} + + +async def _workflow() -> str: + async with SessionLocal() as session: + wf = Workflow(name=f"门店日报-{uuid.uuid4().hex[:6]}", graph=_graph(), status="published", published_version=1) + session.add(wf) + await session.flush() + session.add(WorkflowVersion(workflow_id=wf.id, version=1, graph=_graph())) + await session.commit() + return wf.id + + +def test_target_of_a_legacy_row_reads_the_workflow(): + """加列之前建的任务:target_kind 缺省 workflow、target_id 为空,目标就是 workflow_id。""" + row = Schedule(name="老任务", workflow_id="wf-legacy", cron="0 9 * * *") + assert service.target_of(row) == ("workflow", "wf-legacy") + row.target_kind = "workflow" + row.target_id = None + assert service.target_of(row) == ("workflow", "wf-legacy") + + +def test_target_columns_keep_the_two_columns_in_step(): + assert service.target_columns("workflow", "wf-1") == {"target_kind": "workflow", "target_id": "wf-1", + "workflow_id": "wf-1"} + # 非工作流目标:workflow_id 存空串(老表的非空约束),目标 id 只在 target_id + assert service.target_columns("eval_suite", "suite-1") == {"target_kind": "eval_suite", "target_id": "suite-1", + "workflow_id": ""} + row = Schedule(name="评测", cron="0 9 * * *", **service.target_columns("eval_suite", "suite-1")) + assert service.target_of(row) == ("eval_suite", "suite-1") + with pytest.raises(ValueError): + service.target_columns("nonsense", "x") + with pytest.raises(ValueError): + service.target_columns("workflow", "") + + +async def test_creating_and_retargeting_through_the_api_write_both_columns(): + wf_a, wf_b = await _workflow(), await _workflow() + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: + made = await client.post("/api/schedules", json={ + "name": "每日早报", "workflow_id": wf_a, "cron": "0 9 * * *", "timezone": "Asia/Shanghai", + "input_template": {"stat_date": "{{ schedule.yesterday }}"}}) + assert made.status_code == 201, made.text + body = made.json() + assert "target_kind" not in body and "target_id" not in body, "这一波接口不暴露新种类" + async with SessionLocal() as session: + row = await session.get(Schedule, body["id"]) + assert (row.target_kind, row.target_id, row.workflow_id) == ("workflow", wf_a, wf_a) + moved = await client.patch(f"/api/schedules/{body['id']}", json={"workflow_id": wf_b, "version": body["version"]}) + assert moved.status_code == 200, moved.text + async with SessionLocal() as session: + row = await session.get(Schedule, body["id"]) + assert (row.target_kind, row.target_id, row.workflow_id) == ("workflow", wf_b, wf_b) + + +async def test_a_legacy_schedule_still_fires_its_workflow(): + wf = await _workflow() + async with SessionLocal() as session: + row = Schedule(name="老任务", workflow_id=wf, cron="0 9 * * *", timezone="Asia/Shanghai", run_class="formal", + input_template={"stat_date": "{{ schedule.yesterday }}"}, next_run_at=NINE) + session.add(row) + await session.commit() + schedule_id = row.id + fire = await service.fire_once(schedule_id, NINE, "manual") + assert fire is not None and fire.status == "started" and fire.run_id + async with SessionLocal() as session: + run = await session.get(Run, fire.run_id) + assert run.workflow_id == wf and run.input == {"stat_date": "2026-10-04"} + + +async def test_a_target_kind_this_wave_cannot_fire_is_recorded_not_run(): + async with SessionLocal() as session: + row = Schedule(name="题集回归", cron="0 9 * * *", timezone="Asia/Shanghai", + **service.target_columns("eval_suite", "suite-x"), next_run_at=NINE) + session.add(row) + await session.commit() + schedule_id = row.id + runs_before = await session.scalar(select(func.count()).select_from(Run)) + fire = await service.fire_once(schedule_id, NINE, "manual") + assert fire is not None and fire.status == "failed_to_start" and fire.run_id is None + assert fire.reason == service.TARGET_UNSUPPORTED.format(kind=service.TARGET_LABEL["eval_suite"]) + async with SessionLocal() as session: + assert await session.scalar(select(func.count()).select_from(Run)) == runs_before, "不能发起任何运行" + assert await session.scalar(select(func.count()).select_from(ScheduleFire) + .where(ScheduleFire.schedule_id == schedule_id)) == 1 From 845809486a1d29cd6754b2f5981acfcf67283c28 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Fri, 16 Oct 2026 10:14:27 +0800 Subject: [PATCH 05/39] =?UTF-8?q?=E9=A9=B3=E5=9B=9E=E4=B8=8D=E5=86=8D?= =?UTF-8?q?=E8=AE=B0=E6=88=90=E5=A4=B1=E8=B4=A5=EF=BC=88=E5=90=8E=E7=AB=AF?= =?UTF-8?q?=EF=BC=89=EF=BC=9A=E4=BA=BA=E7=9A=84=E9=A9=B3=E5=9B=9E=E6=94=B6?= =?UTF-8?q?=E6=88=90=20cancelled=20+=20end=5Freason=3Drejected=EF=BC=8C?= =?UTF-8?q?=E6=B6=88=E8=B4=B9=E6=96=B9=E9=83=BD=E8=AF=BB=E8=BF=99=E4=B8=80?= =?UTF-8?q?=E4=B8=AA=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 为什么:动作确认、工具 / 代码审批、审批驳回即终止、填写 / 编辑被驳回、双盲裁决的驳回以前抛普通 NodeError,运行记成 failed。 通知推「实例失败」,定时推「运行失败」,发布单把驳回过的试运行说成「试运行失败了」并挡住发布,记录页还给「继续运行」 (接着跑只会读到同一个驳回答复、再停一次)。人的决定被说成故障,每个消费方各写一份映射,已经漏了四处(05-已知遗留 (a)、总方案 X9)。 做了什么: - engine/context 新增 NodeRejected(NodeError),带 node_id、kind、decision(reject / expired_reject)、actor、note;六处驳回改抛它, 原话一字不变。待办超时驳回的答复带 expired,Decision.expired 据此记成 expired_reject。 - runner._drive 在通用 except 之前单独接住:status=cancelled、end_reason=rejected、error 写原话、error_node 写那一步, run.cancelled 带 reason/node_id/label/kind/decision/actor/note;关审批、回收沙箱、关流、封存走取消的那一套。手动停止 (执行中取消、放弃断点上的运行)写 end_reason=stopped,run.cancelled 带 reason=stopped。 - 继续运行:被驳回的 409 code=run_rejected「被驳回的运行不能接着跑:改好内容后重新发起」;存量 failed+驳回(失败那一步上 最近一条处理完的待办是驳回,和实例页同一份判断 resolve.legacy_rejected)也拒。RunOut 带 end_reason。 - apps/resolve.instance_state:cancelled+rejected 和存量 failed+驳回说同一句(「「付款」未确认(被驳回)」「「x」被驳回」)。 - notify_run:状态那一句取自 instance_state,被驳回的走「实例结束」、ok=False,走到「已退回」成果的不再写「已完成」; 手动停止照旧不推。顺带修 04-对外集成与触达 B1:发起人是认证身份(run.origin.actor_verified)时 users 传 [started_by], 个人渠道订阅了实例完成 / 失败终于收得到;自报署名不算。 - 定时 webhook:cancelled+rejected 按「运行结束」推「未出具(被驳回)」、写驳回的那一步,不写备注和原话、不画成功色; 手动停止仍不推。 - 发布单:试运行被驳回不算 plan.trial_failed,记 warning plan.trial_rejected「最近一次试运行在「x」被驳回、没有走完」(带白话)。 - (h) 重启时排队中的运行改成 failed 时补发 run.failed(同一句原因),时间线里有了失败原因这一行;不写结束时间、不封存,同以前。 - (f) tasks.tick / actions.tick 交回这一拍处理过的 id 列表(调度循环只记条数),测试改成只断言自己造的待办;先加了预置 一条无关到期待办的 fixture,旧的按条数断言当场失败(3 == 1),改完通过。 已封存的老运行不改写。测试:新增 test_run_rejected(五种关卡参数化、超时驳回、继续运行拒绝含存量、手动停止、补发 run.failed、 instance_state 新老形状同一句)、test_notify_run(驳回走实例结束、手动停止不推、已退回不写已完成、个人渠道 B1 两条); webhook、发布单、动作确认、双盲、审批的老断言跟上。去掉 runner 里接 NodeRejected 的分支回跑,新测试 8 条失败。 --- backend/app/api/runs.py | 11 +- backend/app/apps/resolve.py | 48 ++++-- backend/app/blueprints/publish_plan.py | 20 +++ backend/app/engine/approval.py | 4 + backend/app/engine/context.py | 29 ++++ backend/app/engine/nodes/dual_check.py | 4 +- backend/app/engine/nodes/human.py | 6 +- backend/app/engine/nodes/tools.py | 19 ++- backend/app/engine/runner.py | 71 ++++++-- backend/app/notify/service.py | 26 ++- backend/app/plain.py | 3 + backend/app/process/actions.py | 9 +- backend/app/process/tasks.py | 14 +- backend/app/scheduler/loop.py | 4 +- backend/app/scheduler/webhook.py | 41 ++++- backend/tests/test_app_instance_page.py | 2 +- backend/tests/test_approval.py | 13 +- backend/tests/test_blueprint_trial.py | 2 +- backend/tests/test_dual_check_node.py | 10 +- backend/tests/test_notify_run.py | 199 ++++++++++++++++++++++ backend/tests/test_process_actions.py | 20 ++- backend/tests/test_process_tasks.py | 37 ++-- backend/tests/test_publish_plan.py | 24 +++ backend/tests/test_records.py | 2 +- backend/tests/test_run_rejected.py | 214 ++++++++++++++++++++++++ backend/tests/test_scheduler_webhook.py | 24 ++- 26 files changed, 768 insertions(+), 88 deletions(-) create mode 100644 backend/tests/test_notify_run.py create mode 100644 backend/tests/test_run_rejected.py diff --git a/backend/app/api/runs.py b/backend/app/api/runs.py index 3a35364e..2832e340 100644 --- a/backend/app/api/runs.py +++ b/backend/app/api/runs.py @@ -56,6 +56,9 @@ class RunOut(BaseModel): input: dict[str, Any] output: dict[str, Any] error: str | None + #: 结束在 cancelled 的原因(docs/design/next-round/00-总方案.md X9):rejected 人工驳回、stopped 手动停止。 + #: 其余终态、加这一列之前的运行为空。界面、通知、定时推送都读这一个字段,不再解析报错原文 + end_reason: str | None = None usage: dict[str, Any] run_class: str | None = "exploratory" version: int | None = None @@ -289,11 +292,13 @@ async def cancel_run(run_id: str, x_actor: str | None = Header(default=None)) -> run = await session.get(Run, run_id) if run is None: raise HTTPException(404, "运行记录不存在") - raise HTTPException(409, _cannot_cancel(run.status)) + raise HTTPException(409, _cannot_cancel(run.status, run.end_reason)) return {"ok": True, "status": outcome} -def _cannot_cancel(status: str) -> str: +def _cannot_cancel(status: str, end_reason: str | None = None) -> str: + if status == "cancelled" and end_reason == "rejected": + return "这次运行已被驳回结束,没有要停止的。" return { "succeeded": "这次运行已完成,没有要停止的。", "failed": "这次运行已失败结束,没有要停止的;如需继续,请点「继续运行」。", @@ -396,7 +401,7 @@ async def continue_run( except KeyError as e: raise _missing(e) from e except ContinueRefused as e: - # 正式运行带图(formal_graph_locked)、写入结果未知还没核实(action_unresolved):带机器码 + # 正式运行带图(formal_graph_locked)、写入结果未知还没核实(action_unresolved)、被驳回了的(run_rejected):带机器码 raise CodedHTTPException(409, str(e), e.code) from e except ValueError as e: raise HTTPException(409, str(e)) from e diff --git a/backend/app/apps/resolve.py b/backend/app/apps/resolve.py index 97564ea6..0a23a758 100644 --- a/backend/app/apps/resolve.py +++ b/backend/app/apps/resolve.py @@ -51,6 +51,8 @@ NEEDS_UPGRADE = "app_needs_upgrade" STATUS_TEXT = {"queued": "排队中", "running": "运行中", "interrupted": "等待处理", "succeeded": "已完成", "failed": "失败", "cancelled": "已取消"} +#: 被驳回、又说不出是哪一步时(没有节点可指)的说法 +REJECTED_TEXT = "被驳回" TIER_TEXT = {"formal": "完整出具", "degraded": "降档出具", "withheld": "不予出具"} NO_TIER = "未声明出具契约" #: 蓝图试运行(docs/design/normal-user-apps.md N4):印章的来源、说明,趋势只有一个点时的说明,预览里入口按钮为什么点不了 @@ -1277,8 +1279,9 @@ async def _trails(session: AsyncSession, runs: list[Run]) -> dict[str, _Trail]: for row in (await session.execute(select(ActionRecord).where(ActionRecord.run_id.in_(ids)) .order_by(ActionRecord.created_at, ActionRecord.attempt))).scalars(): out[row.run_id].actions.append(row) - # 节点事件只有结束了的运行要看(走到了哪个成果节点、停在哪个节点上) - wanted = [r.id for r in runs if r.status in ("succeeded", "failed")] + # 节点事件只有结束了的运行要看(走到了哪个成果节点、停在哪个节点上)。取消的只有被驳回的要看(停在哪一步) + wanted = [r.id for r in runs if r.status in ("succeeded", "failed") + or (r.status == "cancelled" and r.end_reason == "rejected")] if wanted: for run_id, etype, node_id in (await session.execute( select(RunEvent.run_id, RunEvent.type, RunEvent.node_id).where( @@ -1325,6 +1328,26 @@ def rejected_outputs(graph: Any) -> dict[str, set[str]]: return out +def _rejection_text(trail: _Trail, node_id: str, title: str) -> str | None: + """停在被人驳回的待办上时怎么说:动作确认写「「付款」未确认(被驳回)」,别的写「「x」被驳回」;不是驳回的为 None。""" + kind, decision = _decision(trail, node_id) + if kind == "action_confirm" and decision == "reject": + return f"「{title}」未确认(被驳回)" + if decision in _REJECTIONS: + return f"「{title}」被驳回" + return None + + +async def legacy_rejected(session: AsyncSession, run: Run) -> bool: + """加 end_reason 之前被人驳回的运行记的是 failed(封存不改写):失败的那一步上最近一条处理完的待办是驳回。和实例页 + 认存量驳回(instance_state 的 failed 分支)是同一份判断;runner 据此不许它接着跑。""" + if run.status != "failed": + return False + trail = (await _trails(session, [run]))[run.id] + failed = [nid for etype, nid in trail.events if etype == "node.failed"] + return bool(failed) and _decision(trail, failed[-1])[1] in _REJECTIONS + + def instance_state(run: Run, trail: _Trail) -> dict[str, str]: """实例给人看的状态 {code, text}:code 是界面画状态标记用的状态(运行的那几种,加 held),text 说这件事办到哪了、 结论是什么,不只是运行状态(P1)。 @@ -1334,10 +1357,12 @@ def instance_state(run: Run, trail: _Trail) -> dict[str, str]: 否定的那种结局——这次运行里人工节点驳回了、从它的「驳回」出口一路走到的成果节点(rejected_outputs)——code 记 cancelled(同「未确认(被驳回)」,是人的决定,不画成成功),文字照旧写成果节点的标题; - 停着:有已发出、结果未知的动作,写「等待核实写入结果」;否则按在等的待办说(等待审批、等待确认发送……); - - 失败:停在一个被人驳回的待办上的——动作确认被驳回(「付款」未确认(被驳回))、审批设了驳回即终止、补充输入被 - 驳回——是人的决定,不是故障,code 记 cancelled;核实为未生效、选了放弃的同理。外部系统拒绝之类照写失败。 + - 被驳回(cancelled + end_reason=rejected,engine/context.NodeRejected):动作确认被驳回写「「付款」未确认(被驳回)」, + 审批设了驳回即终止、补充输入被驳回等写「「x」被驳回」,code 记 cancelled;手动停止的写「已取消」; + - 失败:加 end_reason 之前被人驳回的运行记的是 failed(封存不改写),停在被驳回的待办上的照上一条说,和新形状说法 + 一样;核实为未生效、选了放弃的同理,code 记 cancelled。外部系统拒绝之类照写失败。 - 运行本身的状态不改(驳回动作确认在引擎里就是节点失败、运行结束):这里只是把它说清楚。""" + 通知(notify/service.notify_run)的文字也取自这里:实例页和推送说同一句话。""" from app.apps.tree import node_map nodes = node_map(run.graph) @@ -1360,15 +1385,18 @@ def instance_state(run: Run, trail: _Trail) -> dict[str, str]: negative = any(_decision(trail, gate)[1] in _REJECTIONS for gate in gates) return {"code": "cancelled" if negative else "succeeded", "text": title(reached[0])} return {"code": "succeeded", "text": STATUS_TEXT["succeeded"]} + if run.status == "cancelled" and run.end_reason == "rejected": + failed = [nid for etype, nid in trail.events if etype == "node.failed"] + node_id = run.error_node_id or (failed[-1] if failed else None) + said = _rejection_text(trail, node_id, title(node_id)) if node_id else None + # 待办没留下(老的审批、没开出待办)时按驳回的那一步说,不退回「已取消」:这是人的决定,不是有人停了它 + return {"code": "cancelled", "text": said or (f"「{title(node_id)}」被驳回" if node_id else REJECTED_TEXT)} if run.status == "failed": failed = [nid for etype, nid in trail.events if etype == "node.failed"] if failed: node_id = failed[-1] - kind, decision = _decision(trail, node_id) - if kind == "action_confirm" and decision == "reject": - return {"code": "cancelled", "text": f"「{title(node_id)}」未确认(被驳回)"} - if decision in _REJECTIONS: - return {"code": "cancelled", "text": f"「{title(node_id)}」被驳回"} + if said := _rejection_text(trail, node_id, title(node_id)): + return {"code": "cancelled", "text": said} record = next((a for a in reversed(trail.actions) if a.node_id == node_id), None) if record is not None and record.state == "failed": if (record.resolution or {}).get("outcome") == "abandon": diff --git a/backend/app/blueprints/publish_plan.py b/backend/app/blueprints/publish_plan.py index 3bbcecf1..cdc1749b 100644 --- a/backend/app/blueprints/publish_plan.py +++ b/backend/app/blueprints/publish_plan.py @@ -70,6 +70,8 @@ NOT_INSTANTIATED = "plan.not_instantiated" GRAPH_CHANGED = "plan.graph_changed" UNTRIALED = "plan.untrialed" TRIAL_FAILED = "plan.trial_failed" +#: 最近一次试运行在某一步被人驳回、没有走完(cancelled + end_reason=rejected):人的决定,不是故障,不挡发布、照实说 +TRIAL_REJECTED = "plan.trial_rejected" TRIAL_DIFFERS = "plan.trial_differs" #: 蓝图检查:写回的结果不明时没人核实(blueprints/ops.blueprint_issues) RECONCILER_UNBOUND = "bp.reconciler_unbound" @@ -363,6 +365,12 @@ async def _preflight(session: AsyncSession, bp: Blueprint, target: _Target, *, l if is_trial_of(trial, bp) and trial.status == "failed": why = (trial.error or "").strip().splitlines()[0][:200] if (trial.error or "").strip() else "" out.append(_problem(TRIAL_FAILED, f"最近一次试运行失败了{f':{why}' if why else ''}", source="plan")) + elif is_trial_of(trial, bp) and trial.status == "cancelled" and trial.end_reason == "rejected": + # 搭建者在试运行里驳回了写入、审批:那是他的决定,不是图出了错,不说成「试运行失败了」、也不挡发布;只照实说 + # 没走完——驳回之后的步骤没用真数看过 + where = _node_title(trial.graph, trial.error_node_id) + out.append(_problem(TRIAL_REJECTED, f"最近一次试运行{f'在「{where}」' if where else ''}被驳回、没有走完", + "warning", source="plan")) elif not is_trial_of(trial, bp) or trial.status != "succeeded": out.append(_problem(UNTRIALED, "还没有成功的试运行:发布之前没用真数看过结果", "warning", source="plan")) elif graph_hash(trial.graph or {}) != target.digest: @@ -370,6 +378,17 @@ async def _preflight(session: AsyncSession, bp: Blueprint, target: _Target, *, l return [_with_who(p) for p in out] +def _node_title(graph: Any, node_id: str | None) -> str | None: + """节点在图上的名字(没有名字的给 id);没有节点时为 None。""" + if not node_id: + return None + for node in (graph or {}).get("nodes") or []: + if isinstance(node, dict) and node.get("id") == node_id: + data = node.get("data") if isinstance(node.get("data"), dict) else {} + return str(data.get("label") or node_id) + return node_id + + def _with_who(issue: dict[str, Any]) -> dict[str, Any]: """问题带上白话(没登记白话的照旧只有技术说法)和归谁处理(you / assistant / admin,摘要按它分三栏)。""" from app.plain import who_of, with_plain @@ -428,6 +447,7 @@ def _summary(bp: Blueprint, target: _Target, *, level: str, first_run: bool, opt if current and current > target.version else "" codes = {p.get("code") for p in problems} tried = ";发布的内容和最近一次试运行不同(试运行之后又改过),建议再试一次" if TRIAL_DIFFERS in codes else \ + ";最近一次试运行被驳回、没有走完" if TRIAL_REJECTED in codes else \ ";还没有用真数试运行过" if UNTRIALED in codes else "" if target.published_as == level: say("publish_workflow", f"「{name}」v{target.version} 已经按{LEVEL_TEXT[level]}级别发布过,不重复发布{tried}") diff --git a/backend/app/engine/approval.py b/backend/app/engine/approval.py index cb8beb21..5da191b8 100644 --- a/backend/app/engine/approval.py +++ b/backend/app/engine/approval.py @@ -39,6 +39,9 @@ class Decision: #: 审批卡上点的是「始终允许」:批准这次,并且这个工具以后改由门控把关(tools/trust.py)。 #: 只认明确的 True,而且只跟着批准走 always: bool = False + #: 这次驳回是待办超时、按「超时驳回」替人做的(process/tasks.py 的 tick),不是有人点了驳回。只跟着驳回走, + #: 节点据此在 NodeRejected 上记 decision=expired_reject + expired: bool = False def _word(value: Any) -> str: @@ -71,6 +74,7 @@ def read_decision(response: Any) -> Decision: code=code if isinstance(code, str) and code else None, value=response.get("value"), always=verdict is True and response.get("always") is True, + expired=verdict is False and response.get("expired") is True, ) verdict = _verdict(response) # 不是一个是/否词的字符串("拒绝,这个表是生产库")既算不上批准,本身又 diff --git a/backend/app/engine/context.py b/backend/app/engine/context.py index ad197844..9e4654a3 100644 --- a/backend/app/engine/context.py +++ b/backend/app/engine/context.py @@ -106,3 +106,32 @@ class NodeError(RuntimeError): super().__init__(message) self.node_id = node_id self.retryable = retryable + + +#: 驳回发生在哪种关卡上(NodeRejected.kind):人工节点的三种模式、写入前的动作确认、代码执行审批、工具调用审批、双盲裁决 +REJECTION_KINDS = ("approve", "input", "edit", "action_confirm", "code", "tool", "dual_check") + + +class NodeRejected(NodeError): + """人的决定让运行到这里结束:驳回、超时驳回(docs/design/next-round/05-已知遗留.md (a))。 + + 这不是故障。以前六处驳回都抛普通的 NodeError,运行记成 failed:通知推「实例失败」,定时推「运行失败」,发布单把 + 驳回过的试运行说成「试运行失败了」,记录页还给「继续运行」——接着跑只会读到同一个驳回答复、再停一次。现在 runner + 单独接住它,收成 cancelled + end_reason=rejected,run.cancelled 带上在哪一步、谁、结论和备注,消费方都读这一个字段。 + + 是 NodeError 的子类:节点失败的那一套(node.failed、on_error=continue)照旧认它;原话(str)照旧写给人看, + Run.error 和以前一字不差。kind 见 REJECTION_KINDS;decision 是 reject 或 expired_reject(同待办的结论)。""" + + def __init__(self, node_id: str, message: str, *, kind: str, decision: str = "reject", + actor: str | None = None, note: str = "") -> None: + super().__init__(node_id, message) + self.kind = kind + self.decision = decision + self.actor = actor + self.note = note + + @classmethod + def of(cls, ctx: Any, message: str, *, kind: str, decision: Any) -> NodeRejected: + """按节点上下文和读出来的审批结论(engine/approval.Decision)造一个:谁驳回的取 ctx.actor()。""" + return cls(ctx.node.id, message, kind=kind, decision="expired_reject" if decision.expired else "reject", + actor=ctx.actor(), note=decision.note or "") diff --git a/backend/app/engine/nodes/dual_check.py b/backend/app/engine/nodes/dual_check.py index 0152179c..d50cdcc0 100644 --- a/backend/app/engine/nodes/dual_check.py +++ b/backend/app/engine/nodes/dual_check.py @@ -70,7 +70,7 @@ from app.db.base import SessionLocal from app.db.models import DataSource from app.engine import deadline, recompute from app.engine.approval import read_decision -from app.engine.context import NodeContext, NodeError +from app.engine.context import NodeContext, NodeError, NodeRejected from app.engine.dual_check_facts import preview, preview_joint, record_choices, upload_dates from app.engine.evidence import build_catalog, is_number, next_exec from app.engine.expressions import ExpressionError, _cell_call, _chain_root, cell_value, column_kind, numeric_text, \ @@ -1396,7 +1396,7 @@ def _choices(ctx: NodeContext, response: Any, rows: list[dict[str, Any]]) -> tup 填写模式一样:这一步没有驳回的出口。""" decision = read_decision(response) if isinstance(response, dict) and decision.rejected: - raise NodeError(ctx.node.id, f"人工驳回:{decision.note or '未说明原因'}") + raise NodeRejected.of(ctx, f"人工驳回:{decision.note or '未说明原因'}", kind="dual_check", decision=decision) value = decision.value if isinstance(decision.value, dict) else response if isinstance(response, dict) else {} raw = value.get("choices") if isinstance(value.get("choices"), dict) else {} choices = {r["metric_id"]: raw[r["metric_id"]] for r in rows if raw.get(r["metric_id"]) in CHOICES} diff --git a/backend/app/engine/nodes/human.py b/backend/app/engine/nodes/human.py index 0c266bec..1e20e9b6 100644 --- a/backend/app/engine/nodes/human.py +++ b/backend/app/engine/nodes/human.py @@ -8,7 +8,7 @@ from typing import Any from app.core.events import EventType from app.db.base import SessionLocal from app.engine.approval import read_decision -from app.engine.context import NodeContext, NodeError +from app.engine.context import NodeContext, NodeError, NodeRejected from app.engine.replay import ask, once from app.engine.state import GraphState, merge_usage, message_text, template_context from app.providers import catalog @@ -52,7 +52,7 @@ async def run_human(state: GraphState, ctx: NodeContext) -> dict[str, Any]: "__decision__": "approved" if approved else "rejected", } if not approved and ctx.cfg("stop_on_reject", False): - raise NodeError(ctx.node.id, f"人工驳回:{result['note'] or '未说明原因'}") + raise NodeRejected.of(ctx, f"人工驳回:{result['note'] or '未说明原因'}", kind="approve", decision=decision) else: # edit / input # 驳回在这两种模式下以前是个摆设:__decision__ 硬编码成 approved, # approved 字段看都不看。界面上按钮在、toast 说"已驳回",运行却照常 @@ -65,7 +65,7 @@ async def run_human(state: GraphState, ctx: NodeContext) -> dict[str, Any]: # 只有结构体里明说了"不"才算驳回:这两种模式下裸字符串就是内容本身, # 用户在输入框里回一句"no"是在交稿,不是在驳回。 if isinstance(response, dict) and decision.rejected: - raise NodeError(ctx.node.id, f"人工驳回:{decision.note or '未说明原因'}") + raise NodeRejected.of(ctx, f"人工驳回:{decision.note or '未说明原因'}", kind=mode, decision=decision) note = decision.note if isinstance(response, dict) else "" # 裸字符串是内容,不是备注 result = {"value": decision.value, "note": note, "__decision__": "approved"} diff --git a/backend/app/engine/nodes/tools.py b/backend/app/engine/nodes/tools.py index 1b26352d..e3591d35 100644 --- a/backend/app/engine/nodes/tools.py +++ b/backend/app/engine/nodes/tools.py @@ -11,7 +11,7 @@ from langgraph.func import task from app.core.artifact_store import canonical_json, content_hash from app.engine.approval import read_decision -from app.engine.context import NodeContext, NodeError +from app.engine.context import NodeContext, NodeError, NodeRejected from app.engine.evidence import ledger_enabled, next_exec, query_entry_fields, schema_ledger_entry from app.engine.replay import ask, once from app.engine.state import GraphState @@ -127,8 +127,8 @@ async def run_tool(state: GraphState, ctx: NodeContext) -> dict[str, Any]: await once(ctx, EventType.HUMAN_RESOLVED, tool=name, approved=decision.approved, note=decision.note, actor=ctx.actor(), **({"always": True} if always else {})) if not decision.approved: - raise NodeError(ctx.node.id, f"工具 {name} 的调用已被驳回" - + (f":{decision.note}" if decision.note else "")) + raise NodeRejected.of(ctx, f"工具 {name} 的调用已被驳回" + (f":{decision.note}" if decision.note else ""), + kind="tool", decision=decision) limit = limit_of(tool, name, args) ctx.emit(EventType.TOOL_START, tool=name, args=args, **limit_fields(limit)) @@ -299,10 +299,11 @@ async def _run_write(state: GraphState, ctx: NodeContext, name: str, args: dict[ await once(ctx, EventType.HUMAN_RESOLVED, tool=name, approved=decision.approved, note=decision.note, actor=ctx.actor(), payload_hash=confirmed) if not decision.approved: - # 驳回是人的决定,不是故障:运行到这里结束(状态记失败,没有可走的出口),说法写成「未确认(被驳回)」, - # 实例页按待办的结论另说(apps/resolve.instance_states) - raise NodeError(ctx.node.id, f"「{ctx.node.title}」未确认(被驳回),没有发送" - + (f":{decision.note}" if decision.note else "")) + # 驳回是人的决定,不是故障:运行到这里结束(没有可走的出口),收成 cancelled + end_reason=rejected + # (NodeRejected),说法写成「未确认(被驳回)」,实例页、通知按同一个结论说(apps/resolve.instance_state) + raise NodeRejected.of(ctx, f"「{ctx.node.title}」未确认(被驳回),没有发送" + + (f":{decision.note}" if decision.note else ""), + kind="action_confirm", decision=decision) if confirmed != prep.payload_hash: # 动作确认不接受改参数:要改在上游的表单里改。确认之后上游的数据变了,看过的就不是要发的,拒发 raise NodeError(ctx.node.id, f"「{name}」确认时看到的内容和现在要发送的不一致(内容指纹对不上)," @@ -380,8 +381,8 @@ async def run_code(state: GraphState, ctx: NodeContext) -> dict[str, Any]: await once(ctx, EventType.HUMAN_RESOLVED, approved=decision.approved, note=decision.note, actor=ctx.actor()) if not decision.approved: - raise NodeError(ctx.node.id, "代码执行已被驳回" - + (f":{decision.note}" if decision.note else "")) + raise NodeRejected.of(ctx, "代码执行已被驳回" + (f":{decision.note}" if decision.note else ""), + kind="code", decision=decision) # 隔离档位:strict 要硬件级(microVM),fast 要低延迟(Seatbelt/bwrap), # 留空跟随整机默认。要的那档不可用时会安静退回默认,不阻断运行。 diff --git a/backend/app/engine/runner.py b/backend/app/engine/runner.py index ef22bd1e..7fcd0a62 100644 --- a/backend/app/engine/runner.py +++ b/backend/app/engine/runner.py @@ -23,7 +23,7 @@ from app.db.base import SessionLocal from app.db.models import Approval, Run, RunEvent, Workflow from app.engine.approval import read_decision from app.engine.compiler import compile_graph, initial_state -from app.engine.context import NodeError, RunContext +from app.engine.context import NodeError, NodeRejected, RunContext from app.engine.deadline import segment from app.engine.replay import PROTOCOL, PROTOCOL_KEY, protocol_of from app.engine.schema import GraphSpec, NodeType, loop_steps, topology_of, validate_graph @@ -42,9 +42,18 @@ _EPHEMERAL = {EventType.LLM_TOKEN, EventType.LLM_THINKING_DELTA} RUN_OPTIONS_KEY = "run_options" -#: 接着跑被拒的机器码:正式运行带图(F2),有结果未知、还没核实的写入 +#: 接着跑被拒的机器码:正式运行带图(F2),有结果未知、还没核实的写入,被人驳回了的 FORMAL_GRAPH_LOCKED = "formal_graph_locked" ACTION_UNRESOLVED = "action_unresolved" +RUN_REJECTED = "run_rejected" + +#: 运行为什么结束在 cancelled(Run.end_reason,docs/design/next-round/00-总方案.md X9) +END_REJECTED = "rejected" +END_STOPPED = "stopped" +#: 被驳回的运行不能接着跑:从断点重跑会读到同一个驳回答复、再停在同一处 +REJECTED_CONTINUE = "被驳回的运行不能接着跑:改好内容后重新发起" +#: 服务重启时还在排队的运行改成失败时写的话(run.error 和补发的 run.failed 是同一句) +QUEUED_AT_RESTART = "服务重启时这次运行仍在排队,没有可恢复的进度,请重新发起运行" class ContinueRefused(ValueError): @@ -94,6 +103,9 @@ class RunManager: stranded = list((await session.execute( select(Run.id, Run.last_seq).where(Run.status == "running") )).all()) + queued = list((await session.execute( + select(Run.id, Run.last_seq).where(Run.status == "queued") + )).all()) await session.execute( update(Run) .where(Run.status == "running") @@ -102,7 +114,7 @@ class RunManager: await session.execute( update(Run) .where(Run.status == "queued") - .values(status="failed", error="服务重启时这次运行仍在排队,没有可恢复的进度,请重新发起运行") + .values(status="failed", error=QUEUED_AT_RESTART) ) await session.commit() # 进程被强杀时连 server_shutdown 日志都来不及发,事件流停在半路。补上这一条, @@ -113,6 +125,11 @@ class RunManager: "level": "warn", "code": "server_shutdown", "message": "服务上次未正常关闭,这次运行已中断;可从断点继续运行", }) + # 排队中的改成了失败,也补一条 run.failed:以前只改状态,时间线里没有失败原因这一行(05-已知遗留 (h))。 + # 不写结束时间、不封存:它从没开始执行,同以前(定时通知按「没有开始时间」认出它,scheduler/webhook.py) + for run_id, last_seq in queued: + bus.set_seq(run_id, last_seq or 0) + await self._emit(run_id, EventType.RUN_FAILED, data={"error": QUEUED_AT_RESTART}) async def shutdown(self) -> None: self._closing = True @@ -505,8 +522,13 @@ class RunManager: Approval.run_id == run_id, Approval.status.in_(["pending", "answered"]) ).limit(1) )).first() is not None + if run.status == "cancelled" and run.end_reason == END_REJECTED: + raise ContinueRefused(REJECTED_CONTINUE, code=RUN_REJECTED) if not (run.status == "failed" or (run.status == "interrupted" and not has_pending)): raise ValueError(_cannot_continue(run.status, has_pending)) + if run.status == "failed" and await _rejected_before(session, run): + # 加 end_reason 之前被驳回的运行记的是 failed(封存不改写):接着跑会读到同一个驳回答复、再停一次 + raise ContinueRefused(REJECTED_CONTINUE, code=RUN_REJECTED) if graph is not None and run.run_class == "formal": raise ContinueRefused( "正式运行固定在发布的版本上,不能修改配置后继续运行。如需修改,请在画布上修改并发布新版本后重新发起", @@ -613,7 +635,7 @@ class RunManager: # 或者另一个取消先到了,这里就什么都不做 claimed = await session.execute( update(Run).where(Run.id == run_id, Run.status == "interrupted") - .values(status="cancelled", finished_at=now) + .values(status="cancelled", end_reason=END_STOPPED, finished_at=now) ) if claimed.rowcount != 1: await session.rollback() @@ -631,7 +653,7 @@ class RunManager: bus.set_seq(run_id, await _seq_floor(session, run_id, run.last_seq)) # 和状态同一个事务提交,理由同 _finalize event = self._event(run_id, EventType.RUN_CANCELLED, data={ - "timing": timing, "actor": actor, + "reason": END_STOPPED, "timing": timing, "actor": actor, "message": f"放弃了这次运行,{closed} 条待审批一并关闭" if closed else "放弃了这次挂起的运行", }) @@ -684,6 +706,8 @@ class RunManager: #: 终态事件(run.failed / run.cancelled)等 _finalize 算完耗时再发,timing 才带得上 terminal: tuple[EventType, dict[str, Any]] | None = None error_node: str | None = None + #: 收成 cancelled 的原因(Run.end_reason):rejected 人的驳回,stopped 手动停止 + end_reason: str | None = None #: 结果未知、挂起等核实的动作(ActionPending),收尾之后发 action.unknown、安排核实 pending_action: str | None = None @@ -796,7 +820,9 @@ class RunManager: else: status = "cancelled" error = "用户取消" - terminal = (EventType.RUN_CANCELLED, {"actor": self._stopped_by.pop(run_id, None)}) + end_reason = END_STOPPED + terminal = (EventType.RUN_CANCELLED, {"reason": END_STOPPED, + "actor": self._stopped_by.pop(run_id, None)}) raise except ActionPending as e: # 写外部系统的结果未知(app/process/actions.py):不在节点里等,运行收成挂起、不封存,等核实之后 @@ -829,6 +855,12 @@ class RunManager: "(AGENTLAB_MAX_GRAPH_STEPS)" ) terminal = (EventType.RUN_FAILED, {"error": error}) + except NodeRejected as e: + # 人的决定(驳回、超时驳回),不是故障:收成 cancelled,同手动停止走一样的收尾(关掉并行支路上的审批、 + # 回收沙箱、关流、封存、不许继续运行)。原话照旧记在 error 上,在哪一步、谁、结论、备注进 run.cancelled + status, end_reason = "cancelled", END_REJECTED + error, error_node = str(e), e.node_id + terminal = (EventType.RUN_CANCELLED, rejected_data(e, spec)) except Exception as e: # noqa: BLE001 status = "failed" error, terminal = _failure(e, spec) @@ -840,7 +872,7 @@ class RunManager: self._finalizing.add(run_id) try: await self._finalize(run_id, status, error, final_state, elapsed, - terminal=terminal, error_node=error_node) + terminal=terminal, error_node=error_node, end_reason=end_reason) finally: # 收尾被打断(关停时的取消、库连接出错)也得把自己摘掉。以前写在 # _finalize 后面,一被打断就漏掉,留下的任务让下一次 shutdown 的 @@ -944,6 +976,7 @@ class RunManager: *, terminal: tuple[EventType, dict[str, Any]] | None = None, error_node: str | None = None, + end_reason: str | None = None, ) -> None: usage = dict(state.get("usage") or {}) output = dict(state.get("output") or {}) @@ -1000,6 +1033,7 @@ class RunManager: run.output = output run.usage = usage run.error_node_id = error_node + run.end_reason = end_reason if status != "interrupted": run.finished_at = now if status in ("failed", "cancelled"): @@ -1024,11 +1058,12 @@ class RunManager: # 关停时挂起的也要让在线的客户端收尾:进程马上就没了,不说一声, # 界面会一直以为它还在跑 await bus.close(run_id) - if app_id and status in ("succeeded", "failed"): - # 应用实例结束、失败:按应用订阅推送(app/notify/service.py)。放到后台,接收端慢不拖着收尾 + if app_id and (status in ("succeeded", "failed") or end_reason == END_REJECTED): + # 应用实例结束、失败:按应用订阅推送(app/notify/service.py)。放到后台,接收端慢不拖着收尾。被驳回的走 + # 「实例结束」(next-round X6):发起人要知道结果,订阅「实例失败」的人盯的是故障;手动停止的不推,同以前 from app.notify.service import notify_run, spawn - spawn(notify_run(run_id, "run.finished" if status == "succeeded" else "run.failed")) + spawn(notify_run(run_id, "run.failed" if status == "failed" else "run.finished")) async def _seal(self, run_id: str) -> None: """对到此为止的全部事件算清单哈希,记下封到了哪一条。""" @@ -1269,6 +1304,22 @@ def _trust_event(tool_trust: dict[str, str] | None) -> Any: return "formal" if tool_trust.get(FORMAL_MARK) else snapshot_levels(tool_trust) +def rejected_data(exc: NodeRejected, spec: GraphSpec) -> dict[str, Any]: + """被驳回的 run.cancelled 带什么:原因(rejected)、在哪一步(节点 id 和图上的名字)、哪种关卡、结论(驳回 / 超时 + 驳回)、谁、备注。进封存:谁在哪一步驳回、为什么,事后查得到。""" + node = spec.node_map().get(exc.node_id) + return {"reason": END_REJECTED, "node_id": exc.node_id, "label": node.title if node else exc.node_id, + "kind": exc.kind, "decision": exc.decision, "actor": exc.actor, "note": exc.note} + + +async def _rejected_before(session: Any, run: Run) -> bool: + """加 end_reason 之前被人驳回、记成 failed 的运行:失败的那一步上最近一条处理完的待办是驳回(和实例页认驳回的是 + 同一份判断,apps/resolve.legacy_rejected)。""" + from app.apps.resolve import legacy_rejected + + return await legacy_rejected(session, run) + + def _failure(exc: BaseException, spec: GraphSpec) -> tuple[str, tuple[EventType, dict[str, Any]]]: """失败的首行报错(给人看)和 run.failed 的负载(带上定位和原始异常)。""" if isinstance(exc, NodeError): diff --git a/backend/app/notify/service.py b/backend/app/notify/service.py index a836574a..33b8ff78 100644 --- a/backend/app/notify/service.py +++ b/backend/app/notify/service.py @@ -201,22 +201,36 @@ async def notify_task(task_id: str, event: str, *, users: list[str]) -> int: async def notify_run(run_id: str, event: str) -> int: - """应用实例结束、失败(run.finished / run.failed)。只推应用发起的运行。""" + """应用实例结束、失败(run.finished / run.failed)。只推应用发起的运行。 + + 状态那一句取自实例页的说法(apps/resolve.instance_state),两边说同一句话:被驳回的(cancelled + end_reason=rejected) + 走「实例结束」、写「「付款」未确认(被驳回)」;走到「已退回」成果的写「已退回」,不再一律写「已完成」。ok 只在真的 + 办成了(code 是 succeeded)时为真。 + + 发起人是认证过的身份(run.origin.actor_verified)时,他自己订阅了这件事的个人渠道也收得到(「我发起的实例完成 / 失败」, + 04-对外集成与触达 B1)。以前这里给 _channels 的 users 是空的,个人渠道勾了这两项永远收不到。自报的署名不算:关登录 + 时谁都能填别人的名字。""" + from app.apps.resolve import _trails, instance_state + async with SessionLocal() as session: run = await session.get(Run, run_id) if run is None or not run.app_id: return 0 - channels = await _channels(session, event, run, []) + verified = bool((run.origin or {}).get("actor_verified")) and bool(run.started_by) + channels = await _channels(session, event, run, [run.started_by] if verified else []) if not channels: return 0 base = await _base_url(session) where = await _where(session, run) + state = instance_state(run, (await _trails(session, [run]))[run.id]) finished = event == "run.finished" - lines = [("状态", "已完成" if finished else "失败"), ("实例", f"#{run_id[:6]}")] - if not finished and run.error_node_id: + ok = state["code"] == "succeeded" + lines = [("状态", state["text"]), ("实例", f"#{run_id[:6]}")] + if not finished and run.error_node_id and state["code"] == "failed": lines.append(("失败节点", f"「{_label(run.graph, run.error_node_id)}」")) - message = {"event": event, "status": run.status, "ok": finished, - "title": f"{where or '应用'}的实例{'已完成' if finished else '失败'}", "lines": lines, + head = ("已完成" if ok else "已结束") if finished else ("已结束" if state["code"] == "cancelled" else "失败") + message = {"event": event, "status": run.status, "ok": ok, + "title": f"{where or '应用'}的实例{head}", "lines": lines, "links": [("查看实例", f"{base}/apps/{run.app_id}/i/{run_id}")] if base else [], "subject": {"run_id": run_id, "app_id": run.app_id}} return await _push(channels, message, subject_kind="run", subject_id=run_id, event=event, run_id=run_id, diff --git a/backend/app/plain.py b/backend/app/plain.py index 617afe92..6c38d3a4 100644 --- a/backend/app/plain.py +++ b/backend/app/plain.py @@ -181,6 +181,9 @@ PLAIN: dict[str, Plain] = { "plan.trial_failed": Plain( "最近一次试运行失败了", "用真数运行出了错,这样发布出去,每次出数都会是同一个错。", _Y, "回到试运行看出错的原因(可以交给助手修),修好后再用真数试一次。"), + "plan.trial_rejected": Plain( + "最近一次试运行被驳回、没有走完", "那是你(或审批人)的决定,不是出错;驳回之后的步骤没用真数看过。", _Y, + "要看完整结果,再试运行一次并批准;确认不需要的话可以直接发布。"), "plan.untrialed": Plain( "还没用真数试运行过", "没看过真数就发布,可能不是你要的结果。", _Y, "先试运行一次,看过结果再发布。"), "plan.trial_differs": Plain( diff --git a/backend/app/process/actions.py b/backend/app/process/actions.py index e75a269e..87ab7fca 100644 --- a/backend/app/process/actions.py +++ b/backend/app/process/actions.py @@ -572,17 +572,18 @@ async def _continue(run_id: str, actor: str | None) -> Any: return None -async def tick(now: datetime | None = None) -> int: - """调度循环每一拍:对登记了 lookup 的结果未知动作带退避地查询。返回查了几条。""" +async def tick(now: datetime | None = None) -> list[str]: + """调度循环每一拍:对登记了 lookup 的结果未知动作带退避地查询。交回这一拍查过的动作记录 id(同 tasks.tick)。""" now = now or utcnow() async with SessionLocal() as session: due = list((await session.execute(select(ActionRecord.id, ActionRecord.next_lookup_at).where( ActionRecord.state == "unknown", ActionRecord.next_lookup_at.is_not(None), ActionRecord.next_lookup_at <= now))).all()) - done = 0 + done: list[str] = [] for action_id, claimed_at in due: try: - done += await _lookup(action_id, claimed_at, now) + if await _lookup(action_id, claimed_at, now): + done.append(action_id) except Exception: # noqa: BLE001 - 一条出错不拖住这一拍的其他动作 logger.exception("动作 %s 的结果查询失败,下一拍再试", action_id) return done diff --git a/backend/app/process/tasks.py b/backend/app/process/tasks.py index 9f1dd240..6ec167af 100644 --- a/backend/app/process/tasks.py +++ b/backend/app/process/tasks.py @@ -761,8 +761,9 @@ async def _notify(task_id: str, event: str, users: list[str]) -> int: # -------------------------------------------------------------------------- -async def tick(now: datetime | None = None) -> int: - """调度循环每一拍:提醒、升级、超时驳回(永不自动批准)。返回处理的条数。试运行的待办不处理(只显示时限)。""" +async def tick(now: datetime | None = None) -> list[str]: + """调度循环每一拍:提醒、升级、超时驳回(永不自动批准)。交回这一拍处理过的待办 id(生效可查:办了哪几条, + 不只是几条;测试按自己造的那条断言,不受同一个库里别的待办影响)。试运行的待办不处理(只显示时限)。""" now = now or utcnow() action = func.json_extract(Task.on_timeout, "$.action") trial = select(Run.id).where(Run.origin_kind == TRIAL_ORIGIN) @@ -773,10 +774,11 @@ async def tick(now: datetime | None = None) -> int: and_(Task.due_at.is_not(None), Task.due_at <= now, action == "reject"), and_(Task.due_at.is_not(None), Task.due_at <= now, action == "escalate", Task.escalation_level == 0), )).order_by(Task.due_at))).scalars()) - done = 0 + done: list[str] = [] for task_id in due: try: - done += await _on_timeout(task_id, now) + if await _on_timeout(task_id, now): + done.append(task_id) except Exception: # noqa: BLE001 - 一条出错不拖住这一拍的其他待办 logger.exception("待办 %s 的超时处理失败,下一拍再试", task_id) return done @@ -838,7 +840,9 @@ async def _on_timeout(task_id: str, now: datetime) -> int: from app.engine.runner import run_manager try: - await run_manager.resume(run_id, {"approved": False, "note": EXPIRED_NOTE}, approval_id=approval_id) + # expired:节点据此把这次驳回记成超时驳回(engine/approval.Decision.expired → NodeRejected.decision) + await run_manager.resume(run_id, {"approved": False, "note": EXPIRED_NOTE, "expired": True}, + approval_id=approval_id) except (ValueError, KeyError): logger.warning("待办 %s 超时驳回时运行已无法恢复", task_id) async with SessionLocal() as session: diff --git a/backend/app/scheduler/loop.py b/backend/app/scheduler/loop.py index 243f5f97..ee6373c2 100644 --- a/backend/app/scheduler/loop.py +++ b/backend/app/scheduler/loop.py @@ -130,7 +130,9 @@ async def process_due(now: datetime | None = None) -> None: for name, step in (("待办的时限处理", tasks.tick), ("写入结果的查询", actions.tick), ("没有提交的上传文件清理", files_store.cleanup_tick)): try: - await step(now) + done = await step(now) + if isinstance(done, list) and done: + logger.info("%s:这一拍处理了 %d 条", name, len(done)) except Exception: # noqa: BLE001 logger.exception("%s这一拍出错,下一拍照常进行", name) diff --git a/backend/app/scheduler/webhook.py b/backend/app/scheduler/webhook.py index 4a19e1c6..2062db3c 100644 --- a/backend/app/scheduler/webhook.py +++ b/backend/app/scheduler/webhook.py @@ -5,7 +5,8 @@ 再导出,调用方和测试不用改;_transport、RETRY_DELAYS 仍是这个模块自己的(测试照旧替换它们),send 每次调用时读。 通知里只有任务名、状态、出具档位、缺口条数、运行链接、封存哈希(界面上叫清单哈希)前 12 位和起止时间,失败时 -另写失败节点的名字和下一步(不许诺能继续运行);运行成功结束后和同一任务上一次成功的运行的对比(auto_compare)只写 +另写失败节点的名字和下一步(不许诺能继续运行);被人驳回的那一期按「运行结束」推「未出具(被驳回)」、写驳回的那一步, +手动停止的不推;运行成功结束后和同一任务上一次成功的运行的对比(auto_compare)只写 各类指标的个数和对比页链接。不放报告里的具体数字,也不放失败时的原始报错——报错里可能带着查询到的数据,Webhook 推去的群不一定有权看。 """ @@ -71,6 +72,11 @@ NEXT_AS_USUAL = "下一次触发会照常发起" #: 从没开始执行、任务又不在排期上(停用了):原因刚写过,记录页上也没有更多可看的,下一步是重新发起。「立即运行」 #: 是定时任务页上那个按钮的名字,停用的任务也能点 RERUN_ACTION = "如需重新发起,请在 AgentLab 的「定时任务」中点击「立即运行」" +#: 被人驳回的那一期(cancelled + end_reason=rejected,next-round X9):是人的决定,不是故障。按「运行结束」推(订阅 +#: 「运行失败」的人盯的是故障),标题、状态照实写被驳回;驳回的备注不写(可能带着数),驳回的是哪一步写节点名 +REJECTED_HEADLINE = "未出具(被驳回)" +REJECTED_STATUS = "被驳回" +REJECTED_ACTION = "驳回的说明请在 AgentLab 的「记录」中查看" # -------------------------------------------------------------------------- @@ -110,12 +116,18 @@ class Notice: #: 挂起的原因是写外部系统的结果未知(等核实,核实后自动接着跑),不是服务重启。只在 suspended 时有意义 action_pending: bool = False error_node: str | None = None + #: 运行结束在 cancelled 的原因(Run.end_reason):rejected 时按「未出具(被驳回)」说,error_node 是驳回的那一步 + end_reason: str | None = None next_fire: bool = False compare: dict[str, Any] | None = None compare_url: str | None = None fmt: str | None = None signed: bool = False + @property + def rejected(self) -> bool: + return self.status == "cancelled" and self.end_reason == "rejected" + @property def never_started(self) -> bool: """失败的运行从没开始执行(没有开始时间):服务重启时还在排队、被改成失败的就是这种,没有断点可续。""" @@ -142,6 +154,8 @@ def message(notice: Notice) -> tuple[str, list[tuple[str, str]], str | None]: headline = "未能发起运行" elif notice.suspended: headline = "运行已挂起" + elif notice.rejected: + headline = REJECTED_HEADLINE else: headline = _HEADLINE.get(notice.event, notice.event) title = f"定时任务「{notice.schedule_name}」{headline}" @@ -163,6 +177,10 @@ def message(notice: Notice) -> tuple[str, list[tuple[str, str]], str | None]: lines += [("原因", ACTION_PENDING_REASON), ("处理", ACTION_PENDING_ACTION)] elif notice.suspended: lines += [("原因", SUSPENDED_REASON), ("处理", SUSPENDED_ACTION)] + elif notice.rejected: + if notice.error_node: + lines.append(("驳回节点", f"「{notice.error_node}」")) + lines.append(("处理", f"{REJECTED_ACTION};{NEXT_AS_USUAL}" if notice.next_fire else REJECTED_ACTION)) elif notice.status == "failed": # 不写原始报错(可能带着查询到的数据),写失败节点的名字和下一步 if notice.error_node: @@ -219,6 +237,8 @@ def _status_label(notice: Notice) -> str: if notice.pending_approvals: return "等待审批" return "已挂起,等待核实" if notice.action_pending else "已挂起,可继续运行" + if notice.rejected: + return REJECTED_STATUS return _STATUS_LABEL.get(notice.status or "", notice.status or "") @@ -239,7 +259,9 @@ def _links(notice: Notice) -> list[tuple[str, str]]: def _message(notice: Notice) -> core.Message: title, lines, _ = message(notice) - return core.Message(title=title, lines=lines, links=_links(notice), ok=notice.event in ("finished", "test")) + # 被驳回的那一期按 finished 推,但没出具:不画成成功的颜色 + return core.Message(title=title, lines=lines, links=_links(notice), + ok=notice.event in ("finished", "test") and not notice.rejected) def _text(notice: Notice) -> str: @@ -332,10 +354,17 @@ async def _record(schedule_id: str, fire_id: str | None, run_id: str | None, eve #: 运行到了这些状态才可能要推送 _WATCHED = ("succeeded", "failed", "cancelled", "interrupted") -#: 结束的运行 → 事件。cancelled 是有人手动停的,不推。interrupted 分停在审批和挂起两种,见 _state_of +#: 结束的运行 → 事件。cancelled 是有人手动停的,不推;被驳回的(end_reason=rejected)按 finished 推,见 _event_of。 +#: interrupted 分停在审批和挂起两种,见 _state_of _EVENT_OF = {"succeeded": "finished", "failed": "failed", "cancelled": None} +def _event_of(run: Run) -> str | None: + if run.status == "cancelled" and run.end_reason == "rejected": + return "finished" + return _EVENT_OF[run.status] + + @dataclass class _Job: fmt: str @@ -469,7 +498,7 @@ async def _state_of(session: Any, fire: ScheduleFire, run: Run | None, now: date since = run.finished_at or run.updated_at or fire.created_at if not sealed and (now - service.as_utc(since)).total_seconds() < SEAL_WAIT_S: return _Seen(None) - return _Seen(run.status, _EVENT_OF[run.status], sealed=sealed) + return _Seen(run.status, _event_of(run), sealed=sealed) if run.status != "interrupted": return _Seen(None) counts = dict((await session.execute( @@ -516,7 +545,9 @@ async def _notice(session: Any, schedule: Schedule, fire: ScheduleFire, run: Run pending_approvals=pending or None, suspended=run is not None and run.status == "interrupted" and not pending, action_pending=seen.action_pending, - error_node=_failed_node(run) if run is not None and run.status == "failed" else None, + error_node=_failed_node(run) if run is not None and (run.status == "failed" or run.end_reason == "rejected") + else None, + end_reason=run.end_reason if run is not None else None, next_fire=bool(schedule.enabled and schedule.next_run_at is not None), ) diff --git a/backend/tests/test_app_instance_page.py b/backend/tests/test_app_instance_page.py index 764273a0..ce502afb 100644 --- a/backend/tests/test_app_instance_page.py +++ b/backend/tests/test_app_instance_page.py @@ -258,7 +258,7 @@ async def test_after_escalation_the_rule_is_one_director_not_zero_of_three(): run_id = await _start(app_id, "报销审批·钱七") task = await _task(run_id) await tasks.respond(task.id, user(a), decision="approve") - assert await tasks.tick(utcnow() + timedelta(hours=2)) >= 1 + assert task.id in await tasks.tick(utcnow() + timedelta(hours=2)) detail = await tasks.detail(task.id, user(boss)) p = detail["progress"] # 升级加派的那一份不算进会签人数:还是 1/2,另写升级给了谁、他一人说了算 diff --git a/backend/tests/test_approval.py b/backend/tests/test_approval.py index f3d005a6..0f45888c 100644 --- a/backend/tests/test_approval.py +++ b/backend/tests/test_approval.py @@ -138,9 +138,10 @@ async def test_tool_node_takes_a_refusal_as_a_refusal(): run_id = await _start(_calc_graph()) await _wait(run_id, ("interrupted",)) await run_manager.resume(run_id, "拒绝") - run = await _wait(run_id, ("failed", "succeeded")) + run = await _wait(run_id, ("failed", "succeeded", "cancelled")) - assert run.status == "failed" and "驳回" in (run.error or "") + # 驳回是人的决定,运行收成 cancelled + end_reason=rejected(05-已知遗留 (a)) + assert (run.status, run.end_reason) == ("cancelled", "rejected") and "驳回" in (run.error or "") assert not [e for e in await _events(run_id) if e.type == "tool.start"], "被拒绝的工具执行了" @@ -208,8 +209,8 @@ async def test_a_refused_write_leaves_the_data_alone(writable): run_id = await _start(_query_graph(tool, "WITH k AS (SELECT 1) DELETE FROM t")) await _wait(run_id, ("interrupted",)) await run_manager.resume(run_id, "拒绝") - run = await _wait(run_id, ("failed", "succeeded")) - assert run.status == "failed" + run = await _wait(run_id, ("failed", "succeeded", "cancelled")) + assert (run.status, run.end_reason) == ("cancelled", "rejected") assert count() == 5 @@ -392,5 +393,5 @@ async def test_parallel_rejection_with_content_is_still_a_rejection(): approval_id=pending["h1"].id) await run_manager.resume(run_id, {"approved": False, "note": "不行", "value": "乙"}, approval_id=pending["h2"].id) - run = await _wait(run_id, ("failed", "succeeded")) - assert run.status == "failed" and "驳回" in (run.error or ""), run.error + run = await _wait(run_id, ("failed", "succeeded", "cancelled")) + assert (run.status, run.end_reason) == ("cancelled", "rejected") and "驳回" in (run.error or ""), run.error diff --git a/backend/tests/test_blueprint_trial.py b/backend/tests/test_blueprint_trial.py index d5ad468b..957a8549 100644 --- a/backend/tests/test_blueprint_trial.py +++ b/backend/tests/test_blueprint_trial.py @@ -157,7 +157,7 @@ async def test_process_trial_assigns_every_step_to_the_trial_runner_and_never_es assert task.due_at is not None and task.on_timeout["action"] == "escalate" # 时限到了:调度不升级试运行的待办(同一个库里别的测试的待办照常处理,只看这一条) - await tasks.tick(utcnow() + timedelta(hours=2)) + assert task.id not in await tasks.tick(utcnow() + timedelta(hours=2)) async with SessionLocal() as session: task = await session.get(Task, task.id) count = len(list((await session.execute(select(TaskAssignment).where( diff --git a/backend/tests/test_dual_check_node.py b/backend/tests/test_dual_check_node.py index 436cd9bc..3ca9af15 100644 --- a/backend/tests/test_dual_check_node.py +++ b/backend/tests/test_dual_check_node.py @@ -519,15 +519,19 @@ async def test_a_missing_choice_is_recorded_as_no_choice(monkeypatch, shop, clie assert not logs(evs, "dual_check_catalog") and await notes_of(source_id) is None -async def test_rejecting_the_approval_fails_the_node(monkeypatch, shop, client): +async def test_rejecting_the_approval_ends_the_run_as_rejected(monkeypatch, shop, client): + """明确驳回裁决:人的决定,不是故障(05-已知遗留 (a))。运行收成 cancelled + end_reason=rejected,原话照旧。""" name, _ = shop Blind(monkeypatch, name, {"销售额": (RIGHT, "gmv")}) run = await start(graph(name, BY_ORDER_DATE)) approval = await pending(run.id) res = await client.post(f"/api/approvals/{approval.id}/decide", json={"approved": False, "note": "口径要重新对"}) assert res.status_code == 200, res.text - done = await wait(run.id, ("succeeded", "failed")) - assert done.status == "failed" and "人工驳回" in (done.error or "") and "口径要重新对" in done.error + done = await wait(run.id, ("succeeded", "failed", "cancelled")) + assert (done.status, done.end_reason) == ("cancelled", "rejected"), done.error + assert "人工驳回" in (done.error or "") and "口径要重新对" in done.error + [ended] = [e for e in await events(run.id) if e.type == "run.cancelled"] + assert (ended.data["kind"], ended.data["note"]) == ("dual_check", "口径要重新对") async def test_on_reject_fail_stops_the_run(monkeypatch, shop, client): diff --git a/backend/tests/test_notify_run.py b/backend/tests/test_notify_run.py new file mode 100644 index 00000000..a57e32ff --- /dev/null +++ b/backend/tests/test_notify_run.py @@ -0,0 +1,199 @@ +"""应用实例结束的通知(notify/service.notify_run):说法取自实例页,个人渠道按「我发起的实例」推。 + +- 被驳回的实例(cancelled + end_reason=rejected)走「实例结束」,状态写实例页那一句(「「review」被驳回」),不推 + 「实例失败」(05-已知遗留 (a)、总方案 X6); +- 走到「已退回」那种结局的运行,实例页写成果节点的标题、记成人的决定,推送以前却写「已完成」:现在两边一致; +- 个人渠道勾了「实例完成 / 失败」以前永远收不到(_channels 拿到的 users 是空的):发起人是认证过的身份时,他自己的 + 个人渠道收得到;自报的署名不算(04-对外集成与触达 B1)。 +""" +from __future__ import annotations + +import asyncio +import json +import uuid + +import httpx +import pytest +from sqlalchemy import select + +from app.core.crypto import encrypt +from app.db.base import SessionLocal +from app.db.models import App, AppVersion, NotifyChannel, NotifyDelivery, Run, User +from app.engine.runner import run_manager +from app.notify import core, service + +URL = "https://hooks.example.com/agentlab/run-3b8e1f" + + +@pytest.fixture(autouse=True) +async def engine_up(): + await run_manager.setup() + yield + await service.drain() + await run_manager.shutdown() + + +@pytest.fixture +def received(monkeypatch): + got: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + got.append(request) + return httpx.Response(200, json={"ok": True}) + + monkeypatch.setattr(core, "_transport", httpx.MockTransport(handler)) + monkeypatch.setattr(core, "RETRY_DELAYS", (0, 0, 0)) + # 个人渠道发送前要解析域名、核对不是内网:测试里不连外网 + async def public(url: str) -> None: + return None + + monkeypatch.setattr(service, "personal_url_problem", public) + return got + + +def node(nid, ntype, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}} + + +def flow(**human): + nodes = [node("start", "input", fields=[{"name": "q"}]), + node("review", "human", mode="approve", title="审一下", **human), + node("ok", "output", fields=[{"name": "结论", "value": "通过"}]), + node("no", "output", fields=[{"name": "结论", "value": "退回"}])] + edges = [{"source": "start", "target": "review"}, + {"source": "review", "target": "ok", "sourceHandle": "approved"}, + {"source": "review", "target": "no", "sourceHandle": "rejected"}] + return {"nodes": nodes, "edges": edges} + + +async def _app(events: list[str]) -> tuple[str, str]: + async with SessionLocal() as session: + channel = NotifyChannel(name=f"群-{uuid.uuid4().hex[:6]}", format="generic", url=encrypt(URL), events=[]) + session.add(channel) + app = App(name=f"报销-{uuid.uuid4().hex[:6]}", published_version=1) + session.add(app) + await session.flush() + session.add(AppVersion(app_id=app.id, version=1, spec_hash="h" * 64, bindings={}, + spec={"notify": [{"channel": channel.id, "events": events}]})) + await session.commit() + return app.id, channel.id + + +async def _wait(run_id: str, statuses: tuple[str, ...]) -> Run: + for _ in range(400): + async with SessionLocal() as session: + row = await session.get(Run, run_id) + if row is not None and row.status in statuses: + return row + await asyncio.sleep(0.05) + raise AssertionError(f"运行 {run_id} 没有进入 {statuses}") + + +async def _decide(run_id: str, approved: bool) -> Run: + """经待办服务回复(同界面上点同意 / 驳回):实例页认结局靠待办的结论。""" + from app.api.deps import Principal + from app.db.models import Task + from app.process import tasks + + await _wait(run_id, ("interrupted",)) + await run_manager.wait_idle(run_id) + for _ in range(100): + async with SessionLocal() as session: + task = (await session.execute(select(Task).where(Task.run_id == run_id, Task.status == "pending")))\ + .scalars().first() + if task is not None: + break + await asyncio.sleep(0.05) + await tasks.respond(task.id, Principal(username=None, role="admin", mode="off"), + decision="approve" if approved else "reject", note="金额 45678 不对") + final = await _wait(run_id, ("succeeded", "failed", "cancelled")) + await run_manager.wait_idle(run_id) + await service.drain() + return final + + +async def _deliveries(channel_id: str) -> list[tuple[str, str]]: + async with SessionLocal() as session: + rows = (await session.execute(select(NotifyDelivery).where(NotifyDelivery.channel_id == channel_id))).scalars() + return [(r.event, r.status) for r in rows] + + +async def test_a_rejected_instance_is_pushed_as_finished_in_the_instance_page_words(received): + app_id, channel_id = await _app(["run.finished", "run.failed"]) + run = await run_manager.start(graph=flow(stop_on_reject=True), input_payload={"q": "x"}, app_id=app_id, + app_version=1) + final = await _decide(run.id, approved=False) + assert (final.status, final.end_reason) == ("cancelled", "rejected") + assert await _deliveries(channel_id) == [("run.finished", "ok")], "驳回不进「实例失败」" + [request] = received + body = json.loads(request.content) + assert body["event"] == "run.finished" and body["title"].endswith("的实例已结束") + assert "状态:「review」被驳回" in body["text"] and "失败" not in body["text"] + assert "45678" not in request.content.decode(), "驳回的备注不进消息(可能带着数)" + + +async def test_a_manual_stop_is_not_pushed(received): + app_id, channel_id = await _app(["run.finished", "run.failed"]) + run = await run_manager.start(graph=flow(), input_payload={"q": "x"}, app_id=app_id, app_version=1) + await _wait(run.id, ("interrupted",)) + await run_manager.wait_idle(run.id) + assert await run_manager.cancel(run.id) == "cancelled" + await service.drain() + assert await _deliveries(channel_id) == [] and received == [] + + +async def test_reaching_the_returned_outcome_no_longer_says_completed(received): + app_id, channel_id = await _app(["run.finished"]) + run = await run_manager.start(graph=flow(), input_payload={"q": "x"}, app_id=app_id, app_version=1) + final = await _decide(run.id, approved=False) + assert final.status == "succeeded" and final.output.get("结论") == "退回" + [request] = received + body = json.loads(request.content) + # 实例页写走到的成果节点(no),记成人的决定:推送也这么写,不再说「已完成」 + assert "状态:no" in body["text"] and "已完成" not in body["title"] and body["title"].endswith("的实例已结束") + + +async def _personal(username: str, events: list[str]) -> str: + async with SessionLocal() as session: + user = (await session.execute(select(User).where(User.username == username))).scalar_one_or_none() + if user is None: + user = User(username=username, role="user") + session.add(user) + await session.flush() + channel = NotifyChannel(name="我的", scope="user", owner_id=user.id, format="generic", url=encrypt(URL), + events=events) + session.add(channel) + await session.commit() + return channel.id + + +async def _finished_instance(app_id: str, *, started_by: str, verified: bool) -> str: + async with SessionLocal() as session: + run = Run(workflow_name="报销", status="succeeded", graph=flow(), input={}, app_id=app_id, app_version=1, + started_by=started_by, origin_kind="app_entry", + origin={"kind": "app_entry", "actor": started_by, "actor_verified": verified}) + session.add(run) + await session.commit() + return run.id + + +async def test_the_initiator_s_personal_channel_receives_my_instance_finished(received): + """B1:个人渠道订阅了 run.finished,发起人自己发起的实例结束了收得到;应用自己没订阅也一样。""" + app_id, _ = await _app([]) + name = f"uma{uuid.uuid4().hex[:6]}" + mine = await _personal(name, ["run.finished"]) + run_id = await _finished_instance(app_id, started_by=name, verified=True) + assert await service.notify_run(run_id, "run.finished") == 1 + assert await _deliveries(mine) == [("run.finished", "ok")] + body = json.loads(received[0].content) + assert body["title"].endswith("的实例已完成") and "状态:已完成" in body["text"] + + +async def test_a_self_declared_initiator_does_not_reach_a_personal_channel(received): + """关登录时署名是自己填的,谁都能填别人的名字:不按它推给那个人的个人渠道。""" + app_id, _ = await _app([]) + name = f"vic{uuid.uuid4().hex[:6]}" + mine = await _personal(name, ["run.finished"]) + run_id = await _finished_instance(app_id, started_by=name, verified=False) + assert await service.notify_run(run_id, "run.finished") == 0 + assert await _deliveries(mine) == [] and received == [] diff --git a/backend/tests/test_process_actions.py b/backend/tests/test_process_actions.py index f11ae634..b0d0e992 100644 --- a/backend/tests/test_process_actions.py +++ b/backend/tests/test_process_actions.py @@ -173,8 +173,8 @@ async def test_unknown_result_with_lookup_is_reconciled_automatically_and_sent_o await run_manager.continue_failed(run_id) assert refused.value.code == "action_unresolved" - assert await actions.tick(utcnow()) == 0 # 还没到第一次查询的时候 - assert await actions.tick(utcnow() + timedelta(minutes=2)) == 1 + assert rec.id not in await actions.tick(utcnow()) # 还没到第一次查询的时候 + assert rec.id in await actions.tick(utcnow() + timedelta(minutes=2)) final = await _wait(run_id, ("succeeded", "failed")) assert final.status == "succeeded", final.error assert len(erp.writes()) == 1 and len(erp.lookups) == 1 @@ -410,7 +410,7 @@ async def test_a_changed_params_schema_does_not_fail_a_write_that_already_went_o with pytest.raises(actions.ActionPending): await actions.prepare(run_id=run_id, node_id="pay", exec_no=1, capability=caps[PAY], args=args) - assert await actions.tick(utcnow() + timedelta(minutes=2)) == 1 + assert rec.id in await actions.tick(utcnow() + timedelta(minutes=2)) final = await _wait(run_id, ("succeeded", "failed")) assert final.status == "succeeded", final.error assert len(erp.writes()) == 1 @@ -457,9 +457,15 @@ async def test_a_rejected_confirmation_reads_as_a_decision_not_a_fault(erp, capa run_id = await _formal(pay_graph()) task = await _task(run_id, "action_confirm") await tasks.respond(task.id, ANN, decision="reject", note="金额不对") - final = await _wait(run_id, ("succeeded", "failed")) - # 引擎里驳回就是节点失败、运行结束(没有可走的出口);说法写成未确认(被驳回),实例页按人的决定说 - assert final.status == "failed" and final.error.startswith("「pay」未确认(被驳回),没有发送:金额不对") + final = await _wait(run_id, ("succeeded", "failed", "cancelled")) + # 驳回是人的决定,不是故障(05-已知遗留 (a)):运行收成 cancelled + end_reason=rejected,原话照旧写未确认(被驳回), + # run.cancelled 带上在哪一步、谁、备注;实例页按人的决定说 + assert (final.status, final.end_reason) == ("cancelled", "rejected") + assert final.error.startswith("「pay」未确认(被驳回),没有发送:金额不对") + [ended] = await _events(run_id, "run.cancelled") + assert {k: ended.data[k] for k in ("reason", "node_id", "kind", "decision", "note")} == { + "reason": "rejected", "node_id": "pay", "kind": "action_confirm", "decision": "reject", "note": "金额不对"} + assert not await _events(run_id, "run.failed") view = await _instance_view(run_id) assert (view["hdr"]["status_code"], view["hdr"]["status_text"]) == ("cancelled", "「pay」未确认(被驳回)") step = next(s for s in view["tl"]["steps"] if s["node"] == "pay") @@ -481,7 +487,7 @@ async def test_unknown_then_reconciled_reads_as_waiting_then_a_footnote(erp, cap [row] = view["act"]["records"] assert row["state"] == "unknown" and "error" not in row and "ReadTimeout" in row["note"] - assert await actions.tick(utcnow() + timedelta(minutes=2)) == 1 + assert row["id"] in await actions.tick(utcnow() + timedelta(minutes=2)) final = await _wait(run_id, ("succeeded", "failed")) assert final.status == "succeeded", final.error view = await _instance_view(run_id) diff --git a/backend/tests/test_process_tasks.py b/backend/tests/test_process_tasks.py index 3bb85db9..b1f875fc 100644 --- a/backend/tests/test_process_tasks.py +++ b/backend/tests/test_process_tasks.py @@ -271,7 +271,24 @@ async def test_login_on_only_assignees_can_respond(): # -------------------------------------------------------------------------- -async def test_escalation_adds_the_director_whose_answer_decides(): +@pytest.fixture +async def stray_due_task(): + """别的测试留下的、早就到期、可升级的待办(-n 8 下同一个库里常有)。tick 扫全库,按条数断言的测试会被它带偏: + 这里先放一条,测试只断言自己造的那条在不在 tick 交回的 id 里(05-已知遗留 (f))。""" + async with SessionLocal() as session: + run = Run(workflow_name="别的测试留下的", status="interrupted", graph={}, input={}) + session.add(run) + await session.flush() + task = Task(run_id=run.id, node_id="x", kind="approve", title="残留", due_at=utcnow() - timedelta(days=1), + on_timeout={"action": "escalate", "to_slot": "nobody"}) + session.add(task) + await session.commit() + yield task.id + await session.execute(delete(Run).where(Run.id == run.id)) + await session.commit() + + +async def test_escalation_adds_the_director_whose_answer_decides(stray_due_task): a, b, g, boss = f"a{_u()}", f"b{_u()}", f"finance{_u()}", f"boss{_u()}" await _people(a, b, group=g) run_id = await _app_run( @@ -280,9 +297,9 @@ async def test_escalation_adds_the_director_whose_answer_decides(): {"finance": [{"kind": "group", "name": g}], "director": [{"kind": "user", "name": boss}]}) task = await _task(run_id) assert task.due_at is not None - assert await tasks.tick(utcnow()) == 0 # 还没到时限 - assert await tasks.tick(utcnow() + timedelta(hours=2)) == 1 - assert await tasks.tick(utcnow() + timedelta(hours=3)) == 0 # 只升一次 + assert task.id not in await tasks.tick(utcnow()) # 还没到时限 + assert task.id in await tasks.tick(utcnow() + timedelta(hours=2)) + assert task.id not in await tasks.tick(utcnow() + timedelta(hours=3)) # 只升一次 detail = await tasks.detail(task.id, off(None)) assert detail["escalation_level"] == 1 and detail["status"] == "pending" assert ("user", boss, "escalation") in [(x["principal_kind"], x["principal"], x["source"]) @@ -293,13 +310,13 @@ async def test_escalation_adds_the_director_whose_answer_decides(): assert (await _wait(run_id, ("succeeded", "failed"))).output.get("结论") == "通过" -async def test_timeout_reject_never_approves(): +async def test_timeout_reject_never_approves(stray_due_task): run = await run_manager.start( graph=approval_flow(due="PT30M", on_timeout={"action": "reject"}), input_payload={"q": "x"}) await _wait(run.id, ("interrupted",)) await run_manager.wait_idle(run.id) task = await _task(run.id) - assert await tasks.tick(utcnow() + timedelta(hours=1)) == 1 + assert task.id in await tasks.tick(utcnow() + timedelta(hours=1)) final = await _wait(run.id, ("succeeded", "failed")) assert final.output.get("结论") == "退回" assert len(await _events(run.id, "task.expired")) == 1 @@ -308,16 +325,16 @@ async def test_timeout_reject_never_approves(): assert row.status == "resolved" and row.outcome["decision"] == "expired_reject" -async def test_remind_repeats_every_interval(): +async def test_remind_repeats_every_interval(stray_due_task): run = await run_manager.start( graph=approval_flow(due="PT1H", on_timeout={"action": "remind", "every": "PT1H"}), input_payload={"q": "x"}) await _wait(run.id, ("interrupted",)) await run_manager.wait_idle(run.id) task = await _task(run.id) later = utcnow() + timedelta(hours=1, minutes=1) - assert await tasks.tick(later) == 1 - assert await tasks.tick(later) == 0 - assert await tasks.tick(later + timedelta(hours=1)) == 1 + assert task.id in await tasks.tick(later) + assert task.id not in await tasks.tick(later) + assert task.id in await tasks.tick(later + timedelta(hours=1)) assert len(await _events(run.id, "task.reminded")) == 2 async with SessionLocal() as session: assert (await session.get(Task, task.id)).status == "pending" diff --git a/backend/tests/test_publish_plan.py b/backend/tests/test_publish_plan.py index e38b8d97..00afa3ea 100644 --- a/backend/tests/test_publish_plan.py +++ b/backend/tests/test_publish_plan.py @@ -186,6 +186,30 @@ async def test_a_failed_trial_blocks_the_plan(client, tmp_path): assert "plan.untrialed" not in codes +async def test_a_rejected_trial_is_a_warning_not_a_failure(client, tmp_path): + """最近一次试运行在某一步被驳回(cancelled + end_reason=rejected):那是人的决定,不说成「试运行失败了」、不挡发布, + 照实提醒没走完(05-已知遗留 (a))。""" + from sqlalchemy import update + + bp = await _board_from_run(client, tmp_path) + async with SessionLocal() as session: + trial = await session.get(Run, bp["last_trial_run_id"]) + node_id = trial.graph["nodes"][-1]["id"] + label = trial.graph["nodes"][-1]["data"]["label"] + await session.execute(update(Run).where(Run.id == trial.id).values( + status="cancelled", end_reason="rejected", error_node_id=node_id, error="人工驳回:不对")) + await session.commit() + plan = (await client.post(f"/api/blueprints/{bp['id']}/publish-plan", json={"level": "published"})).json() + codes = {p["code"]: p for p in plan["problems"]} + assert "plan.trial_failed" not in codes and "plan.untrialed" not in codes + rejected = codes["plan.trial_rejected"] + assert rejected["level"] == "warning" and rejected["message"] == f"最近一次试运行在「{label}」被驳回、没有走完" + assert rejected["plain"]["title"] == "最近一次试运行被驳回、没有走完" + assert plan["ok"] is True, [p["code"] for p in plan["problems"] if p["level"] == "error"] + summary = {x["step"]: x["text"] for x in plan["summary"]} + assert "被驳回、没有走完" in summary["publish_workflow"] + + async def test_a_process_with_an_unbound_step_stops_at_preflight(client): wf_id = await s.published_workflow(s.expense_graph(), f"报销-{uuid.uuid4().hex[:4]}", description="报销审批") bp = (await client.post("/api/blueprints", json={"source": {"kind": "workflow_version", "id": wf_id}, diff --git a/backend/tests/test_records.py b/backend/tests/test_records.py index 88f435aa..c051c126 100644 --- a/backend/tests/test_records.py +++ b/backend/tests/test_records.py @@ -461,7 +461,7 @@ async def test_unknown_result_is_found_by_the_lookup_and_not_written_again(engin if (await session.get(ActionRecord, rec.id)).next_lookup_at is not None: break await asyncio.sleep(0.05) - assert await actions.tick(utcnow() + timedelta(minutes=5)) == 1 + assert rec.id in await actions.tick(utcnow() + timedelta(minutes=5)) done = await wait(run.id, ("succeeded", "failed")) assert done.status == "succeeded", done.error async with SessionLocal() as session: diff --git a/backend/tests/test_run_rejected.py b/backend/tests/test_run_rejected.py new file mode 100644 index 00000000..520d2d4f --- /dev/null +++ b/backend/tests/test_run_rejected.py @@ -0,0 +1,214 @@ +"""驳回 ≠ 失败(docs/design/next-round/05-已知遗留.md (a)、(h);总方案 X9)。 + +人的驳回(工具调用审批、代码执行审批、审批设了驳回即终止、填写 / 编辑被驳回、动作确认、双盲裁决)以前在引擎里抛普通的 +NodeError,运行记成 failed:通知推「实例失败」,定时推「运行失败」,记录页给「继续运行」——接着跑只会读到同一个驳回答复、 +再停一次。现在统一抛 NodeRejected,runner 收成 cancelled + end_reason=rejected,run.cancelled 带上在哪一步、哪种关卡、 +结论、谁、备注;原话(Run.error)一字不变。手动停止记 end_reason=stopped。被驳回的不能接着跑,存量的 failed+驳回也不能。 + +动作确认、双盲裁决的驳回在 test_process_actions.py、test_dual_check_node.py 里(要假的外部系统、口径卡)。 +""" +from __future__ import annotations + +import asyncio +import uuid +from datetime import timedelta + +import pytest +from httpx import ASGITransport, AsyncClient +from sqlalchemy import select + +from app.db.base import SessionLocal, utcnow +from app.db.models import Run, RunEvent, Task +from app.engine.runner import run_manager + + +@pytest.fixture(autouse=True) +async def engine_up(): + await run_manager.setup() + yield + await run_manager.shutdown() + + +@pytest.fixture +async def client(): + from app.main import app + + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +def node(nid, ntype, label=None, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": label or nid, "config": config}} + + +def chain(*nodes): + return {"nodes": list(nodes), "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]} + + +START = node("start", "input", fields=[{"name": "q"}]) +OUT = node("out", "output", fields=[{"name": "结果", "value": "到了"}]) + +#: (关卡, 那一步, 驳回之后 Run.error 的开头) +GATES = { + "tool": (node("gate", "tool", label="算一下", tool="calculator", args={"expression": "1+1"}, approval="always"), + "工具 calculator 的调用已被驳回:不对"), + "code": (node("gate", "code", label="跑代码", code="print(1)", language="python", approval="always"), + "代码执行已被驳回:不对"), + "approve": (node("gate", "human", label="把关", mode="approve", title="可以吗", stop_on_reject=True), + "人工驳回:不对"), + "edit": (node("gate", "human", label="改稿", mode="edit", title="改一下", draft="初稿"), "人工驳回:不对"), + "input": (node("gate", "human", label="补充", mode="input", title="补一下"), "人工驳回:不对"), +} + + +async def _wait(run_id: str, statuses: tuple[str, ...], timeout: float = 20.0) -> Run: + deadline = asyncio.get_event_loop().time() + timeout + while asyncio.get_event_loop().time() < deadline: + async with SessionLocal() as session: + row = await session.get(Run, run_id) + if row is not None and row.status in statuses: + return row + await asyncio.sleep(0.05) + raise AssertionError(f"运行 {run_id} 没有进入 {statuses}") + + +async def _events(run_id: str, kind: str | None = None) -> list[RunEvent]: + async with SessionLocal() as session: + query = select(RunEvent).where(RunEvent.run_id == run_id).order_by(RunEvent.seq) + if kind: + query = query.where(RunEvent.type == kind) + return list((await session.execute(query)).scalars()) + + +async def _rejected_run(kind: str, *, note: str = "不对", **reply) -> str: + gate, _ = GATES[kind] + run = await run_manager.start(graph=chain(START, gate, OUT), input_payload={"q": "x"}) + await _wait(run.id, ("interrupted",)) + await run_manager.wait_idle(run.id) + await run_manager.resume(run.id, {"approved": False, "note": note, **reply}, actor="张工") + return run.id + + +@pytest.mark.parametrize("kind", list(GATES)) +async def test_a_rejection_ends_the_run_as_cancelled_with_who_and_why(kind): + run_id = await _rejected_run(kind) + final = await _wait(run_id, ("cancelled", "failed", "succeeded")) + assert (final.status, final.end_reason) == ("cancelled", "rejected"), final.error + # 原话不变:给人看的那一句照旧 + assert final.error == GATES[kind][1] and final.error_node_id == "gate" and final.finished_at is not None + assert not await _events(run_id, "run.failed"), "驳回不是失败" + [cancelled] = await _events(run_id, "run.cancelled") + data = {k: v for k, v in cancelled.data.items() if k != "timing"} + assert data == {"reason": "rejected", "node_id": "gate", "label": GATES[kind][0]["data"]["label"], "kind": kind, + "decision": "reject", "actor": "张工", "note": "不对"} + # 收尾和取消一样:封存了,在线的流收到终态 + async with SessionLocal() as session: + row = await session.get(Run, run_id) + assert row.manifest_hash and row.manifest_seq == cancelled.seq + + +async def test_a_timeout_rejection_is_recorded_as_expired_reject(): + from app.process import tasks + + gate = node("gate", "human", label="把关", mode="approve", title="可以吗", stop_on_reject=True, due="PT30M", + on_timeout={"action": "reject"}) + run = await run_manager.start(graph=chain(START, gate, OUT), input_payload={"q": "x"}) + await _wait(run.id, ("interrupted",)) + await run_manager.wait_idle(run.id) + ticked = await tasks.tick(utcnow() + timedelta(hours=1)) + async with SessionLocal() as session: + mine = (await session.execute(select(Task.id).where(Task.run_id == run.id))).scalar_one() + assert mine in ticked + final = await _wait(run.id, ("cancelled", "failed", "succeeded")) + assert (final.status, final.end_reason) == ("cancelled", "rejected") + [cancelled] = await _events(run.id, "run.cancelled") + assert cancelled.data["decision"] == "expired_reject" and cancelled.data["actor"] is None + + +async def test_the_api_carries_end_reason_and_refuses_to_continue_a_rejected_run(client): + run_id = await _rejected_run("approve") + await _wait(run_id, ("cancelled",)) + body = (await client.get(f"/api/runs/{run_id}")).json() + assert (body["status"], body["end_reason"]) == ("cancelled", "rejected") + r = await client.post(f"/api/runs/{run_id}/continue", json={}) + assert r.status_code == 409 and r.json()["code"] == "run_rejected" + assert r.json()["detail"] == "被驳回的运行不能接着跑:改好内容后重新发起" + stop = await client.post(f"/api/runs/{run_id}/cancel") + assert stop.status_code == 409 and "被驳回" in stop.json()["detail"] + + +async def test_a_legacy_failed_rejection_cannot_be_continued_either(client): + """加 end_reason 之前被驳回的运行记的是 failed(封存不改写):失败的那一步上最近一条处理完的待办是驳回,也不许接着跑。 + 别的失败照旧能接着跑(这里断点不在,说的是没有断点,不是被驳回)。""" + async with SessionLocal() as session: + old = Run(workflow_name="老的驳回", status="failed", graph=chain(START, GATES["approve"][0], OUT), input={}, + error="人工驳回:不对", error_node_id="gate") + plain = Run(workflow_name="老的失败", status="failed", graph=chain(START, OUT), input={}, error="出错了") + session.add_all([old, plain]) + await session.flush() + session.add_all([ + RunEvent(run_id=old.id, seq=1, type="node.failed", node_id="gate", ts=1.0, data={"error": "人工驳回:不对"}), + Task(run_id=old.id, node_id="gate", kind="approve", title="可以吗", status="resolved", + outcome={"decision": "reject"}), + RunEvent(run_id=plain.id, seq=1, type="node.failed", node_id="out", ts=1.0, data={"error": "出错了"}), + ]) + await session.commit() + old_id, plain_id = old.id, plain.id + r = await client.post(f"/api/runs/{old_id}/continue", json={}) + assert r.status_code == 409 and r.json()["code"] == "run_rejected", r.text + other = await client.post(f"/api/runs/{plain_id}/continue", json={}) + assert other.status_code == 409 and other.json().get("code") != "run_rejected" + + +async def test_instance_state_says_the_same_for_the_new_shape_and_the_legacy_one(): + from app.apps.resolve import _trails, instance_state + + run_id = await _rejected_run("approve") + await _wait(run_id, ("cancelled",)) + async with SessionLocal() as session: + new = await session.get(Run, run_id) + legacy = Run(workflow_name="老的驳回", status="failed", graph=new.graph, input={}, error=new.error, + error_node_id="gate") + session.add(legacy) + await session.flush() + session.add_all([ + RunEvent(run_id=legacy.id, seq=1, type="node.failed", node_id="gate", ts=1.0, data={}), + Task(run_id=legacy.id, node_id="gate", kind="approve", title="可以吗", status="resolved", + outcome={"decision": "reject"}), + ]) + await session.commit() + trails = await _trails(session, [new, legacy]) + said = {"code": "cancelled", "text": "「把关」被驳回"} + assert instance_state(new, trails[new.id]) == said + assert instance_state(legacy, trails[legacy.id]) == said + + +async def test_stopping_is_recorded_as_stopped(): + gate, _ = GATES["approve"] + run = await run_manager.start(graph=chain(START, gate, OUT), input_payload={"q": "x"}) + await _wait(run.id, ("interrupted",)) + await run_manager.wait_idle(run.id) + assert await run_manager.cancel(run.id, actor="李工") == "cancelled" + final = await _wait(run.id, ("cancelled",)) + assert final.end_reason == "stopped" + [cancelled] = await _events(run.id, "run.cancelled") + assert cancelled.data["reason"] == "stopped" and cancelled.data["actor"] == "李工" + + +async def test_a_run_still_queued_at_restart_gets_its_run_failed_event(): + """(h):重启时排队中的运行改成 failed,以前只改状态,时间线里没有失败原因这一行。""" + from app.core.bus import bus + + async with SessionLocal() as session: + run = Run(workflow_name=f"排队-{uuid.uuid4().hex[:6]}", status="queued", graph={}, input={}, last_seq=2) + session.add(run) + await session.commit() + run_id = run.id + await run_manager.shutdown() + bus._seq.pop(run_id, None) + await run_manager.setup() + async with SessionLocal() as session: + row = await session.get(Run, run_id) + assert row.status == "failed" and row.started_at is None and row.finished_at is None + [failed] = await _events(run_id, "run.failed") + assert failed.seq == 3 and failed.data == {"error": row.error} diff --git a/backend/tests/test_scheduler_webhook.py b/backend/tests/test_scheduler_webhook.py index 635a59ab..0b6207f4 100644 --- a/backend/tests/test_scheduler_webhook.py +++ b/backend/tests/test_scheduler_webhook.py @@ -426,11 +426,33 @@ async def test_without_webhook_old_fires_are_not_pushed_later(recorder): async def test_cancelled_run_is_not_pushed(recorder): - _, fire_id, _ = await make(run_status="cancelled") + _, fire_id, run_id = await make(run_status="cancelled") + await set_run(run_id, end_reason="stopped") await webhook.notify_due() assert recorder.requests == [] and (await fire_row(fire_id)).notified_status == "cancelled" +async def test_a_rejected_run_is_pushed_as_not_issued(recorder): + """被人驳回的那一期(cancelled + end_reason=rejected):按「运行结束」推「未出具(被驳回)」,写驳回的那一步,不写驳回的 + 备注和原话(可能带着数);订阅「运行失败」的不收(05-已知遗留 (a))。""" + _, fire_id, run_id = await make(run_status="cancelled", output={}, fmt="wecom", url=WECOM) + await set_run(run_id, end_reason="rejected", error_node_id="pay", error="「付款」未确认(被驳回),没有发送:金额 0042 不对", + graph={"nodes": [{"id": "pay", "type": "tool", "data": {"label": "付款", "config": {}}}], + "edges": []}) + assert await webhook.notify_due() == 1 + content = json.loads(recorder.requests[0].content)["markdown"]["content"] + assert "定时任务「门店周报」未出具(被驳回)" in content and '状态:被驳回' in content + assert "驳回节点:「付款」" in content and "失败" not in content + assert "0042" not in content and "金额" not in content + assert (await fire_row(fire_id)).notified_status == "cancelled" + [delivery] = await deliveries(fire_id) + assert delivery.event == "finished" + + _, only_failed, other = await make(run_status="cancelled", output={}, events=("failed",)) + await set_run(other, end_reason="rejected") + assert await webhook.notify_due() == 0 and (await fire_row(only_failed)).notified_status == "cancelled" + + async def test_waits_for_the_seal_before_pushing(recorder): _, fire_id, run_id = await make(sealed=False) assert await webhook.notify_due() == 0 and (await fire_row(fire_id)).notified_status is None From ec5d9398b5f872aa9711852ad848e62061358598 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Fri, 16 Oct 2026 10:14:43 +0800 Subject: [PATCH 06/39] =?UTF-8?q?=E5=AF=BC=E5=85=A5=E6=B8=85=E5=8D=95?= =?UTF-8?q?=E3=80=81=E7=B4=AF=E7=A7=AF=E5=BF=AB=E7=85=A7=E6=B8=85=E5=8D=95?= =?UTF-8?q?=E3=80=81=E4=BD=9C=E5=BA=9F=E8=AE=B0=E5=BD=95=E3=80=81=E6=B8=85?= =?UTF-8?q?=E9=99=A4=E5=8E=9F=E4=BB=B6=E8=AE=B0=E5=BD=95=E5=A6=82=E5=AE=9E?= =?UTF-8?q?=E5=86=99=E3=80=8C=E6=98=AF=E5=90=A6=E8=AE=A4=E8=AF=81=E3=80=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 为什么:这四处的 signed_by.verified / signed_by_verified 写死 False。开了登录时署名就是登录的人,封存证据里却记着 「未认证」,和审计表、待办派单、run.origin 的 verified 口径对不上——留痕记错了事实(05-已知遗留 (c))。 做了什么:api/deps 新增 signer_verified()(开了登录、这次请求的身份认证过时为真;关登录、不在请求里为假),和 signer_of 配对。表格导入提交、按期移除 / 作废接受、清除原件三个写接口在请求那一刻算好,和 signed_by 一起往下传到 recipe_imports._manifest、source_versions.manifest_doc / _revoked_record、table_versions.purge_import_raw;没有署名的 (退回暂存区上记的那个)照实记未认证。新参数都带默认值 False:运行里按配方导入一期(files/imports.py,署名取动作确认人 或发起人)没有请求身份,照旧记未认证。已封存的旧清单不改写,读取复验照旧。 测试:test_signer_verified(signer_verified 跟请求身份走;作废记录照给的写;开登录首次导入的清单 verified=True、清除原件 记录 signed_by 是登录的人且 verified=True,请求体里填的名字不看)。关登录的老断言(False)照旧通过。 --- backend/app/api/datasources.py | 5 ++- backend/app/api/deps.py | 9 ++++ backend/app/api/source_versions.py | 7 +-- backend/app/api/table_imports.py | 5 ++- backend/app/data/recipe_imports.py | 15 ++++--- backend/app/data/source_versions.py | 30 ++++++++----- backend/app/data/table_versions.py | 6 ++- backend/tests/test_signer_verified.py | 63 +++++++++++++++++++++++++++ 8 files changed, 114 insertions(+), 26 deletions(-) create mode 100644 backend/tests/test_signer_verified.py diff --git a/backend/app/api/datasources.py b/backend/app/api/datasources.py index 38bebb9b..3f8a972f 100644 --- a/backend/app/api/datasources.py +++ b/backend/app/api/datasources.py @@ -18,7 +18,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.api import health from app.api.coded import PROFILE_SETTINGS_INVALID, CodedHTTPException -from app.api.deps import signer_of +from app.api.deps import signer_of, signer_verified from app.core.errors import AUTH, NETWORK, TIMEOUT, classify, explain, raw from app.core.config import settings from app.core.crypto import encrypt, mask @@ -1104,7 +1104,8 @@ async def purge_raw( if imp.raw_state == "purged": raise CodedHTTPException(409, "这次导入的原件已经清除", "raw_already_purged") try: - purged = await table_versions.purge_import_raw(session, imp, reason=reason, signed_by=signed_by) + purged = await table_versions.purge_import_raw(session, imp, reason=reason, signed_by=signed_by, + signer_verified=signer_verified()) except table_versions.StoreUnavailable as e: raise CodedHTTPException(503, str(e), "store_unavailable") from e deleted, others = purged.deleted, purged.others diff --git a/backend/app/api/deps.py b/backend/app/api/deps.py index 6fa03e2d..eebbc76e 100644 --- a/backend/app/api/deps.py +++ b/backend/app/api/deps.py @@ -105,6 +105,15 @@ def signer_of(given: Any, header: str | None, limit: int = 100) -> str | None: return decode_actor(header) +def signer_verified() -> bool: + """signer_of 记下的署名是不是认证过的身份:开了登录、这次请求的身份认证过时为真;关登录时署名是自报的,为假。 + + 导入清单、累积快照清单、作废记录、清除原件记录的「是否认证」记这个(docs/design/next-round/05-已知遗留.md (c)), + 和审计表、待办派单、run.origin 的 verified 同一个口径。证据要在生成那一刻定下:调用方和 signed_by 一起往下传。""" + principal = _principal.get() + return principal is not None and principal.mode != "off" and bool(principal.verified) + + def current_principal() -> Principal: """这次请求的完整身份。关登录时:等同 admin、未认证,username 是中间件按 X-Actor 解出的自报署名;中间件没写 (不在请求里,比如测试直接调函数、启动时的整理、调度循环)时同样等同 admin、未认证,username 为空。""" diff --git a/backend/app/api/source_versions.py b/backend/app/api/source_versions.py index 7d8a07ba..bd77947f 100644 --- a/backend/app/api/source_versions.py +++ b/backend/app/api/source_versions.py @@ -21,7 +21,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.api.coded import CodedHTTPException from app.api.datasources import _out -from app.api.deps import signer_of +from app.api.deps import signer_of, signer_verified from app.data import source_versions, table_versions from app.data.recipe_imports import ImportRefused from app.db.base import get_session @@ -170,7 +170,8 @@ async def remove_period(source_id: str, import_id: str, request: Request, x_acto source = await _source(session, source_id) try: out = await source_versions.remove_period(session, source, import_id, expected_current=expected, - reason=reason or "", signed_by=_signed(body, x_actor)) + reason=reason or "", signed_by=_signed(body, x_actor), + signer_verified=signer_verified()) except _REFUSALS as e: raise _refused(e) from e return {"source": await _source_out(session, source_id), **out} @@ -193,7 +194,7 @@ async def revoke_acceptance(source_id: str, import_id: str, request: Request, try: out = await source_versions.revoke_acceptance( session, source, import_id, expected_current=expected, expected_target=target, ack_mask_lost=ack, - reason=reason or "", signed_by=_signed(body, x_actor)) + reason=reason or "", signed_by=_signed(body, x_actor), signer_verified=signer_verified()) except _REFUSALS as e: raise _refused(e) from e return {"source": await _source_out(session, source_id), **out} diff --git a/backend/app/api/table_imports.py b/backend/app/api/table_imports.py index 24e13370..1fdf408e 100644 --- a/backend/app/api/table_imports.py +++ b/backend/app/api/table_imports.py @@ -28,7 +28,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.api.coded import CodedHTTPException from app.api.datasources import _NAME_PATTERN, _out, read_upload -from app.api.deps import current_principal, signer_of +from app.api.deps import current_principal, signer_of, signer_verified from app.data import recipe_imports, table_versions from app.data.recipe_imports import EditRequest, ImportRefused from app.data.recipe_types import Acceptance, AiAvailability, PeriodInput, Recipe, Selection @@ -422,7 +422,8 @@ async def commit(staging_id: str, request: Request, x_actor: str | None = Header try: result = await recipe_imports.commit_staging( session, staging, trial_id=trial_id if isinstance(trial_id, str) else "", - confirmations=confirmations, acceptances=_acceptances(body.get("acceptances"), who), signed_by=who) + confirmations=confirmations, acceptances=_acceptances(body.get("acceptances"), who), signed_by=who, + signer_verified=signer_verified()) except _REFUSALS as e: raise _refused(e) from e source = await session.get(DataSource, result.source_id, populate_existing=True) diff --git a/backend/app/data/recipe_imports.py b/backend/app/data/recipe_imports.py index 40dba2b1..34acb9f0 100644 --- a/backend/app/data/recipe_imports.py +++ b/backend/app/data/recipe_imports.py @@ -2361,7 +2361,7 @@ def _manifest(*, staging_id: str, kind: str, source_id: str, info: table_version ex_dict: dict[str, Any], file: dict[str, Any], recipe_meta: dict[str, Any], checks: list[dict[str, Any]], overrides: list[dict[str, Any]], waivers: list[dict[str, Any]], confirmed: list[dict[str, Any]], notes: SchemaNotes, ai: dict[str, Any], signed_by: str | None, - edits: list[dict[str, Any]] | None = None) -> dict[str, Any]: + edits: list[dict[str, Any]] | None = None, signer_verified: bool = False) -> dict[str, Any]: receipt = copy.deepcopy(ex_dict) receipt.pop("problems", None) return { @@ -2383,7 +2383,8 @@ def _manifest(*, staging_id: str, kind: str, source_id: str, info: table_version "ai": ai, # 期 3:修复、框选的修改记录(含已被整份替换覆盖的,作为历史;不带撤销用的修改前起点) "edits": copy.deepcopy(list(edits or [])), - "signed_by": {"name": signed_by, "verified": False}, + # 是否认证照实记(05-已知遗留 (c)):开了登录、署名是登录的人时为真。以前写死 False,封存的证据记错了事实 + "signed_by": {"name": signed_by, "verified": bool(signer_verified and signed_by)}, "staging_id": staging_id, "kind": kind, "created_at": _now_iso(), } @@ -2411,7 +2412,8 @@ def _file_sha(path: Path) -> str | None: async def commit_staging(session: AsyncSession, staging: ImportStaging, *, trial_id: str, confirmations: list[str], - acceptances: list[Acceptance], signed_by: str | None) -> CommitResult: + acceptances: list[Acceptance], signed_by: str | None, + signer_verified: bool = False) -> CommitResult: """POST imports/{id}/commit(7.8):预检 → (与现行导入相同时)不新建记录 → 发布构建、写配方、导入记录、 快照、清单,切指针。失败时指针不动,暂存区退回 drafting(试运行库已被发布流程消耗)。 @@ -2532,6 +2534,8 @@ async def commit_staging(session: AsyncSession, staging: ImportStaging, *, trial ex = extraction_from_dict(ex_dict) who = signed_by or staging.signed_by + # 署名是这次请求的(signed_by)才谈得上认证;退回暂存区上记的那个,是否认证这里不知道,照实记未认证 + who_verified = bool(signer_verified and signed_by) acc = [Acceptance(check_id=a.check_id, reason=a.reason.strip(), signed_by=who) for a in given.values()] # 这一期自己的说明(带实际的接受)进它的导入清单:版本页逐期显示「改写后的那一句」从这里取(第 5 节)。 # 快照冻结的说明在多期时另按各期最弱的结论合并(2.8),只剩本期时就是它 @@ -2603,7 +2607,8 @@ async def commit_staging(session: AsyncSession, staging: ImportStaging, *, trial file={"name": file_name, "size": file_size, "raw_sha256": raw_sha}, recipe_meta={"id": recipe_id, "sha256": recipe_sha, "origin": origin, "canonical": canonical}, checks=checks_full, overrides=overrides, waivers=waivers, confirmed=confirmed, - notes=notes if union_info is None else period_notes, ai=ai, signed_by=who, edits=edits) + notes=notes if union_info is None else period_notes, ai=ai, signed_by=who, edits=edits, + signer_verified=who_verified) async def schema_cache_for(view: table_versions.SourceView, info: table_versions.PublishInfo) -> dict[str, Any]: from app.core import artifact_store @@ -2663,7 +2668,7 @@ async def commit_staging(session: AsyncSession, staging: ImportStaging, *, trial "table_hashes": dict(union_report.get("table_hashes") or {}), "union_ver": recipe_types.UNION_VER}, parts=ordered, report=union_report, label_sets=list((plan or {}).get("label_sets") or []), gaps=list((plan or {}).get("gaps") or []), replaced=(plan or {}).get("replaces"), removed=None, - notes=notes, signed_by=who) + notes=notes, signed_by=who, signer_verified=who_verified) smid = await artifact_store.put_json(doc, kind="snapshot_manifest", meta={"source_id": source_id, "snapshot_id": info.snapshot_id}) cache["import_manifests"] = [mid if iid is None else by_imp[iid].info.manifest_artifact diff --git a/backend/app/data/source_versions.py b/backend/app/data/source_versions.py index a3dcb73f..4e389fb0 100644 --- a/backend/app/data/source_versions.py +++ b/backend/app/data/source_versions.py @@ -394,7 +394,7 @@ def columns_doc(report: dict[str, Any], periods: list[str]) -> dict[str, dict[st def manifest_doc(*, source_id: str, snapshot_id: str, action: str, recipe: dict[str, Any], union: dict[str, Any], parts: list[Part | dict[str, Any]], report: dict[str, Any], label_sets: list[Any], gaps: list[dict[str, str]], replaced: Any, removed: Any, - notes: SchemaNotes, signed_by: str | None) -> dict[str, Any]: + notes: SchemaNotes, signed_by: str | None, signer_verified: bool = False) -> dict[str, Any]: """快照清单(2.9)。parts 按统计期排序,与 report.part_rows 同序;本期写成 dict(导入记录还没落库)。""" part_rows = report.get("part_rows") or [] keys = [] @@ -422,7 +422,8 @@ def manifest_doc(*, source_id: str, snapshot_id: str, action: str, recipe: dict[ "checks": copy.deepcopy(report.get("checks") or []), "notes": {"templates": dict(notes.templates), "rendered": {t: {"comment": n.comment, "columns": dict(n.columns)} for t, n in notes.tables.items()}}, - "signed_by": {"name": signed_by, "verified": False}, + # 是否认证照实记(05-已知遗留 (c),同导入清单) + "signed_by": {"name": signed_by, "verified": bool(signer_verified and signed_by)}, "created_at": utcnow().isoformat(), } @@ -684,8 +685,10 @@ async def _current(session: AsyncSession, source: DataSource) -> _Current: mode=snap.mode or "replace", recipe=recipe) -def _revoked_record(reason: str, signed_by: str | None) -> dict[str, Any]: - return {"at": utcnow().isoformat(), "reason": reason, "signed_by": signed_by or None, "signed_by_verified": False} +def _revoked_record(reason: str, signed_by: str | None, signer_verified: bool = False) -> dict[str, Any]: + """作废接受的记录。是否认证照实记(05-已知遗留 (c)):开了登录、署名是登录的人时为真。""" + return {"at": utcnow().isoformat(), "reason": reason, "signed_by": signed_by or None, + "signed_by_verified": bool(signer_verified and signed_by)} async def rollback_target(session: AsyncSession, source: DataSource, import_id: str) -> SourceSnapshot | None: @@ -739,7 +742,8 @@ async def revoke_plan(session: AsyncSession, source: DataSource, imp: TableImpor async def _rebuild_without(session: AsyncSession, cur: _Current, imp: TableImport, *, expected_current: str | None, - kind: str, reason: str, signed_by: str | None, revoke: bool) -> tuple[str, bool]: + kind: str, reason: str, signed_by: str | None, revoke: bool, + signer_verified: bool = False) -> tuple[str, bool]: """从当前版本去掉一期、生成(或复用、复活)新快照(7.4)。配方和导入模式不变。返回 (快照 id, 是否复用)。 revoke=True 时同一个事务里写 table_imports.revoked(7.5):复用路径在调 activate_snapshot 之前改好(随它的提交 @@ -758,7 +762,7 @@ async def _rebuild_without(session: AsyncSession, cur: _Current, imp: TableImpor sid = table_versions.snapshot_id(source_id, keep, recipe_types.RECIPE_ENGINE_VER, mode="accumulate", recipe_sha=target_sha) imp_id = imp.id - revoked = _revoked_record(reason, signed_by) if revoke else None + revoked = _revoked_record(reason, signed_by, signer_verified) if revoke else None activation = {"kind": kind, "import_id": imp_id, "reason": reason, "signed_by": signed_by} existing = await session.get(SourceSnapshot, sid) @@ -871,7 +875,7 @@ async def _rebuild_without(session: AsyncSession, cur: _Current, imp: TableImpor "table_hashes": dict((info.union_report or report_json or {}).get("table_hashes") or {}), "union_ver": recipe_types.UNION_VER}, parts=parts, report=info.union_report or report_json or {}, label_sets=label_sets, gaps=gaps, - replaced=None, removed=removed, notes=notes, signed_by=signed_by) + replaced=None, removed=removed, notes=notes, signed_by=signed_by, signer_verified=signer_verified) cache["snapshot_manifest"] = await artifact_store.put_json( doc, kind="snapshot_manifest", meta={"source_id": source_id, "snapshot_id": info.snapshot_id}) return cache @@ -931,7 +935,7 @@ def _check_base(cur: _Current, expected_current: str | None) -> None: async def remove_period(session: AsyncSession, source: DataSource, import_id: str, *, expected_current: str | None, - reason: str, signed_by: str | None) -> dict[str, Any]: + reason: str, signed_by: str | None, signer_verified: bool = False) -> dict[str, Any]: """POST …/imports/{id}/remove(7.4)。""" imp = await session.get(TableImport, import_id) if imp is None or imp.source_id != source.id: @@ -947,13 +951,15 @@ async def remove_period(session: AsyncSession, source: DataSource, import_id: st if len(cur.imports) < 2: raise ImportRefused(409, "last_period", _LAST_PERIOD) sid, reused = await _rebuild_without(session, cur, imp, expected_current=expected_current, kind="remove_period", - reason=reason, signed_by=signed_by, revoke=False) + reason=reason, signed_by=signed_by, revoke=False, + signer_verified=signer_verified) return {"snapshot_id": sid, "removed_import_id": import_id, "reused": reused} async def revoke_acceptance(session: AsyncSession, source: DataSource, import_id: str, *, expected_current: str | None, expected_target: str | None, - ack_mask_lost: list[str] | None, reason: str, signed_by: str | None) -> dict[str, Any]: + ack_mask_lost: list[str] | None, reason: str, signed_by: str | None, + signer_verified: bool = False) -> dict[str, Any]: """POST …/imports/{id}/revoke-acceptance(7.5)。累积模式(至少两期)= 移除这一期;否则回滚到第 5 节定义的目标。""" imp = await session.get(TableImport, import_id) if imp is None or imp.source_id != source.id: @@ -968,7 +974,7 @@ async def revoke_acceptance(session: AsyncSession, source: DataSource, import_id if _accumulating(cur): sid, _reused = await _rebuild_without(session, cur, imp, expected_current=expected_current, kind="revoke_acceptance", reason=reason, signed_by=signed_by, - revoke=True) + revoke=True, signer_verified=signer_verified) return {"snapshot_id": sid, "action": "remove_period"} target = await rollback_target(session, source, imp.id) if target is None: @@ -976,7 +982,7 @@ async def revoke_acceptance(session: AsyncSession, source: DataSource, import_id if target.id != expected_target: raise ImportRefused(409, "revoke_target_changed", _TARGET_CHANGED) target_id = target.id - imp.revoked = _revoked_record(reason, signed_by) + imp.revoked = _revoked_record(reason, signed_by, signer_verified) try: await table_versions.activate_snapshot( session, source, target_id, expected_current=expected_current, kind="revoke_acceptance", reason=reason, diff --git a/backend/app/data/table_versions.py b/backend/app/data/table_versions.py index 2407d0e3..3302344b 100644 --- a/backend/app/data/table_versions.py +++ b/backend/app/data/table_versions.py @@ -2130,7 +2130,7 @@ class RawPurge: async def purge_import_raw( - session: AsyncSession, imp: TableImport, *, reason: str, signed_by: str | None + session: AsyncSession, imp: TableImport, *, reason: str, signed_by: str | None, signer_verified: bool = False ) -> RawPurge: """清除一次导入的原件(提交)。按两元组解包得到 (是否删了文件, 一并标成已清除的其他导入记录), discarded_stagings 是一并放弃的暂存区。 @@ -2160,7 +2160,9 @@ async def purge_import_raw( now = utcnow() record = { "at": now.isoformat(), "reason": reason, - "signed_by": (signed_by or "").strip() or None, "signed_by_verified": False, + # 是否认证照实记(05-已知遗留 (c)):开了登录、署名是登录的人时为真 + "signed_by": (signed_by or "").strip() or None, + "signed_by_verified": bool(signer_verified and (signed_by or "").strip()), } imp.raw_state, imp.purged = "purged", record for other in others: diff --git a/backend/tests/test_signer_verified.py b/backend/tests/test_signer_verified.py new file mode 100644 index 00000000..6d4df7f7 --- /dev/null +++ b/backend/tests/test_signer_verified.py @@ -0,0 +1,63 @@ +"""导入清单、累积快照清单、作废记录、清除原件记录如实写「是否认证」(docs/design/next-round/05-已知遗留.md (c))。 + +以前四处都写死 False:开了登录、署名就是登录的人,封存的证据里却记着「未认证」,和审计表、待办派单、run.origin 的 +verified 对不上。现在由 api/deps.signer_verified 在请求那一刻定下、和署名一起往下传:开登录为真,关登录(自报的署名) +为假。已封存的旧清单不改写(关登录时的形状见 test_table_imports_api、test_source_versions_api、test_datasources_upload)。 +""" +from __future__ import annotations + +import pytest + +from app.api import deps +from app.core import artifact_store +from app.db.models import TableImport +from tests.fakes import auth_env +from tests.test_table_imports_api import first_import, get, store, unique # noqa: F401 - store 是 autouse 夹具 + + +@pytest.fixture +async def admin(): + await auth_env.reset() + c = await auth_env.turn_on_local("root") + try: + yield c + finally: + await c.aclose() + await auth_env.reset() + + +def test_signer_verified_follows_the_request_identity(): + token = deps._principal.set(deps.Principal(username="root", role="admin", verified=True, mode="local")) + try: + assert deps.signer_verified() is True + finally: + deps._principal.reset(token) + token = deps._principal.set(deps.Principal(username="张三", role="admin", verified=False, mode="off")) + try: + assert deps.signer_verified() is False, "关登录时署名是自报的" + finally: + deps._principal.reset(token) + assert deps.signer_verified() is False, "不在请求里(启动整理、调度循环)不算认证" + + +def test_the_records_write_what_they_are_given(): + from app.data import source_versions + + assert source_versions._revoked_record("理由", "root", True)["signed_by_verified"] is True + assert source_versions._revoked_record("理由", "root")["signed_by_verified"] is False + assert source_versions._revoked_record("理由", None, True)["signed_by_verified"] is False, "没有署名谈不上认证" + + +async def test_login_on_import_manifest_and_purge_record_say_verified(admin): + out = (await first_import(admin, unique("signer")))["commit"] + imp = await get(TableImport, out["import_id"]) + manifest = artifact_store.load(imp.manifest_artifact) + assert manifest["signed_by"] == {"name": "root", "verified": True} + assert imp.signed_by == "root" + + source_id = out["source"]["id"] + resp = await admin.post(f"/api/datasources/{source_id}/imports/{imp.id}/purge-raw", + json={"reason": "含有不应保留的内容", "signed_by": "随手填的名字"}) + assert resp.status_code == 200, resp.text + purged = resp.json()["import"]["purged"] + assert (purged["signed_by"], purged["signed_by_verified"]) == ("root", True) From a05efbf27a784a96052157318c9b47ef0032f6a9 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Fri, 16 Oct 2026 10:17:37 +0800 Subject: [PATCH 07/39] =?UTF-8?q?=E5=AE=89=E5=85=A8=E5=93=8D=E5=BA=94?= =?UTF-8?q?=E5=A4=B4=EF=BC=9A=E6=89=80=E6=9C=89=20HTTP=20=E5=93=8D?= =?UTF-8?q?=E5=BA=94=E8=A1=A5=20nosniff=E3=80=81Referrer-Policy=20?= =?UTF-8?q?=E5=92=8C=20CSP=20frame-ancestors=EF=BC=8C=E9=A1=B5=E9=9D=A2?= =?UTF-8?q?=E5=8F=AA=E4=B8=8A=E6=8A=A5=E4=B8=A5=E6=A0=BC=E7=9A=84=20script?= =?UTF-8?q?-src?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 以前服务端一个安全头都不发,任何站点都能把 AgentLab 嵌进 iframe:同站门户(Lax 会话 cookie 会随请求带上)和关登录的部署 都有被点击劫持的可能,也没有「允许谁嵌」的白名单可配。 - 新增 app/web_security.py,纯 ASGI 中间件,main.py 里加在最外层:CORS 预检、登录检查自己回的 401 / 403 也带上; WebSocket 不经过它;接口自己设过的同名头(文件下载的 nosniff)不覆盖、不重复 - frame-ancestors 缺省只有 'self',这时另加 X-Frame-Options: SAMEORIGIN;AGENTLAB_FRAME_ANCESTORS 配了同站门户的白名单时 只发 CSP(XFO 写不了白名单,两个都发会让只认 XFO 的老浏览器把门户也拦掉) - 分享页 /s 只认 AGENTLAB_EMBED_ORIGINS、Referrer-Policy 用 no-referrer。分享链接第 3 波才加,这一波先把配置读进来、 头的规则写好 - 严格的 script-src 先只发 Content-Security-Policy-Report-Only:mount_web 启动时给构建产物 index.html 里的内联主题 脚本算 sha256,只在页面响应上发;接口返回的 HTML(导出的报告自带 meta CSP)不发、不动。观察一个版本没有误报再强制执行 - HSTS 交给终结 https 的反向代理 - 两个白名单只收「协议://主机[:端口]」(可以 *. 开头),带路径、裸 *、没写协议的启动时就报错;deploy/agentlab.env 写明 两个变量和缺省行为 测试 tests/test_web_security.py:页面、/api、中间件自己的 401、CORS 预检都带头;白名单生效时不发 XFO;/s 的规则; 上报策略只在页面上、哈希对得上;已有的头不重复;WebSocket 不受影响;白名单写错启动报错;子进程起真实的 app.main 核对 环境变量接线。 --- backend/app/core/config.py | 34 ++++- backend/app/main.py | 3 + backend/app/web.py | 11 +- backend/app/web_security.py | 90 +++++++++++ backend/tests/test_web_security.py | 235 +++++++++++++++++++++++++++++ deploy/agentlab.env | 11 ++ 6 files changed, 381 insertions(+), 3 deletions(-) create mode 100644 backend/app/web_security.py create mode 100644 backend/tests/test_web_security.py diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 9dcf013a..27533ecd 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -1,9 +1,10 @@ from __future__ import annotations import ipaddress +import re from functools import lru_cache from pathlib import Path -from typing import Annotated +from typing import Annotated, Any from pydantic import field_validator from pydantic_settings import BaseSettings, NoDecode, SettingsConfigDict @@ -11,6 +12,9 @@ from pydantic_settings import BaseSettings, NoDecode, SettingsConfigDict BACKEND_DIR = Path(__file__).resolve().parents[2] PROJECT_DIR = BACKEND_DIR.parent +#: 嵌入白名单里一项的主机部分:域名(可以 *. 开头)或方括号里的 IPv6,可以带端口;不许路径、查询、用户名 +_ORIGIN_HOST = re.compile(r"(?:\*\.)?(?:[A-Za-z0-9-]+\.)*[A-Za-z0-9-]+(?::\d{1,5})?|\[[0-9A-Fa-f:.]+\](?::\d{1,5})?") + # session_id 拼进路径前必须过这一关。允许的字符刻意收得很窄:字母数字加 # 连字符下划线——点号也不放行,免得出现 ".." 这种能往上走的名字。 _SESSION_MAX = 64 @@ -64,6 +68,13 @@ class Settings(BaseSettings): # 不设时按请求的 Host 拼 public_url: str = "" + # --- 安全响应头(app/web_security.py)--- + # 允许哪些站点用 iframe 嵌整站(同站的门户,如 https://portal.example.com),逗号分隔。本站自己总是允许的。 + # 不设时只有本站能嵌(frame-ancestors 'self',另加 X-Frame-Options: SAMEORIGIN) + frame_ancestors: Annotated[list[str], NoDecode] = [] + # 分享页(/s)允许被哪些站点嵌,逗号分隔,写法同上。分享页只认这一项,不认上面那一项 + embed_origins: Annotated[list[str], NoDecode] = [] + # --- 沙箱 --- sandbox_backend: str = "auto" # auto | microvm | seatbelt | bubblewrap | local | off sandbox_timeout: int = 30 @@ -121,7 +132,8 @@ class Settings(BaseSettings): "https://223.5.5.5/resolve", ] - @field_validator("http_tool_allowlist", "http_tool_fake_ip_ranges", "http_tool_doh_urls", mode="before") + @field_validator("http_tool_allowlist", "http_tool_fake_ip_ranges", "http_tool_doh_urls", "frame_ancestors", + "embed_origins", mode="before") @classmethod def _comma_list(cls, value: object) -> object: # 环境变量里写逗号分隔;也兼容 JSON 数组的写法 @@ -134,6 +146,24 @@ class Settings(BaseSettings): return [part.strip() for part in text.split(",") if part.strip()] return value + @field_validator("frame_ancestors", "embed_origins") + @classmethod + def _valid_origins(cls, value: list[str], info: Any) -> list[str]: + # 只收「协议://主机[:端口]」,主机可以写 *.example.com。路径、裸 *、没写协议的写错了启动时就报出来,别到嵌入 + # 失败(或者整站被谁都能嵌)才发现。'self' 本来就有,写了也不重复 + name = f"AGENTLAB_{(info.field_name or '').upper()}" + out: list[str] = [] + for raw in value: + item = raw.strip().rstrip("/") + if item == "'self'": + continue + scheme, _, host = item.partition("://") + if scheme.lower() not in ("http", "https") or not _ORIGIN_HOST.fullmatch(host): + raise ValueError(f"{name} 里的「{raw}」需要写成「协议://主机」(可以带端口),例如 https://portal.example.com") + if (origin := f"{scheme.lower()}://{host.lower()}") not in out: + out.append(origin) + return out + @field_validator("auth_cookie_secure", mode="before") @classmethod def _blank_is_auto(cls, value: object) -> object: diff --git a/backend/app/main.py b/backend/app/main.py index 657c917a..62cfea6f 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -30,6 +30,7 @@ from app.core.config import settings from app.db.base import init_db from app.engine.runner import run_manager from app.seed import seed_defaults +from app.web_security import SecurityHeaders logging.basicConfig( level=logging.INFO, @@ -192,6 +193,8 @@ app.add_middleware( allow_methods=["*"], allow_headers=["*"], ) +# 安全响应头(app/web_security.py):最后加、在最外层,CORS 预检和登录检查自己回的 401 / 403 也带上 +app.add_middleware(SecurityHeaders) # 路由级权限(§3.12)。关登录时这些依赖全部放行;开了登录,中间件先挡掉没登录的,这里再按角色和运行挡 _BUILDER = [require_role("builder")] diff --git a/backend/app/web.py b/backend/app/web.py index d2c36605..30602b95 100644 --- a/backend/app/web.py +++ b/backend/app/web.py @@ -9,7 +9,9 @@ - /api 下不存在的路径照旧是 JSON 404。排除 /api 写在路由的路径规则里,不在处理函数里判断—— 否则 POST /api/不存在 会因为「路径对上了、方法不对」变成 405; - 只注册 GET / HEAD 的 HTTP 路由,WebSocket 不经过这里; -- index.html 不缓存,发版后刷新即生效;assets/ 下的文件名带内容哈希,长缓存。 +- index.html 不缓存,发版后刷新即生效;assets/ 下的文件名带内容哈希,长缓存; +- index.html 另带严格 script-src 的上报策略(Content-Security-Policy-Report-Only):启动时按构建产物里的内联脚本算 + sha256(web_security.script_policy),只上报、不拦。其余安全头由 web_security 的中间件统一补。 """ from __future__ import annotations @@ -20,6 +22,8 @@ from fastapi import FastAPI, HTTPException from fastapi.responses import FileResponse from starlette.convertors import Convertor, register_url_convertor +from app.web_security import REPORT_ONLY_HEADER, script_policy + _NO_CACHE = "no-cache" _IMMUTABLE = "public, max-age=31536000, immutable" @@ -72,7 +76,12 @@ def mount_web(app: FastAPI, dist: Path) -> None: "先构建前端(cd frontend && pnpm build),或者去掉这个设置" ) + # 哈希按启动这一刻的 index.html 算:换了构建产物要重启服务,哈希才跟着换(只上报不拦,对不上也只多几条上报) + page_headers = {"Cache-Control": _NO_CACHE, REPORT_ONLY_HEADER: script_policy(index.read_text(encoding="utf-8"))} + def respond(target: Path, *, immutable: bool = False) -> FileResponse: + if target in (index, index.resolve()): + return FileResponse(target, media_type=_media_type(target), headers=page_headers) return FileResponse( target, media_type=_media_type(target), diff --git a/backend/app/web_security.py b/backend/app/web_security.py new file mode 100644 index 00000000..a7ae8646 --- /dev/null +++ b/backend/app/web_security.py @@ -0,0 +1,90 @@ +"""安全响应头(04 细案 C1):页面不能被别的站点套进 iframe 诱导点击,浏览器不按内容猜类型,链接不把完整地址带给别的站点。 + +以前服务端一个安全头都不发:任何站点都能把 AgentLab 嵌进 iframe。同站的门户页面(SameSite=Lax 的会话 cookie 会随请求 +带上)和关登录的部署,都有被点击劫持的可能;也没有一条「允许谁嵌」的白名单可配。 + +纯 ASGI 中间件(同 auth/middleware.py,main.py 里加在最外层):在 http.response.start 上补头,WebSocket 不经过它。 +所有 HTTP 响应(页面、/api、登录检查自己回的 401 / 403、CORS 预检)都补: +- X-Content-Type-Options: nosniff; +- Referrer-Policy: strict-origin-when-cross-origin(分享页 /s 用 no-referrer:分享的 key 在地址的 # 后面,本来不随 + Referer 走,这里再收一道); +- Content-Security-Policy: frame-ancestors。缺省只有 'self';AGENTLAB_FRAME_ANCESTORS 列出允许嵌整站的同站门户;分享页 + /s 只认 AGENTLAB_EMBED_ORIGINS(分享链接第 3 波才加,路由在那时挂,头的规则先在这里定好); +- 只有 frame-ancestors 是 'self' 时另加 X-Frame-Options: SAMEORIGIN,给不认 CSP 的老浏览器。XFO 写不了白名单,有白名单 + 时只发 CSP:两个都发的话,只认 XFO 的老浏览器会把白名单里的门户也拦掉(认 CSP 的浏览器以 frame-ancestors 为准)。 +接口自己设过的同名头(文件下载的 nosniff)不覆盖、不重复。 + +严格的 script-src 先只上报、不拦(Content-Security-Policy-Report-Only):页面里有一段首帧前定主题的内联脚本,强制执行 +的话写错一个哈希页面就会先闪一帧深色,构建会不会产出别的内联内容也没核实。所以 mount_web(app/web.py)启动时给构建产物 +index.html 里的内联脚本算 sha256(script_policy),只在页面这一个响应上发;观察一个版本没有误报,再改成强制执行。没配 +上报地址:违规只在浏览器的控制台里看得到。接口返回的 HTML(导出的报告、可验证报告)文件里自带 meta CSP,不发它,也不动。 + +HSTS 交给终结 https 的反向代理:这里不知道对外是不是 https、要不要连子域名一起管,发错了浏览器会按 max-age 记很久, +撤不回来。 +""" +from __future__ import annotations + +import base64 +import hashlib +import re +from typing import Any + +#: 分享页的路径(/s#<分享 key>):只认 /s 这一段,/settings、/studio 不算 +SHARE_PATH = "/s" +_SELF = "'self'" +_DEFAULT_REFERRER = b"strict-origin-when-cross-origin" +_SHARE_REFERRER = b"no-referrer" +#: 页面里不带 src 的 " + "" + "
") +HASH = base64.b64encode(hashlib.sha256(THEME.encode()).digest()).decode() +REPORT = ("" + "") + + +@pytest.fixture +def dist(tmp_path: Path) -> Path: + root = tmp_path / "dist" + (root / "assets").mkdir(parents=True) + (root / "index.html").write_text(INDEX) + (root / "assets" / "index-AbC123.js").write_text("console.log('bundle')") + return root + + +def _app(dist: Path | None = None, **config) -> FastAPI: + app = FastAPI() + api = APIRouter(prefix="/api") + + @api.get("/ping") + async def ping() -> dict[str, str]: + return {"pong": "1"} + + @api.get("/report.html") + async def report() -> HTMLResponse: + return HTMLResponse(REPORT) + + @api.get("/download") + async def download() -> Response: + # 文件下载自己设了 nosniff(api/files.py):不能再加一份 + return Response(b"x", headers={"X-Content-Type-Options": "nosniff", "Referrer-Policy": "no-referrer"}) + + @api.websocket("/echo") + async def echo(ws: WebSocket) -> None: + await ws.accept() + await ws.send_text(await ws.receive_text()) + await ws.close() + + app.include_router(api) + if dist is not None: + from app.web import mount_web + + mount_web(app, dist) + app.add_middleware(SecurityHeaders, **config) + return app + + +async def _get(app: FastAPI, path: str) -> httpx.Response: + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://t") as client: + return await client.get(path) + + +def _single(r: httpx.Response, name: str) -> str | None: + values = r.headers.get_list(name) + assert len(values) <= 1, f"{name} 发了 {len(values)} 份:{values}" + return values[0] if values else None + + +async def test_api_and_page_carry_the_defaults(dist: Path) -> None: + app = _app(dist, frame_ancestors=[], embed_origins=[]) + for path in ("/api/ping", "/api/nope", "/", "/studio", "/assets/index-AbC123.js"): + r = await _get(app, path) + assert _single(r, "x-content-type-options") == "nosniff", path + assert _single(r, "referrer-policy") == "strict-origin-when-cross-origin", path + assert _single(r, "content-security-policy") == "frame-ancestors 'self'", path + assert _single(r, "x-frame-options") == "SAMEORIGIN", path + + +async def test_report_only_script_policy_is_only_on_the_page(dist: Path) -> None: + app = _app(dist, frame_ancestors=[], embed_origins=[]) + for path in ("/", "/studio", "/runs/abc/evidence"): + r = await _get(app, path) + assert r.text == INDEX + assert _single(r, "content-security-policy-report-only") == f"script-src 'self' 'sha256-{HASH}'", path + # 脚本文件、接口不带;接口返回的 HTML(导出的报告)自带 meta CSP,不加一份对不上哈希的上报策略,正文原样 + for path in ("/assets/index-AbC123.js", "/api/ping", "/api/report.html"): + r = await _get(app, path) + assert "content-security-policy-report-only" not in r.headers, path + r = await _get(app, "/api/report.html") + assert r.text == REPORT + + +async def test_whitelist_wins_and_drops_x_frame_options() -> None: + app = _app(frame_ancestors=["https://portal.example.com", "https://*.corp.example"], embed_origins=[]) + r = await _get(app, "/api/ping") + assert _single(r, "content-security-policy") == \ + "frame-ancestors 'self' https://portal.example.com https://*.corp.example" + assert "x-frame-options" not in r.headers, "有白名单时以 CSP 为准:XFO 写不了白名单" + + +async def test_share_page_uses_the_embed_origins(dist: Path) -> None: + app = _app(dist, frame_ancestors=["https://portal.example.com"], embed_origins=["https://bi.example.org"]) + for path in ("/s", "/s/", "/s/anything"): + r = await _get(app, path) + assert _single(r, "content-security-policy") == "frame-ancestors 'self' https://bi.example.org", path + assert _single(r, "referrer-policy") == "no-referrer", path + assert "x-frame-options" not in r.headers, path + # 只认 /s 这一段:/settings、/studio 照整站的规则 + for path in ("/settings", "/studio", "/share"): + r = await _get(app, path) + assert _single(r, "content-security-policy") == "frame-ancestors 'self' https://portal.example.com", path + assert _single(r, "referrer-policy") == "strict-origin-when-cross-origin", path + # 没配嵌入白名单:分享页只能被本站嵌,同样另加 XFO + r = await _get(_app(dist, frame_ancestors=[], embed_origins=[]), "/s") + assert _single(r, "content-security-policy") == "frame-ancestors 'self'" + assert _single(r, "x-frame-options") == "SAMEORIGIN" + + +async def test_headers_already_set_are_kept() -> None: + r = await _get(_app(frame_ancestors=[], embed_origins=[]), "/api/download") + assert _single(r, "x-content-type-options") == "nosniff" + assert _single(r, "referrer-policy") == "no-referrer" + assert _single(r, "content-security-policy") == "frame-ancestors 'self'" + + +def test_websocket_is_not_touched() -> None: + with TestClient(_app(frame_ancestors=[], embed_origins=[])) as client: + with client.websocket_connect("/api/echo") as ws: + ws.send_text("hi") + assert ws.receive_text() == "hi" + + +async def test_main_app_and_its_own_refusals_carry_the_headers() -> None: + """真实的 app.main:加在最外层,登录检查自己回的 401、CORS 预检也带。""" + app = main_app + r = await _get(app, "/api/health") + assert _single(r, "content-security-policy") == "frame-ancestors 'self'" + assert _single(r, "x-frame-options") == "SAMEORIGIN" and _single(r, "x-content-type-options") == "nosniff" + await auth_env.reset() + try: + await auth_env.turn_on_local() + r = await _get(app, "/api/workflows") + assert r.status_code == 401 and r.json()["code"] == "auth_required" + assert _single(r, "x-content-type-options") == "nosniff" + assert _single(r, "content-security-policy") == "frame-ancestors 'self'" + finally: + await auth_env.reset() + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://t") as client: + r = await client.options("/api/health", headers={"Origin": "http://localhost:5273", + "Access-Control-Request-Method": "GET"}) + assert r.status_code == 200 and _single(r, "x-content-type-options") == "nosniff" + + +def test_origin_lists_parse_and_bad_entries_fail_at_startup(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("AGENTLAB_FRAME_ANCESTORS", raising=False) + monkeypatch.delenv("AGENTLAB_EMBED_ORIGINS", raising=False) + assert Settings().frame_ancestors == [] and Settings().embed_origins == [] + monkeypatch.setenv("AGENTLAB_FRAME_ANCESTORS", " https://portal.example.com/ , https://*.corp.example,'self'") + monkeypatch.setenv("AGENTLAB_EMBED_ORIGINS", '["http://bi.example.org:8080"]') + got = Settings() + assert got.frame_ancestors == ["https://portal.example.com", "https://*.corp.example"], "末尾的 / 去掉,'self' 本来就有" + assert got.embed_origins == ["http://bi.example.org:8080"] + monkeypatch.delenv("AGENTLAB_EMBED_ORIGINS") + # 只收「协议://主机[:端口]」:路径、通配一切、没写协议、'none' 这类写法启动时就报出来,别到嵌入失败才发现 + for bad in ("*", "portal.example.com", "https://portal.example.com/app", "ftp://portal.example.com", "'none'", + "https://", "https://portal.example.com?x=1"): + monkeypatch.setenv("AGENTLAB_FRAME_ANCESTORS", bad) + with pytest.raises(ValidationError, match="AGENTLAB_FRAME_ANCESTORS"): + Settings() + + +def test_deploy_env_documents_both_lists() -> None: + text = (PROJECT_DIR / "deploy" / "agentlab.env").read_text() + assert "# AGENTLAB_FRAME_ANCESTORS=" in text and "# AGENTLAB_EMBED_ORIGINS=" in text + + +_PROBE = r""" +import asyncio, json, httpx +from app.main import app + +async def main(): + t = httpx.ASGITransport(app=app) + async with httpx.AsyncClient(transport=t, base_url="http://t") as c: + out = {} + for p in ("/", "/api/health"): + r = await c.get(p) + out[p] = {k: r.headers.get(k) for k in ("content-security-policy", "x-frame-options", + "content-security-policy-report-only")} + print(json.dumps(out)) + +asyncio.run(main()) +""" + + +def test_main_app_reads_the_whitelist_and_hashes_the_page(dist: Path, tmp_path: Path) -> None: + """真实的 app.main:环境变量里的白名单生效、不发 XFO;托管页面时 index.html 带上报策略。settings、app 都是模块级 + 单例,起一个子进程导入。""" + env = {**os.environ, "AGENTLAB_DATA_DIR": str(tmp_path / "data"), "AGENTLAB_WEB_DIST": str(dist), + "AGENTLAB_SECRET_KEY": "test-only-not-a-real-key", "AGENTLAB_FRAME_ANCESTORS": "https://portal.example.com"} + proc = subprocess.run([sys.executable, "-c", _PROBE], cwd=BACKEND_DIR, env=env, capture_output=True, text=True, + timeout=120) + assert proc.returncode == 0, proc.stderr[-2000:] + out = json.loads(proc.stdout.strip().splitlines()[-1]) + for path in ("/", "/api/health"): + assert out[path]["content-security-policy"] == "frame-ancestors 'self' https://portal.example.com", path + assert out[path]["x-frame-options"] is None, path + assert out["/"]["content-security-policy-report-only"] == f"script-src 'self' 'sha256-{HASH}'" + assert out["/api/health"]["content-security-policy-report-only"] is None diff --git a/deploy/agentlab.env b/deploy/agentlab.env index 15b615d8..3a411c0b 100644 --- a/deploy/agentlab.env +++ b/deploy/agentlab.env @@ -31,6 +31,17 @@ BIND_ADDR=0.0.0.0 # 会话 cookie 是否带 Secure:留空按请求是不是 https 决定;反向代理终结了 https 时填 true # AGENTLAB_AUTH_COOKIE_SECURE= +# ---- 嵌入(可选)---- +# 默认只有 AgentLab 自己能把页面放进 iframe(响应头 Content-Security-Policy: frame-ancestors 'self',另加 +# X-Frame-Options: SAMEORIGIN),别的站点嵌进去会被浏览器拦下,防点击劫持。HSTS 不在这里加,交给终结 https 的反向代理。 +# 同站的门户(和 AgentLab 在同一个可注册域名下,比如 portal.example.internal 和 agentlab.example.internal)要嵌整站时, +# 把门户的地址写在下面,逗号分隔,只写 协议://主机[:端口],主机可以写 *.example.internal。填了之后只发 +# frame-ancestors、不再发 X-Frame-Options。登录用的 cookie 只随同站的请求带上,跨站的门户嵌整站登录不了。 +# 写错了(带路径、没写协议、只写 *)服务启动时报错。 +# AGENTLAB_FRAME_ANCESTORS=https://portal.example.internal +# 分享页(/s)允许被哪些站点嵌,写法同上。分享页只认这一项、不认上面那一项。分享链接还没有上线,现在填了不起作用 +# AGENTLAB_EMBED_ORIGINS=https://bi.example.internal + # ---- 数据 ---- # 数据目录:数据库、执行断点、上传的文件、沙箱工作区,还有加密密钥 .secret_key 都在这里。 # 要备份的就是这个目录(丢了 .secret_key,存进去的 API Key 就解不开了)。 From d8034ca71a49144d83a98cbcb3b8b09ee972ece6 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Fri, 16 Oct 2026 10:18:12 +0800 Subject: [PATCH 08/39] =?UTF-8?q?=E5=AF=B9=E5=A4=96=E8=AE=BF=E9=97=AE?= =?UTF-8?q?=E5=9C=B0=E5=9D=80=E5=8F=AA=E7=95=99=E4=B8=80=E4=B8=AA=EF=BC=9A?= =?UTF-8?q?core/public=5Furl.py=20=E7=9A=84=20public=5Fbase=5Furl()?= =?UTF-8?q?=EF=BC=8C=E9=80=9A=E7=9F=A5=E3=80=81Webhook=20=E9=93=BE?= =?UTF-8?q?=E6=8E=A5=E3=80=81OIDC=20=E5=9B=9E=E8=B0=83=E3=80=81=E6=9D=A5?= =?UTF-8?q?=E6=BA=90=E6=A0=B8=E5=AF=B9=E9=83=BD=E6=94=B9=E7=94=A8=E5=AE=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 以前有两个「对外地址」:环境变量 AGENTLAB_PUBLIC_URL 只管 OIDC 回调和 Origin 核对,设置里 notify.public_base_url 只管 通知里的链接。只配了环境变量的部署,通知里不带链接;只在设置里填了的,回调又按请求的 Host 拼——违背「一种说法」。 - 新增 core/public_url.py:环境变量优先,其次设置(沿用 notify.public_base_url 这个键),都没有是 None。设置的值缓存在 进程里(中间件按请求核对来源,不能每次查库):启动时 refresh 一次,PUT /api/settings 写完库就 remember,不用重启; 通知、定时任务 Webhook 手里本来就有会话,用 refresh 取,顺手按库里的值对齐缓存 - 地址的形状检查和规范化只有一份(url_problem / normalize),设置接口的中文报错和环境变量的启动校验都用它;环境变量 写错了(没有 http:// 或 https://、带 ? 或 #)启动时就报错,不再拼出坏的回调地址 - GET /api/settings 另给只读的 public_url 一组 {value, env_locked},设置页据此显示「由环境变量指定」;环境变量设了时 PUT 把这一项改成别的值回 409(public_url_env_locked,field 指到这一项),原样写回不算改;public_url 一组写不进库 - notify/service 取地址的那一处、scheduler/webhook 链接里的地址、api/auth 的 OIDC 回调、auth/middleware 的 Origin 核对 都改走 public_base_url;deploy/agentlab.env 把这个变量的说法改成全站唯一的对外访问地址 测试 tests/test_public_url.py:先后顺序(环境变量 > 设置 > 无)、规范化、启动校验、PUT 刷新缓存、环境变量锁定(409、 不挡原样写回)、只配环境变量时通知带链接、OIDC 回调和 Origin 核对认设置里的地址。去掉修复回跑,只配环境变量的通知 链接、按设置核对 Origin 两条都会失败。设置页把它挪到「通用」在后面的前端提交里。 --- backend/app/api/auth.py | 6 +- backend/app/api/coded.py | 3 + backend/app/api/settings.py | 67 +++++++----- backend/app/auth/middleware.py | 9 +- backend/app/core/config.py | 18 +++- backend/app/core/public_url.py | 97 ++++++++++++++++++ backend/app/main.py | 10 +- backend/app/notify/service.py | 7 +- backend/app/scheduler/webhook.py | 5 +- backend/tests/test_public_url.py | 169 +++++++++++++++++++++++++++++++ deploy/agentlab.env | 4 +- 11 files changed, 351 insertions(+), 44 deletions(-) create mode 100644 backend/app/core/public_url.py create mode 100644 backend/tests/test_public_url.py diff --git a/backend/app/api/auth.py b/backend/app/api/auth.py index 6a4c87d9..60e58654 100644 --- a/backend/app/api/auth.py +++ b/backend/app/api/auth.py @@ -25,7 +25,6 @@ from app.auth import oidc, sessions from app.auth.audit import record from app.auth.passwords import hash_password, password_problem, verify_password from app.core import crypto -from app.core.config import settings from app.db.base import get_session from app.db.models import Setting, User @@ -113,7 +112,10 @@ async def logout(request: Request, session: AsyncSession = Depends(get_session)) def _redirect_uri(request: Request) -> str: - base = settings.public_url.rstrip("/") or str(request.base_url).rstrip("/") + """回调地址:对外访问地址(环境变量优先,其次设置,core/public_url.py),都没有按请求自己的地址拼。""" + from app.core.public_url import public_base_url + + base = public_base_url() or str(request.base_url).rstrip("/") return f"{base}/api/auth/oidc/callback" diff --git a/backend/app/api/coded.py b/backend/app/api/coded.py index d8fdba23..da319c42 100644 --- a/backend/app/api/coded.py +++ b/backend/app/api/coded.py @@ -29,6 +29,9 @@ CATALOG_DRAFT_BUSY = "catalog_draft_busy" CATALOG_DRAFT_MODEL_UNAVAILABLE = "catalog_draft_model_unavailable" CATALOG_DRAFT_FINISHED = "catalog_draft_finished" +#: 设置(api/settings.py):对外访问地址由环境变量 AGENTLAB_PUBLIC_URL 指定时,设置里那一项不能改成别的值(409) +PUBLIC_URL_ENV_LOCKED = "public_url_env_locked" + class CodedHTTPException(HTTPException): """field:出错的是哪一项(表单据此把报错落到那一格)。可选,没有就不出现在返回里。""" diff --git a/backend/app/api/settings.py b/backend/app/api/settings.py index 83fcaaf7..c45a685a 100644 --- a/backend/app/api/settings.py +++ b/backend/app/api/settings.py @@ -10,7 +10,8 @@ from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from app.api import health -from app.api.coded import CodedHTTPException +from app.api.coded import PUBLIC_URL_ENV_LOCKED, CodedHTTPException +from app.core import public_url from app.core.errors import explain, not_configured, raw from app.core.config import settings as app_settings from app.core.crypto import encrypt, mask @@ -604,35 +605,32 @@ def catalog_draft_values(stored: Any) -> dict[str, Any]: # -------------------------------------------------------------------------- -# 通知:定时任务推送 Webhook 时拼运行链接用的对外访问地址 +# 对外访问地址:通知、Webhook 里的链接,单点登录的回调,来源核对(core/public_url.py) # -------------------------------------------------------------------------- #: 设置里 notify 这一组。public_base_url 是别人打开 AgentLab 用的地址(https://agentlab.example.com),通知里的 -#: 运行链接 = 它 + /runs/{运行 id}。服务端不知道自己在反向代理后面叫什么,只能由人填;留空的通知里不附链接 -NOTIFY_GROUP = "notify" -NOTIFY_DEFAULTS: dict[str, Any] = {"public_base_url": None} -NOTIFY_SETTING_LABEL = {"public_base_url": "对外访问地址"} +#: 运行链接 = 它 + /runs/{运行 id}。服务端不知道自己在反向代理后面叫什么,只能由人填;留空的通知里不附链接。 +#: 全站只认一个对外地址(core/public_url.py):环境变量 AGENTLAB_PUBLIC_URL 设了的话这一项不生效、不能改(PUT 回 409)。 +#: 键名沿用 notify 一组,界面上在「设置 → 通用」 +NOTIFY_GROUP = public_url.SETTING_GROUP +NOTIFY_DEFAULTS: dict[str, Any] = {public_url.SETTING_KEY: None} +NOTIFY_SETTING_LABEL = {public_url.SETTING_KEY: "对外访问地址"} +#: GET /api/settings 另给的只读一组:{value: 生效的对外访问地址, env_locked: 是不是环境变量指定的}。设置页据此显示 +#: 「由环境变量指定」;PUT 带上它也不落库(同 limits) +PUBLIC_URL_GROUP = "public_url" def _base_url_problem(value: Any) -> str | None: if value is None or (isinstance(value, str) and not value.strip()): return None - label = NOTIFY_SETTING_LABEL["public_base_url"] + label = NOTIFY_SETTING_LABEL[public_url.SETTING_KEY] hint = (f"「{label}」需要是 http:// 或 https:// 开头的完整地址(例如 https://agentlab.example.com)," - f"留空表示通知里不附运行链接,当前为「{value}」") + f"留空表示通知里不附链接,当前为「{value}」") if not isinstance(value, str): return hint - from urllib.parse import urlsplit - - try: - parts = urlsplit(value.strip()) - except ValueError: - return hint - if parts.scheme not in ("http", "https") or not parts.hostname or " " in value.strip(): - return hint - if parts.query or parts.fragment: - return f"「{label}」不能带 ? 或 # 之后的部分,运行链接会接在地址后面,当前为「{value}」" - return None + if (problem := public_url.url_problem(value)) == "suffix": + return f"「{label}」不能带 ? 或 # 之后的部分,链接会接在地址后面,当前为「{value}」" + return hint if problem else None def notify_problem(value: Any) -> str | None: @@ -651,18 +649,23 @@ def notify_problem(value: Any) -> str | None: def notify_values(stored: Any) -> dict[str, Any]: """库里存的(或刚写进来的)notify 一组 → 补齐默认、规范化:去掉首尾空白和末尾的 /,空串当没填;存坏了的按没填。""" stored = stored if isinstance(stored, dict) else {} - url = stored.get("public_base_url") - if not isinstance(url, str) or not url.strip() or _base_url_problem(url): - return {"public_base_url": None} - return {"public_base_url": url.strip().rstrip("/")} + return {public_url.SETTING_KEY: public_url.normalize(stored.get(public_url.SETTING_KEY))} async def notify_settings(session: AsyncSession) -> dict[str, Any]: - """设置里的 notify 一组,没存过的补默认。定时任务推送 Webhook 时从这里读。""" + """设置里存的 notify 一组,没存过的补默认。要拼链接的地方不读它:一律经 core/public_url(环境变量优先)。""" row = await session.get(Setting, NOTIFY_GROUP) return notify_values(row.value if row else None) +def public_url_values(stored_notify: Any) -> dict[str, Any]: + """GET /api/settings 里只读的 public_url 一组:生效的那个地址(环境变量优先,其次库里存的这一项),是不是环境变量 + 指定的。按库里存的算,不读进程缓存:设置页看到的就是落库的样子。""" + locked = public_url.env_locked() + value = public_url.env_url() if locked else notify_values(stored_notify)[public_url.SETTING_KEY] + return {"value": value, "env_locked": locked} + + # -------------------------------------------------------------------------- # 应用:按需视图的每日上限(D-j)、使用者能不能建个人应用(docs/design/normal-user-apps.md N7) # -------------------------------------------------------------------------- @@ -754,7 +757,7 @@ DEFAULT_SETTINGS: dict[str, Any] = { JUDGE_GROUP: dict(JUDGE_DEFAULTS), # 核查(双盲复算、反方审稿、审批质询):核查模型、三项的开关和上限、每日金额上限。和裁判一组一样由用户定 VERIFY_GROUP: verify_values(None), - # 通知:定时任务的 Webhook 里拼运行链接用的对外访问地址 + # 对外访问地址(存在 notify 一组):通知、Webhook 里的链接,单点登录的回调,来源核对。环境变量优先(core/public_url.py) NOTIFY_GROUP: notify_values(None), # 起草模型:数据目录起草、按配方导入的 AI 起草用的模型,留空沿用助手的模型 CATALOG_DRAFT_GROUP: catalog_draft_values(None), @@ -808,6 +811,7 @@ async def get_settings(session: AsyncSession = Depends(get_session)) -> dict[str # limits 是服务端配置(环境变量),只读:库里可能存着旧版设置页写进去的一份 #(见过 max_agent_steps: 25),它从来不生效,却会盖住真实的上限,界面照着错的数许诺 stored.pop("limits", None) + stored.pop(PUBLIC_URL_GROUP, None) # 裁判、核查的每日计数是引擎记的账,不是用户设置:设置页整组回写时带着一份旧的,会把今天的花费清零 stored.pop(JUDGE_SPEND, None) stored.pop(VERIFY_SPEND, None) @@ -818,6 +822,7 @@ async def get_settings(session: AsyncSession = Depends(get_session)) -> dict[str merged[JUDGE_GROUP] = judge_values(stored.get(JUDGE_GROUP)) merged[VERIFY_GROUP] = verify_values(stored.get(VERIFY_GROUP)) merged[NOTIFY_GROUP] = notify_values(stored.get(NOTIFY_GROUP)) + merged[PUBLIC_URL_GROUP] = public_url_values(stored.get(NOTIFY_GROUP)) merged[CATALOG_DRAFT_GROUP] = catalog_draft_values(stored.get(CATALOG_DRAFT_GROUP)) merged[APPS_GROUP] = apps_values(stored.get(APPS_GROUP)) for key, value in stored.items(): @@ -910,6 +915,11 @@ async def put_settings( raise HTTPException(422, problem) # 存规范化之后的:末尾带 / 的地址拼链接会多出一个斜杠 payload.values[NOTIFY_GROUP] = notify_values(payload.values[NOTIFY_GROUP]) + if public_url.env_locked() and payload.values[NOTIFY_GROUP] != await notify_settings(session): + # 环境变量指定了对外访问地址:改这一项不会生效,照实拒绝,不悄悄存一个用不上的值。原样写回(整组回写)不算改 + raise CodedHTTPException( + 409, f"对外访问地址由环境变量 {public_url.ENV_NAME} 指定({public_url.env_url()}),不能在设置里修改。" + "要改请改部署配置后重启服务", PUBLIC_URL_ENV_LOCKED, field=f"{NOTIFY_GROUP}.{public_url.SETTING_KEY}") if APPS_GROUP in payload.values: if problem := apps_problem(payload.values[APPS_GROUP]): raise HTTPException(422, problem) @@ -917,9 +927,9 @@ async def put_settings( if found := await _provider_problem(session, payload.values): raise CodedHTTPException(422, found[1], PROVIDER_INVALID, field=found[0]) for key, value in payload.values.items(): - if key in (TRUST_KEY, "limits", JUDGE_SPEND, VERIFY_SPEND, *_hidden_groups()): + if key in (TRUST_KEY, "limits", PUBLIC_URL_GROUP, JUDGE_SPEND, VERIFY_SPEND, *_hidden_groups()): # 信任三档只走 PUT /api/tools/trust:设置页整组回写时带着一份旧的, - # 会把刚在工具页改的档位冲掉。limits 是环境变量定的,写进库也不生效。 + # 会把刚在工具页改的档位冲掉。limits 是环境变量定的、public_url 是算出来的,写进库也不生效。 # 裁判、核查的每日计数是引擎记的账,只有引擎自己写。登录设置只经 /api/auth/settings 改(_hidden_groups) continue row = await session.get(Setting, key) @@ -928,6 +938,9 @@ async def put_settings( else: session.add(Setting(key=key, value=value)) await session.commit() + if NOTIFY_GROUP in payload.values: + # 中间件、单点登录的回调按进程缓存取对外访问地址:写完库就刷新,不用重启 + public_url.remember(payload.values[NOTIFY_GROUP][public_url.SETTING_KEY]) return await get_settings(session) diff --git a/backend/app/auth/middleware.py b/backend/app/auth/middleware.py index 05da0def..b3e44a2a 100644 --- a/backend/app/auth/middleware.py +++ b/backend/app/auth/middleware.py @@ -19,7 +19,6 @@ from __future__ import annotations import json from http.cookies import SimpleCookie from typing import Any -from urllib.parse import urlsplit from app.core.config import settings @@ -50,10 +49,12 @@ def _cookie(headers: dict[str, str], name: str) -> str | None: def _origin_ok(origin: str, headers: dict[str, str], scope: dict[str, Any]) -> bool: + from app.core.public_url import origin_of, public_base_url + allowed = {o.rstrip("/") for o in settings.cors_origins} - if settings.public_url: - parts = urlsplit(settings.public_url) - allowed.add(f"{parts.scheme}://{parts.netloc}") + # 对外访问地址(环境变量或设置里的那一项,core/public_url.py):反向代理后面请求的 Host 不一定是它 + if address := public_base_url(): + allowed.add(origin_of(address)) host = headers.get("x-forwarded-host") or headers.get("host") if host: scheme = headers.get("x-forwarded-proto") or scope.get("scheme") or "http" diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 27533ecd..955e7242 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -64,8 +64,9 @@ class Settings(BaseSettings): auth_session_max_days: int = 7 # 会话 cookie 带不带 Secure。不设时按请求本身是不是 https 定:本机 http 开发照常能登录 auth_cookie_secure: bool | None = None - # 对外的访问地址(反向代理后面时填,如 https://agentlab.example.com)。OIDC 的回调地址按它拼; - # 不设时按请求的 Host 拼 + # 对外的访问地址(反向代理后面时填,如 https://agentlab.example.com)。全站只认一个(core/public_url.py): + # 设了它,设置里的「对外访问地址」就不生效、不能改;通知和 Webhook 的链接、OIDC 的回调、来源核对都按它拼。 + # 不设时用设置里的那一项,再没有就按请求的 Host 拼(通知里不附链接) public_url: str = "" # --- 安全响应头(app/web_security.py)--- @@ -146,6 +147,19 @@ class Settings(BaseSettings): return [part.strip() for part in text.split(",") if part.strip()] return value + @field_validator("public_url") + @classmethod + def _valid_public_url(cls, value: str) -> str: + # 写错了启动时就报出来:以前拼出 agentlab.example.com/api/auth/oidc/callback 这样的回调,到登录时才发现 + from app.core.public_url import ENV_NAME, normalize, url_problem + + if not value.strip(): + return "" + if problem := url_problem(value): + what = "不能带 ? 或 # 之后的部分" if problem == "suffix" else "需要是 http:// 或 https:// 开头的完整地址" + raise ValueError(f"{ENV_NAME}={value!r} {what},例如 https://agentlab.example.com") + return normalize(value) or "" + @field_validator("frame_ancestors", "embed_origins") @classmethod def _valid_origins(cls, value: list[str], info: Any) -> list[str]: diff --git a/backend/app/core/public_url.py b/backend/app/core/public_url.py new file mode 100644 index 00000000..1bd5a2cc --- /dev/null +++ b/backend/app/core/public_url.py @@ -0,0 +1,97 @@ +"""对外访问地址:别人打开 AgentLab 用的那个地址(https://agentlab.example.com),全站只有这一个说法(04 细案 B4)。 + +用到它的地方:通知和定时任务 Webhook 里的链接、单点登录的回调地址、非 GET 请求的来源核对(auth/middleware.py); +之后的分享链接、接口说明也从这里取。服务端不知道自己在反向代理后面叫什么,只能由人告诉它。 + +取值(public_base_url): +- 环境变量 AGENTLAB_PUBLIC_URL 优先:部署时定死,设置页上显示「由环境变量指定」、不能改(同登录方式的 env_locked); +- 没设时取设置里 notify 一组的 public_base_url:沿用这个键,界面上叫「对外访问地址」,在「设置 → 通用」里改; +- 都没有是 None:通知里不附链接,单点登录的回调和来源核对按请求自己的 Host 认。 + +以前两处各管各的:环境变量只管回调和来源核对、设置只管通知里的链接——只配了环境变量的部署,通知里不带链接;只在设置里 +填了的,回调又按请求的 Host 拼。违背「一种说法」。 + +设置的值缓存在进程里:中间件按请求核对来源,不能每次查库。启动时读一次(main._platform_startup),改设置的接口写完库就 +remember;手里本来就有会话的异步路径(通知、定时任务 Webhook)用 refresh 取,顺手按库里的值对齐缓存。和登录设置的缓存一样 +以单进程部署为前提。 + +这个模块不在顶层导入 core/config:config 的校验器要用这里的 url_problem,顶层互相导入会在启动时绕成环。 +""" +from __future__ import annotations + +from typing import Any +from urllib.parse import urlsplit + +#: 设置里存它的那一组、那一项(api/settings.py 的 NOTIFY_GROUP 就是这一组) +SETTING_GROUP = "notify" +SETTING_KEY = "public_base_url" +#: 环境变量的名字(core/config.Settings.public_url 按 AGENTLAB_ 前缀读它) +ENV_NAME = "AGENTLAB_PUBLIC_URL" + +_stored: str | None = None + + +def url_problem(value: str) -> str | None: + """地址的形状有什么问题:None 是没问题;"form" 不是 http:// 或 https:// 开头的完整地址;"suffix" 带了 ? 或 # + (链接要接在地址后面)。设置接口按它出中文的说法,环境变量按它在启动时报错。可以带路径(反向代理挂在子路径下)。""" + text = value.strip() + try: + parts = urlsplit(text) + except ValueError: + return "form" + if parts.scheme not in ("http", "https") or not parts.hostname or " " in text: + return "form" + if parts.query or parts.fragment: + return "suffix" + return None + + +def normalize(value: Any) -> str | None: + """存着的(或刚填的)值 → 去掉首尾空白和末尾的 /;空的、形状不对的当没填(None)。末尾带 / 的地址拼链接会多出一个斜杠。""" + if not isinstance(value, str) or not value.strip() or url_problem(value): + return None + return value.strip().rstrip("/") + + +def env_url() -> str | None: + """环境变量给的地址;没设为 None。形状在启动时由 config 的校验器查过、末尾的 / 去掉了。""" + from app.core.config import settings + + return settings.public_url or None + + +def env_locked() -> bool: + """环境变量设了:设置里的那一项不生效、不能改。""" + return env_url() is not None + + +def public_base_url() -> str | None: + """对外访问地址:环境变量优先,其次设置(进程内缓存),都没有是 None。不查库,中间件里也能调。""" + return env_url() or _stored + + +def remember(value: Any) -> None: + """改设置的接口写完库之后调:缓存换成新存的值(形状不对、空的当没填)。""" + global _stored + _stored = normalize(value) + + +async def refresh(session: Any = None) -> str | None: + """按库里存的设置对齐缓存,交回对外访问地址。启动时、手里有会话的异步路径用。""" + from app.db.base import SessionLocal + from app.db.models import Setting + + if session is None: + async with SessionLocal() as own: + row = await own.get(Setting, SETTING_GROUP) + else: + row = await session.get(Setting, SETTING_GROUP) + value = row.value if row is not None else None + remember(value.get(SETTING_KEY) if isinstance(value, dict) else None) + return public_base_url() + + +def origin_of(url: str) -> str: + """地址里的「协议://主机[:端口]」:来源核对按它比。""" + parts = urlsplit(url) + return f"{parts.scheme}://{parts.netloc}" diff --git a/backend/app/main.py b/backend/app/main.py index 62cfea6f..a794593d 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -146,16 +146,20 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: async def _platform_startup() -> None: - """应用平台的启动整理(BE-1):登录设置读进缓存;自定义工具请求头、MCP env 和请求头里的存量明文加密一遍(幂等); - 存量的内网 MCP 服务改为绑定登记端点(只在升级后第一次启动时做);能力登记同步一次。后三样出错只记日志:最坏是登记 - 暂时不全,不该让服务起不来。""" + """应用平台的启动整理(BE-1):登录设置、对外访问地址读进缓存;自定义工具请求头、MCP env 和请求头里的存量明文加密 + 一遍(幂等);存量的内网 MCP 服务改为绑定登记端点(只在升级后第一次启动时做);能力登记同步一次。后三样出错只记 + 日志:最坏是登记暂时不全,不该让服务起不来。""" from app.auth import current_mode, load_settings from app.capabilities import registry as capability_registry from app.capabilities.secrets import seal_existing + from app.core import public_url from app.db.base import SessionLocal await load_settings(force=True) logger.info("登录方式:%s", current_mode()) + # 对外访问地址(core/public_url.py)同样读进缓存:中间件核对来源时不查库 + if address := await public_url.refresh(): + logger.info("对外访问地址:%s(%s)", address, "环境变量" if public_url.env_locked() else "设置") try: if sealed := await seal_existing(): logger.info("已加密 %d 处存量的明文密钥(自定义工具请求头、MCP 环境变量和请求头)", sealed) diff --git a/backend/app/notify/service.py b/backend/app/notify/service.py index a836574a..466a3730 100644 --- a/backend/app/notify/service.py +++ b/backend/app/notify/service.py @@ -7,7 +7,7 @@ 同一个渠道只推一次。待办改派、升级、催办都是「有一条待办要你处理」,订阅了 task.opened 的渠道一并收到。 消息里只放标题、状态、链接(P4,沿用定时任务 Webhook 的规矩):不放报告里的数字,待办的标题用节点在图上的名字 -(搭图时起的),不用按运行数据渲染出来的那一句。链接要设了对外访问地址(设置里的 notify)才有。 +(搭图时起的),不用按运行数据渲染出来的那一句。链接要有对外访问地址(环境变量或设置,core/public_url.py)才有。 推送放在后台(spawn):接收端挂住时超时加重试要几十秒,不能拖着开待办、运行收尾。测试用 drain 等它们推完。 """ @@ -91,9 +91,10 @@ def generic_body(message: dict[str, Any], msg: Message) -> dict[str, Any]: async def _base_url(session: Any) -> str | None: - from app.api.settings import notify_settings + """拼链接用的对外访问地址:全站同一个(环境变量优先,其次设置),手里有会话就按库里的值对齐缓存。""" + from app.core import public_url - return (await notify_settings(session))["public_base_url"] + return await public_url.refresh(session) async def _channels(session: Any, event: str, run: Run | None, users: list[str]) -> list[NotifyChannel]: diff --git a/backend/app/scheduler/webhook.py b/backend/app/scheduler/webhook.py index 4a19e1c6..1a9c690d 100644 --- a/backend/app/scheduler/webhook.py +++ b/backend/app/scheduler/webhook.py @@ -395,9 +395,10 @@ async def notify_due(now: datetime | None = None) -> int: .order_by(ScheduleFire.created_at) )).all() if rows: - from app.api.settings import notify_settings + # 链接里的地址:全站同一个对外访问地址(环境变量优先,其次设置,core/public_url.py) + from app.core import public_url - base_url = (await notify_settings(session))["public_base_url"] + base_url = await public_url.refresh(session) for fire, run in rows: seen = await _state_of(session, fire, run, now) if seen.state is None or seen.state == fire.notified_status: diff --git a/backend/tests/test_public_url.py b/backend/tests/test_public_url.py new file mode 100644 index 00000000..c13667dc --- /dev/null +++ b/backend/tests/test_public_url.py @@ -0,0 +1,169 @@ +"""对外访问地址只有一个说法(04 细案 B4,core/public_url.py)。 + +以前有两个:环境变量 AGENTLAB_PUBLIC_URL 只管单点登录的回调和来源核对,设置里 notify 一组的 public_base_url 只管通知里的 +链接——只配了环境变量的部署,通知里不带链接。现在全站经 public_base_url() 取: + +- 环境变量优先,其次设置,都没有是 None; +- 设置的值缓存在进程里(中间件按请求核对来源,不能每次查库),改设置的接口写完就刷新,不用重启; +- 环境变量设了:设置页显示「由环境变量指定」(GET /api/settings 的 public_url 一组),改成别的值回 409、带 code; +- 通知、定时任务 Webhook 的链接,单点登录的回调地址,非 GET 请求的 Origin 核对都用它。 +""" +from __future__ import annotations + +import pytest +from httpx import ASGITransport, AsyncClient +from sqlalchemy import delete + +from app.api.settings import NOTIFY_GROUP +from app.core import public_url +from app.core.config import Settings, settings +from app.db.base import SessionLocal +from app.db.models import Setting +from app.main import app as main_app +from tests.fakes import auth_env + +ENV_URL = "https://agentlab.example.com" +SET_URL = "https://intra.example.net/agentlab" + + +@pytest.fixture(autouse=True) +async def clean(monkeypatch): + monkeypatch.setattr(settings, "public_url", "") + async with SessionLocal() as session: + await session.execute(delete(Setting).where(Setting.key == NOTIFY_GROUP)) + await session.commit() + public_url.remember(None) + yield + async with SessionLocal() as session: + await session.execute(delete(Setting).where(Setting.key == NOTIFY_GROUP)) + await session.commit() + public_url.remember(None) + + +def client() -> AsyncClient: + return AsyncClient(transport=ASGITransport(app=main_app), base_url="http://test") + + +async def put_address(c: AsyncClient, value: str | None): + return await c.put("/api/settings", json={"values": {NOTIFY_GROUP: {"public_base_url": value}}}) + + +# ---------------------------------------------------------------- 取值的先后 + + +def test_env_wins_over_the_setting(monkeypatch): + public_url.remember(SET_URL) + assert public_url.public_base_url() == SET_URL and not public_url.env_locked() + monkeypatch.setattr(settings, "public_url", ENV_URL) + assert public_url.public_base_url() == ENV_URL and public_url.env_locked() + public_url.remember(None) + assert public_url.public_base_url() == ENV_URL, "设置清空了也还是环境变量的" + monkeypatch.setattr(settings, "public_url", "") + assert public_url.public_base_url() is None, "两处都没有:None(通知里不附链接,回调按请求的 Host 拼)" + + +def test_remember_normalizes_and_drops_broken_values(): + public_url.remember(" https://agentlab.example.com/ ") + assert public_url.public_base_url() == "https://agentlab.example.com" + for broken in ("agentlab.example.com", "ftp://files.example.com", "https://a.example.com/?x=1", 8080, ""): + public_url.remember(broken) + assert public_url.public_base_url() is None, broken + + +def test_env_value_is_normalized_and_checked_at_startup(monkeypatch): + monkeypatch.setenv("AGENTLAB_PUBLIC_URL", " https://agentlab.example.com/ ") + assert Settings().public_url == "https://agentlab.example.com" + monkeypatch.setenv("AGENTLAB_PUBLIC_URL", "") + assert Settings().public_url == "" + # 写错了启动时就报出来:以前拼出 agentlab.example.com/api/auth/oidc/callback 这样的回调,到登录时才发现 + for broken in ("agentlab.example.com", "https://agentlab.example.com/?from=env"): + monkeypatch.setenv("AGENTLAB_PUBLIC_URL", broken) + with pytest.raises(ValueError, match="AGENTLAB_PUBLIC_URL"): + Settings() + + +async def test_refresh_reads_the_stored_setting(): + async with SessionLocal() as session: + session.add(Setting(key=NOTIFY_GROUP, value={"public_base_url": SET_URL + "/"})) + await session.commit() + assert public_url.public_base_url() is None, "缓存还没对齐" + assert await public_url.refresh() == SET_URL + assert public_url.public_base_url() == SET_URL + + +# ---------------------------------------------------------------- 设置接口 + + +async def test_saving_the_setting_refreshes_the_cache_without_restart(): + async with client() as c: + assert (await put_address(c, SET_URL + "/")).status_code == 200 + assert public_url.public_base_url() == SET_URL + body = (await c.get("/api/settings")).json() + assert body["public_url"] == {"value": SET_URL, "env_locked": False} + assert (await put_address(c, None)).status_code == 200 + assert public_url.public_base_url() is None + + +async def test_env_locks_the_setting(monkeypatch): + monkeypatch.setattr(settings, "public_url", ENV_URL) + async with client() as c: + body = (await c.get("/api/settings")).json() + assert body["public_url"] == {"value": ENV_URL, "env_locked": True} + assert body[NOTIFY_GROUP] == {"public_base_url": None}, "notify 一组照旧是库里存的那一份" + r = await put_address(c, SET_URL) + assert r.status_code == 409, r.text + assert r.json()["code"] == "public_url_env_locked" and r.json()["field"] == "notify.public_base_url" + assert "AGENTLAB_PUBLIC_URL" in r.json()["detail"] + # 没改这一项(整组原样写回)不算改:不挡同一请求里别的设置 + assert (await put_address(c, None)).status_code == 200 + # 只读的 public_url 一组写不进库 + r = await c.put("/api/settings", json={"values": {"public_url": {"value": SET_URL, "env_locked": False}}}) + assert r.status_code == 200 + assert (await c.get("/api/settings")).json()["public_url"] == {"value": ENV_URL, "env_locked": True} + async with SessionLocal() as session: + assert await session.get(Setting, "public_url") is None + + +# ---------------------------------------------------------------- 用到它的地方 + + +async def test_notifications_link_with_the_env_address_alone(monkeypatch): + """只配了环境变量的部署,通知里也带链接(以前只认设置里的那一项)。""" + from app.notify import service + + monkeypatch.setattr(settings, "public_url", ENV_URL) + async with SessionLocal() as session: + assert await service._base_url(session) == ENV_URL + monkeypatch.setattr(settings, "public_url", "") + async with SessionLocal() as session: + session.add(Setting(key=NOTIFY_GROUP, value={"public_base_url": SET_URL})) + await session.commit() + async with SessionLocal() as session: + assert await service._base_url(session) == SET_URL + + +async def test_oidc_callback_uses_the_one_address(monkeypatch): + from starlette.requests import Request + + from app.api.auth import _redirect_uri + + request = Request({"type": "http", "method": "GET", "path": "/api/auth/oidc/start", "headers": [], + "scheme": "http", "server": ("10.0.0.5", 8000), "query_string": b""}) + assert _redirect_uri(request) == "http://10.0.0.5:8000/api/auth/oidc/callback", "都没有:按请求的地址拼" + public_url.remember(SET_URL) + assert _redirect_uri(request) == f"{SET_URL}/api/auth/oidc/callback", "只在设置里填了的也按它拼" + monkeypatch.setattr(settings, "public_url", ENV_URL) + assert _redirect_uri(request) == f"{ENV_URL}/api/auth/oidc/callback" + + +async def test_origin_check_accepts_the_one_address(): + await auth_env.reset() + try: + c = await auth_env.turn_on_local() + r = await c.post("/api/groups", json={"name": "ops"}, headers={"Origin": "https://intra.example.net"}) + assert r.status_code == 403 and r.json()["code"] == "csrf_origin" + public_url.remember(SET_URL) + r = await c.post("/api/groups", json={"name": "ops"}, headers={"Origin": "https://intra.example.net"}) + assert r.status_code == 201, r.text + finally: + await auth_env.reset() diff --git a/deploy/agentlab.env b/deploy/agentlab.env index 3a411c0b..d121040b 100644 --- a/deploy/agentlab.env +++ b/deploy/agentlab.env @@ -23,7 +23,9 @@ BIND_ADDR=0.0.0.0 # 不能再改登录方式。local:第一次打开登录页时建第一个管理员(只在还没有管理员时可用);oidc 的身份提供方地址、 # 客户端、管理员映射在「设置 → 身份」里配。忘了密码:docker exec -it agentlab python -m app.auth reset-admin # AGENTLAB_AUTH_MODE=local -# 对外的地址(在反向代理后面时填):OIDC 的回调地址按它拼,非 GET 请求的 Origin 核对也认它 +# 对外访问地址(在反向代理后面时填):通知和定时任务 Webhook 里的链接、OIDC 的回调地址都按它拼,非 GET 请求的 +# Origin 核对也认它。设了这一项就以它为准,界面上「设置 → 通用」里的「对外访问地址」显示「由环境变量指定」、不能改; +# 不设时用界面上填的那一项。写错了(没有 http:// 或 https://、带 ? 或 #)服务启动时报错 # AGENTLAB_PUBLIC_URL=https://agentlab.example.internal # 会话:闲置多少小时、最长多少天过期 # AGENTLAB_AUTH_SESSION_IDLE_HOURS=12 From dd002346227d0d27546c79c83f75d7fef74338b0 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Fri, 16 Oct 2026 10:18:12 +0800 Subject: [PATCH 09/39] =?UTF-8?q?=E7=B3=BB=E7=BB=9F=E6=B4=BE=E7=94=9F?= =?UTF-8?q?=E7=9A=84=E8=BF=90=E8=A1=8C=E4=B8=8D=E8=BF=9B=E4=BD=BF=E7=94=A8?= =?UTF-8?q?=E7=BB=9F=E8=AE=A1=EF=BC=88=E6=80=BB=E6=96=B9=E6=A1=88=20X1?= =?UTF-8?q?=E3=80=81X2=EF=BC=8C=E7=BB=86=E6=A1=88=2001=20=E5=81=9A?= =?UTF-8?q?=E6=B3=95=204=EF=BC=89=EF=BC=9A=E8=AF=84=E6=B5=8B=E4=B8=80?= =?UTF-8?q?=E8=B7=91=EF=BC=8C=E8=81=9A=E7=B1=BB=E3=80=81=E7=99=BD=E5=90=8D?= =?UTF-8?q?=E5=8D=95=E3=80=81=E7=9B=AE=E5=BD=95=E4=BD=BF=E7=94=A8=E7=83=AD?= =?UTF-8?q?=E5=BA=A6=E9=83=BD=E4=BC=9A=E8=A2=AB=E5=88=B7=E9=AB=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - engine/origin.py:SYSTEM_ORIGINS = (eval, reconcile, recall) 和 counted_runs()(写法同 hidden_runs,coalesce 防 NULL:来源为空的老运行、外连接没对上运行的行照常计入);ledger_issue 是替用户正式出具,照常计入 - 取问题、聚类全站只留一份:新增 app/chat/questions.py(recent_questions / cluster_questions)。只取问题原文、不取整份输入(以前把含 history 的 input JSON 拿去聚,同一段对话里的不同问题会聚成一类);问数据按轮次一轮一条(conversation_turns.question),同一轮里的自动重来只是换了挂着的运行,不多算;别的探索运行取 input.question;排除系统运行和别人的私有运行。/governance/exploratory-clusters 改用它,多给 turn_ids 和 embedder(本地哈希向量的结果只是下限) - 工具白名单衰减、目录的使用次数和「被哪些运行用过」、审阅队列的使用权重(都走 catalog.query_snapshot_tables)只算 counted_runs 圈进来的运行 - 记录页:没按来源筛时默认不列这三类,按来源点名就列;新增 GET /api/runs/unlisted(同一份筛选,_filtered 和列表共用)照实给出没列出几次、各是哪种来源。界面在列表底下(空列表时也写)写「另有 N 次系统运行未列出(评测 2 次、口径对账 1 次)」,来源下拉的「全部」改说「全部来源(不含系统运行)」;文案进 lib/terms.ts 的 SYSTEM_RUNS_TEXT;check-runs 补一段 - test_personal_apps 里代表「别的探索运行」的两条运行改成画布试跑:问数据的问题按轮次取,没挂在任何一轮上的问数据运行不单独算 - 新增 tests/test_eval_origin.py(去掉 counted_runs 的过滤回跑,5 项都会失败) --- backend/app/api/governance.py | 71 +++----- backend/app/api/runs.py | 50 +++++- backend/app/chat/__init__.py | 5 + backend/app/chat/questions.py | 115 ++++++++++++ backend/app/data/catalog.py | 7 +- backend/app/engine/origin.py | 18 ++ backend/tests/test_eval_origin.py | 232 +++++++++++++++++++++++++ backend/tests/test_personal_apps.py | 5 +- frontend/src/api/client.ts | 9 + frontend/src/lib/terms.ts | 15 +- frontend/src/pages/RunsPage.tsx | 5 +- frontend/src/pages/runs/RunList.tsx | 54 ++++-- frontend/src/pages/runs/useRunsData.ts | 27 ++- frontend/src/typesApp.ts | 9 + scripts/check-runs.mjs | 20 +++ 15 files changed, 570 insertions(+), 72 deletions(-) create mode 100644 backend/app/chat/__init__.py create mode 100644 backend/app/chat/questions.py create mode 100644 backend/tests/test_eval_origin.py diff --git a/backend/app/api/governance.py b/backend/app/api/governance.py index 710e1707..a5a73074 100644 --- a/backend/app/api/governance.py +++ b/backend/app/api/governance.py @@ -1,6 +1,5 @@ from __future__ import annotations -import json from collections import defaultdict from typing import Any @@ -45,11 +44,14 @@ async def tool_usage( merged.extend(agent.get("tools") or []) granted[node.id] = sorted(set(merged)) + from app.engine.origin import counted_runs + + # 只算计入统计的运行(engine/origin.counted_runs):评测、口径对账、召回替人跑的运行用到了哪些工具不说明白名单该不该收 run_ids = [ rid for (rid,) in await session.execute( select(Run.id) - .where(Run.workflow_id == workflow_id) + .where(Run.workflow_id == workflow_id, counted_runs()) .order_by(Run.created_at.desc()) .limit(limit_runs) ) @@ -95,62 +97,39 @@ async def exploratory_clusters( ) -> dict[str, Any]: """探索性问题聚类:同一个问题被反复问,就该长成模板里的固定一节。 - 这就是模板生长的触发器。用本地 embedding 做贪心聚类, - 出现 ≥3 次的簇标记为晋升候选。 + 这就是模板生长的触发器。取问题、聚类全站只有一份(app/chat/questions.py):只取问题原文(不含 history),问数据一轮 + 一条(自动重来不多算),评测这类系统派生的运行、别人的个人应用刷新和个人蓝图试运行都不进。出现 ≥3 次的簇标记为晋升 + 候选。embedder 是聚类用的向量模型:本地哈希的语义能力弱,结果只是下限。 """ - from sqlalchemy import not_ - from app.api.deps import current_principal - from app.auth.access import hidden_runs - from app.memory.embeddings import cosine, embed_texts + from app.chat.questions import cluster_questions, embedder, recent_questions - rows = list( - await session.execute( - select(Run.id, Run.input, Run.workflow_name, Run.created_at) - # 别人的个人应用刷新、个人蓝图试运行只给拥有者(同运行列表),不进聚类 - .where(Run.run_class != "formal", not_(hidden_runs(current_principal()))) - .order_by(Run.created_at.desc()) - .limit(limit) - ) - ) - items = [] - for rid, input_payload, wf_name, created in rows: - text = json.dumps(input_payload or {}, ensure_ascii=False, sort_keys=True) - if len(text) <= 4: # 空输入没有聚类意义 - continue - items.append({"run_id": rid, "text": text, "workflow": wf_name, - "at": created.isoformat() if created else None}) - if not items: - return {"clusters": [], "considered": 0} + questions = await recent_questions(session, current_principal(), limit=limit) + if not questions: + return {"clusters": [], "considered": 0, "embedder": embedder()} + clusters = await cluster_questions(questions, threshold=threshold) - vectors = await embed_texts([i["text"] for i in items]) - clusters: list[dict[str, Any]] = [] - for i, item in enumerate(items): - placed = False - for cluster in clusters: - if cosine(vectors[i], vectors[cluster["seed"]]) >= threshold: - cluster["members"].append(item) - placed = True - break - if not placed: - clusters.append({"seed": i, "members": [item]}) + def _at(value: Any) -> str | None: + return value.isoformat() if value else None out = [ { - "count": len(c["members"]), - "representative": c["members"][0]["text"][:200], - "workflow": c["members"][0]["workflow"], - "run_ids": [m["run_id"] for m in c["members"]][:20], - "first_at": c["members"][-1]["at"], - "last_at": c["members"][0]["at"], - "promote_candidate": len(c["members"]) >= 3, + "count": c.count, + "representative": c.members[0].text[:200], + "workflow": c.members[0].workflow, + "run_ids": [m.run_id for m in c.members if m.run_id][:20], + "turn_ids": [m.turn_id for m in c.members if m.turn_id][:20], + "first_at": _at(c.members[-1].at), + "last_at": _at(c.members[0].at), + "promote_candidate": c.count >= 3, } for c in clusters - if len(c["members"]) >= 2 + if c.count >= 2 ] out.sort(key=lambda c: c["count"], reverse=True) return { "clusters": out, - "considered": len(items), + "considered": len(questions), + "embedder": embedder(), "hint": "被标记为可固化的问题反复出现,建议用「提取模板」把它固定为工作流中的一步", } diff --git a/backend/app/api/runs.py b/backend/app/api/runs.py index 3a35364e..c6b38a3e 100644 --- a/backend/app/api/runs.py +++ b/backend/app/api/runs.py @@ -171,13 +171,54 @@ async def list_runs( 以前只有 limit:待审批的运行排在第 145 位,列表只取 100 条,点了徽标也找不到它。 筛选都走服务端,翻页用 created_at 做游标(before 传上一页最后一条的 created_at)。 别人的个人应用刷新、个人蓝图试运行不列(只给拥有者,auth/access.hidden_runs),管理员也一样。 + 没按来源筛时,系统派生的运行(评测、口径对账、口径召回:engine/origin.SYSTEM_ORIGINS)默认不列——几十道评测题的运行 + 会把真实用户的运行挤出第一页;按来源点名就列。没列出几次由 GET /api/runs/unlisted 照实说,界面写在列表底下。 """ + stmt = _filtered(select(Run), workflow_id=workflow_id, status=status, run_class=run_class, q=q, + schedule_id=schedule_id, origin_kind=origin_kind) + stmt = stmt.order_by(Run.created_at.desc()).limit(limit) + if before is not None: + stmt = stmt.where(Run.created_at < before) + return list((await session.execute(stmt)).scalars()) + + +@router.get("/unlisted") +async def unlisted_runs( + workflow_id: str | None = Query(default=None), + status: str | None = Query(default=None), + run_class: str | None = None, + q: str | None = Query(default=None), + schedule_id: str | None = Query(default=None), + origin_kind: str | None = Query(default=None), + session: AsyncSession = Depends(get_session), +) -> dict[str, Any]: + """同样的筛选下,运行列表因为「系统派生的运行默认不列」而没列出的有几次:{total, by_origin: {来源: 次数}}。 + + 按来源筛了的(origin_kind 给了)什么都不藏,total 为 0。别人的个人应用刷新、个人蓝图试运行不算在内:它们不是「没列出」, + 是读不了。""" + from sqlalchemy import func + + from app.engine.origin import SYSTEM_ORIGINS + + if _csv(origin_kind): + return {"total": 0, "by_origin": {}} + stmt = _filtered(select(Run.origin_kind, func.count()), workflow_id=workflow_id, status=status, run_class=run_class, + q=q, schedule_id=schedule_id, origin_kind=",".join(SYSTEM_ORIGINS)) + rows = (await session.execute(stmt.group_by(Run.origin_kind))).all() + by_origin = {kind: int(n) for kind, n in rows if kind and n} + return {"total": sum(by_origin.values()), "by_origin": by_origin} + + +def _filtered(stmt: Any, *, workflow_id: str | None, status: str | None, run_class: str | None, q: str | None, + schedule_id: str | None, origin_kind: str | None) -> Any: + """运行列表的筛选条件(列表和「没列出几次」共用一份)。""" from sqlalchemy import not_ from app.api.deps import current_principal from app.auth.access import hidden_runs + from app.engine.origin import counted_runs - stmt = select(Run).where(not_(hidden_runs(current_principal()))).order_by(Run.created_at.desc()).limit(limit) + stmt = stmt.where(not_(hidden_runs(current_principal()))) if workflow_id == UNSAVED: stmt = stmt.where(Run.workflow_id.is_(None)) elif workflow_id: @@ -193,9 +234,10 @@ async def list_runs( if origins := _csv(origin_kind): kinds = [o for o in origins if o != NO_ORIGIN] stmt = stmt.where(or_(Run.origin_kind.in_(kinds), *([Run.origin_kind.is_(None)] if NO_ORIGIN in origins else []))) - if before is not None: - stmt = stmt.where(Run.created_at < before) - return list((await session.execute(stmt)).scalars()) + else: + # 没点名来源:只列计入统计的运行,系统派生的(评测、口径对账、召回)不列;来源为空的老运行照常列出 + stmt = stmt.where(counted_runs()) + return stmt def _csv(value: str | None) -> list[str]: diff --git a/backend/app/chat/__init__.py b/backend/app/chat/__init__.py new file mode 100644 index 00000000..508c2ca1 --- /dev/null +++ b/backend/app/chat/__init__.py @@ -0,0 +1,5 @@ +"""问数据(一轮:提问 → 建图 → 运行 → 复核)在后端的部分。 + +- turn.py:一轮里的几个「决定」(要不要运行、输入怎么填、要不要带原图重来、重来时说什么),问数据页和评测走同一套; +- questions.py:问过的问题原文和它们的聚类,全站只有这一份(治理接口、以后的口径账本、评测都调它)。 +""" diff --git a/backend/app/chat/questions.py b/backend/app/chat/questions.py new file mode 100644 index 00000000..6e7e9201 --- /dev/null +++ b/backend/app/chat/questions.py @@ -0,0 +1,115 @@ +"""问过的问题和它们的聚类:全站只有这一份(docs/design/next-round/00-总方案.md X2)。 + +同一个问题被反复问,就该长成模板里的固定一节、沉淀成口径——探索问题聚类(/governance/exploratory-clusters)、以后口径 +账本的「可沉淀的问题」、评测的问题重复度(E3)量的都是这件事,各算一份的话几个数对不上,所以取问题和聚类只写在这里。 + +**取什么。** 只取问题原文,不取整份输入:以前把运行的整份 input(含 history)拿去聚,同一段对话里前几轮的问答会把不同的 +问题聚成一类,重复度偏高。 + +- 问数据:一轮一条(conversation_turns.question,用户原本问的那句话)。同一轮里的自动重来、用户点的「重试」只是换了这一轮 + 挂着的运行,不多出一条——按运行数的话,答坏了自动重来一次就被算成问了两遍。没挂运行的轮次(只回了一句、助手反问) + 也算:问题照样被问过。 +- 别的探索运行(画布试跑、蓝图试运行、接口……):输入里有 question 这个字段的取它;没有的不收(输入里别的字段不是问题)。 + 问数据的运行由上面一轮一条代表,这里不再数。 + +**不收什么。** 系统派生的运行(评测、口径对账、口径召回:engine/origin.counted_runs)不收,评测题不能把聚类刷高; +这个人读不了的只给拥有者的运行(auth/access.hidden_runs)不收;正式运行不收(探索才谈得上「沉淀」)。 + +聚类用 memory/embeddings 的向量贪心聚:按时间从新到旧,和某一类的种子足够像就并进去,否则自立一类。本地向量是词频 +哈希、语义能力弱,结果是下限——调用方据 embedder 照实写明用的是哪个向量模型。 +""" +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any + +from sqlalchemy import func, not_, select +from sqlalchemy.ext.asyncio import AsyncSession + +#: 缺省的相似度阈值(同治理接口一直以来的 0.62) +DEFAULT_THRESHOLD = 0.62 + + +@dataclass(frozen=True) +class Question: + """问过的一个问题。turn_id 是问数据的那一轮(别的探索运行没有);run_id 是它挂着的运行(只回了一句的轮次没有)。""" + + text: str + at: datetime | None + run_id: str | None = None + turn_id: str | None = None + workflow: str | None = None + origin: str | None = None + + +@dataclass +class QuestionCluster: + """一类问题:members 按时间从新到旧,第一个是种子。""" + + members: list[Question] = field(default_factory=list) + + @property + def count(self) -> int: + return len(self.members) + + +async def recent_questions(session: AsyncSession, principal: Any, *, limit: int) -> list[Question]: + """最近问过的问题,从新到旧,最多 limit 条(取法见模块开头)。principal 用来去掉这个人读不了的只给拥有者的运行。""" + from app.auth.access import hidden_runs + from app.db.models import Conversation, ConversationTurn, Run + from app.engine.origin import counted_runs + + hidden = hidden_runs(principal) + turns = (await session.execute( + select(ConversationTurn.id, ConversationTurn.question, ConversationTurn.created_at, ConversationTurn.run_id, + Run.workflow_name) + .join(Conversation, Conversation.id == ConversationTurn.conversation_id) + .outerjoin(Run, Run.id == ConversationTurn.run_id) + .where(Conversation.kind == "chat", func.trim(ConversationTurn.question) != "", counted_runs(), not_(hidden)) + .order_by(ConversationTurn.created_at.desc()).limit(limit) + )).all() + runs = (await session.execute( + select(Run.id, Run.input, Run.workflow_name, Run.created_at, Run.origin_kind) + .where(Run.run_class != "formal", func.coalesce(Run.origin_kind, "") != "chat", counted_runs(), not_(hidden), + func.json_type(Run.input, "$.question") == "text") + .order_by(Run.created_at.desc()).limit(limit) + )).all() + out = [Question(text=question.strip(), at=at, run_id=run_id, turn_id=turn_id, workflow=workflow, origin="chat") + for turn_id, question, at, run_id, workflow in turns] + for run_id, payload, workflow, at, origin in runs: + text = (payload or {}).get("question") if isinstance(payload, dict) else None + if isinstance(text, str) and text.strip(): + out.append(Question(text=text.strip(), at=at, run_id=run_id, workflow=workflow, origin=origin)) + out.sort(key=lambda q: q.at.timestamp() if q.at is not None else 0.0, reverse=True) + return out[:limit] + + +async def cluster_questions(questions: Sequence[Question], *, threshold: float = DEFAULT_THRESHOLD, + ) -> list[QuestionCluster]: + """向量贪心聚类(顺序同 questions:从新到旧)。""" + from app.memory import embeddings + + if not questions: + return [] + vectors = await embeddings.embed_texts([q.text for q in questions]) + clusters: list[tuple[int, QuestionCluster]] = [] + for i, question in enumerate(questions): + for seed, cluster in clusters: + if embeddings.cosine(vectors[i], vectors[seed]) >= threshold: + cluster.members.append(question) + break + else: + clusters.append((i, QuestionCluster(members=[question]))) + return [cluster for _, cluster in clusters] + + +def embedder() -> str: + """聚类用的是哪个向量模型(memory/embeddings.embedder_id):本地哈希的结果只是下限,调用方照实写出来。""" + from app.memory import embeddings + + return embeddings.embedder_id() + + +__all__ = ["DEFAULT_THRESHOLD", "Question", "QuestionCluster", "cluster_questions", "embedder", "recent_questions"] diff --git a/backend/app/data/catalog.py b/backend/app/data/catalog.py index 6ca00287..3247aea4 100644 --- a/backend/app/data/catalog.py +++ b/backend/app/data/catalog.py @@ -2408,15 +2408,18 @@ async def query_snapshot_tables(session: AsyncSession, source: Any) -> list[tupl 使用次数(table_usage)和「这张表被哪些运行用过」(data/catalog_review.py)共用这一份取法:数据源查询工具每次成功 查询都存一份查询快照工件(tools/datasource.py),工件引用表 artifacts 里按 (内容哈希, 运行) 各记一行;只收属于某次运行 - (runs 里有这次运行)、属于这个源的行,工具库里的试用(不属于任何运行)不算。 + (runs 里有这次运行)、属于这个源的行,工具库里的试用(不属于任何运行)不算;系统派生的运行(评测、口径对账、召回, + engine/origin.counted_runs)也不算——一跑评测,常用表的排序、起草的默认范围、审阅队列的权重就跟着被评测题刷高了。 表名在查询当时就记进了引用的 meta(query_snapshot_meta),这里只读 artifacts 一张表;升级前的老引用没有 meta,退回读 工件内容里的源名和 SQL,读过的按工件 id 缓存(内容永不改变)。SQL 里写的表名去掉 schema 前缀后按 name_key(只把 ASCII 字母转小写,和 SQLite 比较标识符的口径一致)对到表结构上,对不上的不计。 """ + from app.engine.origin import counted_runs + rows = (await session.execute( select(Artifact.id, Artifact.meta, Artifact.run_id).join(Run, Run.id == Artifact.run_id) - .where(Artifact.kind == QUERY_SNAPSHOT_KIND) + .where(Artifact.kind == QUERY_SNAPSHOT_KIND, counted_runs()) )).all() tables = (getattr(source, "schema_cache", None) or {}).get("tables") or {} lookup: dict[str, str] = {} diff --git a/backend/app/engine/origin.py b/backend/app/engine/origin.py index 6e418db8..25081ff7 100644 --- a/backend/app/engine/origin.py +++ b/backend/app/engine/origin.py @@ -34,6 +34,24 @@ ORIGIN_KINDS = ( #: 客户端在 POST /api/runs 里能声明的来源 CLIENT_ORIGINS = ("chat", "canvas", "launcher", "api") +#: 系统派生的运行(docs/design/next-round/00-总方案.md X1):评测、口径对账、口径召回都是服务端为了量、为了比替人跑的, +#: 不是有人在用。它们是完整的运行(证据、封存都在、点得开),但不进使用统计——探索问题聚类、工具白名单衰减、数据目录的 +#: 使用次数和「被哪些运行用过」、审阅队列的使用权重、以后的账本统计——否则一跑评测,治理信号和目录的使用热度就失真了。 +#: 记录页默认也不列(筛选里能选,列表照实写有几次没列出)。ledger_issue 是替用户正式出具,照常计入 +SYSTEM_ORIGINS = ("eval", "reconcile", "recall") + + +def counted_runs() -> Any: + """SQL 条件:计入使用统计的运行(排除 SYSTEM_ORIGINS)。所有使用统计都只算它圈进来的运行,别处不另写一份过滤。 + + 来源按非空比较(coalesce,同 auth/access.hidden_runs):加来源这一列之前的老运行 origin_kind 为空,NOT IN 碰上 NULL + 会把它们一起筛掉;外连接没对上运行的行(问数据只回了一句、没有运行的那一轮)同理照常计入。""" + from sqlalchemy import func + + from app.db.models import Run + + return func.coalesce(Run.origin_kind, "").not_in(SYSTEM_ORIGINS) + def origin_note(kind: str, **data: Any) -> dict[str, Any]: """run.origin 事件的数据:kind 加上调用方给的字段(app_id、app_version、spec_hash、entry / view、hook、 diff --git a/backend/tests/test_eval_origin.py b/backend/tests/test_eval_origin.py new file mode 100644 index 00000000..951b12be --- /dev/null +++ b/backend/tests/test_eval_origin.py @@ -0,0 +1,232 @@ +"""系统派生的运行不进使用统计(docs/design/next-round/00-总方案.md X1,细案 01 做法 4)。 + +评测、口径对账、口径召回(engine/origin.SYSTEM_ORIGINS)是服务端替人跑的完整运行,证据点得开,但不是有人在用。这里守: + +- 客户端声明不了这几种来源;ledger_issue(替用户正式出具)照常计入; +- counted_runs() 只去掉这三种,来源为空的老运行照常计入(防 NULL); +- 探索问题聚类只取问题原文(不含 history),问数据一轮一条(自动重来不多算),评测运行不进; +- 工具白名单衰减、目录的使用次数和「被哪些运行用过」、审阅队列的使用权重都不计评测运行; +- 记录页默认不列这三种,按来源点名就列;GET /api/runs/unlisted 照实说没列出几次。 + +表名、问题、工作流名都是虚构的。 +""" +from __future__ import annotations + +import uuid +from datetime import datetime, timedelta, timezone + +import pytest +from httpx import ASGITransport, AsyncClient +from sqlalchemy import delete, select + +from app.data import catalog, catalog_review +from app.db.base import SessionLocal +from app.db.models import Artifact, Conversation, ConversationTurn, DataSource, Run, RunEvent, Workflow +from app.engine.origin import CLIENT_ORIGINS, ORIGIN_KINDS, SYSTEM_ORIGINS, counted_runs +from app.main import app +from tests.fixtures.sources import drop_source + +BASE = datetime(2026, 9, 1, tzinfo=timezone.utc) +MARK = f"计入统计-{uuid.uuid4().hex[:6]}" + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +@pytest.fixture(autouse=True) +async def _clean(): + yield + async with SessionLocal() as session: + await session.execute(delete(Run).where(Run.workflow_name.like(f"{MARK}%"))) + await session.execute(delete(Conversation).where(Conversation.title.like(f"{MARK}%"))) + await session.commit() + + +async def _run(origin: str | None, *, minutes: int = 0, **extra) -> str: + rid = uuid.uuid4().hex + async with SessionLocal() as session: + session.add(Run(id=rid, graph={}, workflow_name=f"{MARK}-{origin}", run_class="exploratory", status="succeeded", + origin_kind=origin, created_at=BASE + timedelta(minutes=minutes), **extra)) + await session.commit() + return rid + + +def test_system_origins_are_server_only_and_ledger_issue_counts(): + assert set(SYSTEM_ORIGINS) == {"eval", "reconcile", "recall"} + assert set(SYSTEM_ORIGINS) <= set(ORIGIN_KINDS) + assert not set(SYSTEM_ORIGINS) & set(CLIENT_ORIGINS) + assert "ledger_issue" in ORIGIN_KINDS and "ledger_issue" not in SYSTEM_ORIGINS, "替用户正式出具照常计入" + + +async def test_a_client_cannot_claim_a_system_origin(client): + for origin in (*SYSTEM_ORIGINS, "ledger_issue"): + r = await client.post("/api/runs", json={"graph": {"nodes": [], "edges": []}, "origin": origin}) + assert r.status_code == 422 and r.json()["code"] == "origin_not_allowed", (origin, r.text) + + +async def test_counted_runs_keeps_legacy_null_origins(): + ids = {origin: await _run(origin) for origin in (None, "chat", "ledger_issue", *SYSTEM_ORIGINS)} + async with SessionLocal() as session: + counted = set((await session.execute( + select(Run.id).where(Run.id.in_(list(ids.values())), counted_runs()))).scalars()) + assert counted == {ids[None], ids["chat"], ids["ledger_issue"]} + + +# -------------------------------------------------------------------------- +# 探索问题聚类 +# -------------------------------------------------------------------------- + + +async def _turn(conversation_id: str, question: str, run_id: str | None, seq: int, minutes: int) -> str: + async with SessionLocal() as session: + turn = ConversationTurn(conversation_id=conversation_id, seq=seq, question=question, run_id=run_id, + status="done", created_at=BASE + timedelta(minutes=minutes)) + session.add(turn) + await session.commit() + return turn.id + + +async def _conversation() -> str: + async with SessionLocal() as session: + row = Conversation(title=f"{MARK}-对话", kind="chat") + session.add(row) + await session.commit() + return row.id + + +async def test_clusters_take_only_the_question_once_per_turn_and_skip_eval(client, monkeypatch): + from app.memory import embeddings + + question = f"{MARK} 上周各门店的销售额" + + async def by_text(texts): + # 同一句话同一个向量,别的话各不相同:聚类只看取了什么文字 + return [[1.0, 0.0] if t == question else [0.0, float(i + 1)] for i, t in enumerate(texts)] + + monkeypatch.setattr(embeddings, "embed_texts", by_text) + conv = await _conversation() + # 第一轮:答坏了自动重来一次——两次运行,轮次只挂着后一次;输入里的 history 各不相同也不影响 + first = await _run("chat", minutes=1, input={"question": question, "history": "问:别的\n答:别的"}) + retried = await _run("chat", minutes=2, input={"question": question, "history": "问:别的\n答:别的"}) + turn_a = await _turn(conv, question, retried, 1, 2) + # 第二轮:同一个问题又问了一次,这次只回了一句、没有运行 + turn_b = await _turn(conv, question, None, 2, 3) + # 评测作答的运行:同一个问题,不进聚类 + graded = await _run("eval", minutes=4, input={"question": question}) + # 画布试跑:输入里有 question 的照样收 + canvas = await _run("canvas", minutes=5, input={"question": question}) + + body = (await client.get("/api/governance/exploratory-clusters", params={"limit": 500})).json() + mine = [c for c in body["clusters"] if c["representative"] == question] + assert len(mine) == 1, body + cluster = mine[0] + assert cluster["count"] == 3, "一轮一条(自动重来不多算)+ 画布那一次;评测不算" + assert set(cluster["turn_ids"]) == {turn_a, turn_b} + assert set(cluster["run_ids"]) == {retried, canvas} + assert first not in cluster["run_ids"] and graded not in cluster["run_ids"] + assert cluster["promote_candidate"] is True + assert body["embedder"] + + +async def test_the_history_never_reaches_the_clusterer(monkeypatch): + from app.api.deps import current_principal + from app.chat.questions import recent_questions + + conv = await _conversation() + rid = await _run("chat", minutes=10, input={"question": f"{MARK} 本月退款", "history": "问:上周\n答:100"}) + await _turn(conv, f"{MARK} 本月退款", rid, 1, 10) + async with SessionLocal() as session: + got = [q for q in await recent_questions(session, current_principal(), limit=500) if MARK in q.text] + assert [q.text for q in got] == [f"{MARK} 本月退款"] + + +# -------------------------------------------------------------------------- +# 工具白名单衰减 +# -------------------------------------------------------------------------- + + +async def test_tool_usage_ignores_eval_runs(client): + graph = {"nodes": [{"id": "a", "type": "agent", "position": {"x": 0, "y": 0}, + "data": {"label": "a", "config": {"tools": ["calc", "kb_search"]}}}], "edges": []} + async with SessionLocal() as session: + wf = Workflow(name=f"{MARK}-白名单", graph=graph) + session.add(wf) + await session.commit() + wf_id = wf.id + try: + used = await _run("canvas", workflow_id=wf_id) + graded = await _run("eval", workflow_id=wf_id) + async with SessionLocal() as session: + session.add(RunEvent(run_id=used, seq=1, type="tool.start", node_id="a", data={"tool": "calc"}, ts=0.0)) + session.add(RunEvent(run_id=graded, seq=1, type="tool.start", node_id="a", data={"tool": "kb_search"}, ts=0.0)) + await session.commit() + body = (await client.get("/api/governance/tool-usage", params={"workflow_id": wf_id})).json() + assert body["runs_considered"] == 1 + node = body["nodes"][0] + assert node["used"] == {"calc": 1} and node["unused"] == ["kb_search"], "评测用过的工具不能让白名单显得都在用" + finally: + async with SessionLocal() as session: + await session.execute(delete(Workflow).where(Workflow.id == wf_id)) + await session.commit() + + +# -------------------------------------------------------------------------- +# 目录的使用次数、被哪些运行用过、审阅队列的权重 +# -------------------------------------------------------------------------- + + +async def test_catalog_usage_ignores_eval_runs(): + name = f"eo_{uuid.uuid4().hex[:8]}" + tables = {"orders": {"qualified": "orders", "primary_key": ["id"], "comment": None, "is_view": False, + "foreign_keys": [], "unique": [], + "columns": [{"name": "id", "type": "INTEGER", "nullable": True, "comment": None}]}} + async with SessionLocal() as session: + row = DataSource(name=name, kind="sqlite", database=":memory:", schema_cache={"tables": tables}) + session.add(row) + await session.commit() + sid = row.id + try: + used = await _run("chat") + graded = await _run("eval") + legacy = await _run(None) + async with SessionLocal() as session: + for rid in (used, graded, legacy): + session.add(Artifact(id=uuid.uuid4().hex, run_id=rid, kind=catalog.QUERY_SNAPSHOT_KIND, + meta={"source_id": sid, "source": name, "tables": ["orders"]})) + await session.commit() + source = await session.get(DataSource, sid) + assert await catalog.table_usage(session, source) == {"orders": 2} + counts = await catalog_review.table_usage_counts(session, source) + assert counts.queries == {"orders": 2} and counts.runs == {"orders": 2} + listed = await catalog_review.table_runs(session, source, "orders") + assert listed["total"] == 2 and {r["run_id"] for r in listed["runs"]} == {used, legacy} + finally: + await drop_source(sid) + + +# -------------------------------------------------------------------------- +# 记录页 +# -------------------------------------------------------------------------- + + +async def test_the_run_list_hides_system_runs_by_default_and_says_how_many(client): + chat = await _run("chat") + graded = [await _run("eval", minutes=i) for i in range(2)] + recon = await _run("reconcile") + legacy = await _run(None) + q = {"q": MARK, "limit": 200} + + listed = {r["id"] for r in (await client.get("/api/runs", params=q)).json()} + assert {chat, legacy} <= listed and not (set(graded) | {recon}) & listed + unlisted = (await client.get("/api/runs/unlisted", params={"q": MARK})).json() + assert unlisted == {"total": 3, "by_origin": {"eval": 2, "reconcile": 1}} + + # 按来源点名就列,此时没有藏起来的 + only = {r["id"] for r in (await client.get("/api/runs", params={**q, "origin_kind": "eval"})).json()} + assert only == set(graded) + assert (await client.get("/api/runs/unlisted", params={"q": MARK, "origin_kind": "eval"})).json()["total"] == 0 + # 同样的筛选一起作用:按状态筛时只数这个状态的 + assert (await client.get("/api/runs/unlisted", params={"q": MARK, "status": "failed"})).json()["total"] == 0 diff --git a/backend/tests/test_personal_apps.py b/backend/tests/test_personal_apps.py index 8439a8d7..ff25c447 100644 --- a/backend/tests/test_personal_apps.py +++ b/backend/tests/test_personal_apps.py @@ -964,9 +964,10 @@ async def test_the_other_doors_to_a_private_run_are_shut_too(monkeypatch): bob_trial = await wait(r.json()["run_id"]) question = {"question": "上周各门店的销售额"} async with SessionLocal() as session: - # 聚类只列两条以上的簇:别的探索运行放两条 + # 聚类只列两条以上的簇:别的探索运行放两条。用画布试跑:问数据的问题按轮次取(app/chat/questions.py), + # 没挂在任何一轮上的问数据运行不单独算 shared, also = (Run(workflow_name="共享的运行", graph=KPI, status="succeeded", run_class="exploratory", - origin_kind="chat", started_by="bob", input=question) for _ in range(2)) + origin_kind="canvas", started_by="bob", input=question) for _ in range(2)) session.add_all([shared, also]) await session.flush() for run_id in (trial.id, refresh.id, bob_trial.id): diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index 832089de..eafdec5b 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -13,6 +13,7 @@ import type { RunEvent, RunStatus, Skill, ToolChange, ToolInfo, ToolTrust, UpgradeResult, UploadDecision, UploadMixedColumn, UploadResult, UploadShapeReason, ValidationIssue, VarIssue, Variable, Workflow, WorkflowVersion, } from '../types' +import type { UnlistedRuns } from '../typesApp' import { ApiError, BASE, NETWORK_MESSAGE, actorHeader, codeOf, del, describeDetail, failure, get, isGatewayFailure, parseBody, patch, post, put, qs, report, request, upload, type RequestOptions, type UploadOptions, @@ -563,6 +564,14 @@ export const api = { /** 按运行来源筛(X3);none 是来源未记录的老运行 */ origin_kind?: string }) => get(`/runs${qs(params)}`), + /** + * 同样的筛选下,列表因为「系统派生的运行默认不列」而没列出的次数(评测、口径对账、口径召回,next-round X1)。按来源 + * 筛了的什么都不藏,total 为 0。老后端没有这个接口(404),调用方当作 0 + */ + unlisted: (params?: { + workflow_id?: string; status?: RunStatus | RunStatus[] | string; run_class?: 'formal' | 'exploratory' + q?: string; origin_kind?: string + }) => get(`/runs/unlisted${qs(params)}`), get: (id: string) => get(`/runs/${id}`), /** 这次运行实际跑的那张图(运行时的快照),不是工作流现在的样子 */ graph: (id: string) => get<{ graph: GraphSpec; workflow_id: string | null; version: number | null }>( diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index 51fb6dba..fd44bf8d 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -5124,7 +5124,8 @@ export const ORIGIN_TEXT = { unknown: '来源未记录', unknownHint: '这次运行发起时还没有记录来源', hint: '这次运行从哪里发起。发起时写进运行记录,在封存范围内', - filterAll: '全部来源', + // 不点名来源时系统派生的运行(评测、口径对账、口径召回)不列,「全部」得照实说不含它们 + filterAll: '全部来源(不含系统运行)', filterLabel: '按运行来源筛选', /** 时间线里 run.origin 那一行 */ step: (label: string) => `来源:${label}`, @@ -5135,6 +5136,18 @@ export const ORIGIN_TEXT = { actor: (who: string, verified: boolean) => `${verified ? '已登录用户' : '署名'}「${who}」`, } +/** + * 记录页没列出的系统派生运行(评测、口径对账、口径召回:服务端 engine/origin.SYSTEM_ORIGINS,next-round X1)。它们是完整的 + * 运行、不进使用统计,默认不列;照实写有几次没列出、在哪儿能看 + */ +export const SYSTEM_RUNS_TEXT = { + /** parts 是「评测 2 次、口径对账 1 次」 */ + unlisted: (total: number, parts: string) => + `另有 ${total} 次系统运行未列出(${parts}),在来源筛选中选择对应来源即可查看`, + part: (label: string, n: number) => `${label} ${n} 次`, + hint: '评测、口径对账、口径召回是系统代为发起的运行:证据仍可查看,但不计入使用统计,记录页默认不列', +} + /** 来源标签:老运行没有来源时写「来源未记录」 */ export const originLabel = (kind: string | null | undefined): string => kind && kind in ORIGIN_LABEL ? ORIGIN_LABEL[kind as OriginKind] : ORIGIN_TEXT.unknown diff --git a/frontend/src/pages/RunsPage.tsx b/frontend/src/pages/RunsPage.tsx index af6fdf58..10a3f3fc 100644 --- a/frontend/src/pages/RunsPage.tsx +++ b/frontend/src/pages/RunsPage.tsx @@ -18,7 +18,7 @@ import { } from './runs/model' import { ORIGIN_TEXT, TASK_LOG_TEXT, UNSAVED_NAME, originLabel } from '../lib/terms' import { RunTabs, type TabItem } from './runs/parts' -import { useApprovalQueue, useRunList, useStatusCounts, useTaskQueue } from './runs/useRunsData' +import { useApprovalQueue, useRunList, useStatusCounts, useTaskQueue, useUnlistedRuns } from './runs/useRunsData' import './runs/runs.css' /** @@ -95,6 +95,8 @@ export function RunsPage() { const filters = { codes, runClass, workflowId, q: parsed.q, origin } const list = useRunList(filters, tab !== 'approvals' && !impossible) + // 系统派生的运行(评测、口径对账、召回)默认不列:列表底下照实写有几次没列出 + const unlisted = useUnlistedRuns(filters, epoch, tab !== 'approvals' && !impossible) // 计数两份:页签和分段上的跟着当前的类别、工作流、名称条件走;右侧总览说的 // 是全局——筛掉了不代表事情没了 const narrowed = !!(runClass || workflowId || parsed.q || origin) @@ -251,6 +253,7 @@ export function RunsPage() { emptyBody={tab === 'all' ? '在问数据或画布中发起运行后,完整轨迹会记录在这里。' : undefined} filtered={filterNote} onClear={clearFilters} + unlisted={unlisted} /> )} diff --git a/frontend/src/pages/runs/RunList.tsx b/frontend/src/pages/runs/RunList.tsx index 01e9e699..0c0989cc 100644 --- a/frontend/src/pages/runs/RunList.tsx +++ b/frontend/src/pages/runs/RunList.tsx @@ -7,12 +7,12 @@ import { formatCost, formatDateTime, formatDay, formatSpan, formatTime, formatTokens, parseServerTime, shortId, } from '../../lib/format' import { - ORIGIN_LABEL, ORIGIN_TEXT, RUN_CLASS_LABEL, TASK_KIND_LABEL, TASK_LOG_TEXT, UNSAVED_HINT, UNSAVED_NAME, isUnsaved, originLabel, - runName, + ORIGIN_LABEL, ORIGIN_TEXT, RUN_CLASS_LABEL, SYSTEM_RUNS_TEXT, TASK_KIND_LABEL, TASK_LOG_TEXT, UNSAVED_HINT, UNSAVED_NAME, isUnsaved, + originLabel, runName, } from '../../lib/terms' import { summarizeRun } from '../../run/decode' import type { Approval, Run, Workflow } from '../../types' -import type { OriginKind, Task } from '../../typesApp' +import type { OriginKind, Task, UnlistedRuns } from '../../typesApp' import { UNSAVED_WORKFLOW_ID } from '../../api/client' import { explainRunError } from '../../lib/explain' import { @@ -195,7 +195,7 @@ function Segment({ on, onClick, children, data, title }: { // ------------------------------------------------------------------------- export function RunRows({ - list, codeOf, pendingOf, selectedId, onOpen, dupNames, emptyTitle, emptyBody, filtered, onClear, + list, codeOf, pendingOf, selectedId, onOpen, dupNames, emptyTitle, emptyBody, filtered, onClear, unlisted, }: { list: RunList codeOf: (run: Run) => StatusCode @@ -208,6 +208,8 @@ export function RunRows({ /** 带着筛选条件:空了要说"没有匹配",不能说"还没有运行记录" */ filtered: string | null onClear: () => void + /** 没列出的系统派生运行(评测、口径对账、口径召回)。没有或点名了来源时不给 */ + unlisted?: UnlistedRuns | null }) { const changed = useChangedRows(list.rows) const now = useTicker(60_000) @@ -221,18 +223,25 @@ export function RunRows({ return void list.reload()} compact className="m-3" /> } const rows = list.rows + // 空着的时候也照实说有几次没列出:不能让「还没有运行记录」盖住几十次评测运行 + const note = unlistedNote(unlisted) if (!rows.length) { - return filtered - ? ( - } - title="没有匹配的运行" - body={filtered} - offline={false} - action={} - /> - ) - : } title={emptyTitle} body={emptyBody} /> + return ( + <> + {filtered + ? ( + } + title="没有匹配的运行" + body={filtered} + offline={false} + action={} + /> + ) + : } title={emptyTitle} body={emptyBody} />} + {note} + + ) } // 按天分组,组头吸顶:翻到哪天一眼可见 @@ -278,10 +287,25 @@ export function RunRows({ )} {!list.hasMore && rows.length > 0 && · 已全部加载} + {note} ) } +/** 「另有 N 次系统运行未列出(评测 2 次、口径对账 1 次)……」:没有没列出的就不画 */ +function unlistedNote(unlisted: UnlistedRuns | null | undefined): ReactNode { + if (!unlisted?.total) return null + const parts = (Object.keys(ORIGIN_LABEL) as OriginKind[]) + .filter((k) => (unlisted.by_origin[k] ?? 0) > 0) + .map((k) => SYSTEM_RUNS_TEXT.part(ORIGIN_LABEL[k], unlisted.by_origin[k] ?? 0)) + .join('、') + return ( +

+ {SYSTEM_RUNS_TEXT.unlisted(unlisted.total, parts)} +

+ ) +} + /** * 实时刷新时哪几行的状态刚变了:给它们闪一下底色。值是第几次变化,行里那层 * 闪光按它换 key——同一行接连变两次(停到审批上、又被批掉接着跑)会重新挂载、 diff --git a/frontend/src/pages/runs/useRunsData.ts b/frontend/src/pages/runs/useRunsData.ts index e96a3a4a..cc8f8f6e 100644 --- a/frontend/src/pages/runs/useRunsData.ts +++ b/frontend/src/pages/runs/useRunsData.ts @@ -3,7 +3,7 @@ import { api } from '../../api/client' import { toast } from '../../components/ui' import type { StatusCode } from '../../lib/status' import type { Approval, Run } from '../../types' -import type { Task } from '../../typesApp' +import type { Task, UnlistedRuns } from '../../typesApp' import { useCatalog } from '../../store/catalog' import { runCode, serverStatuses } from './model' @@ -123,6 +123,31 @@ export function useRunList(filters: ListFilters, enabled: boolean): RunList { return { rows, state, error, hasMore, loadingMore, reload, loadMore, patchRow, removeRow } } +// ------------------------------------------------------------------------- +// 没列出的系统运行 +// ------------------------------------------------------------------------- + +/** + * 同样的筛选下,列表没列出的系统派生运行(评测、口径对账、口径召回)有几次。点名了来源的什么都不藏,不用问;老后端 + * 没有这个接口时当作 0(不报错:列表本身照常) + */ +export function useUnlistedRuns(filters: ListFilters, epoch: number, enabled: boolean): UnlistedRuns | null { + const [got, setGot] = useState(null) + const params = paramsOf(filters) + const key = JSON.stringify(params) + const ask = enabled && !params.origin_kind + + useEffect(() => { + if (!ask) { setGot(null); return } + let live = true + api.runs.unlisted(JSON.parse(key)) + .then((r) => { if (live) setGot(r) }, () => { if (live) setGot(null) }) + return () => { live = false } + }, [key, epoch, ask]) + + return ask ? got : null +} + // ------------------------------------------------------------------------- // 计数 // ------------------------------------------------------------------------- diff --git a/frontend/src/typesApp.ts b/frontend/src/typesApp.ts index df2b6c2a..949d192a 100644 --- a/frontend/src/typesApp.ts +++ b/frontend/src/typesApp.ts @@ -705,6 +705,15 @@ export type OriginKind = | 'trial' | 'personal_app' | 'eval' | 'ledger_issue' | 'reconcile' | 'recall' +/** + * GET /api/runs/unlisted:同样的筛选下,记录页因为「系统派生的运行默认不列」而没列出的次数。系统派生的是评测、口径对账、 + * 口径召回(服务端 engine/origin.SYSTEM_ORIGINS):不进使用统计,按来源点名才列 + */ +export interface UnlistedRuns { + total: number + by_origin: Partial> +} + /** 查看的对象。caliber 是已发布口径卡上的指标(蓝图助手复用口径时看,服务端 look/calibers.py) */ export interface LookObject { kind: 'table' | 'capability' | 'workflow_version' | 'caliber' diff --git a/scripts/check-runs.mjs b/scripts/check-runs.mjs index 7e82526f..80c41fe3 100644 --- a/scripts/check-runs.mjs +++ b/scripts/check-runs.mjs @@ -2407,6 +2407,26 @@ await section('运行来源:标签与筛选', async () => { check('只剩那一种来源', (await ids()).join(',') === 'fake0origin0canvas', (await ids()).join(',')) await sel.selectOption('') await settle() + // next-round X1:系统派生的运行(评测、口径对账、召回)默认不列,列表底下照实写有几次没列出;点名来源就不再说 + const unlistedAsked = [] + fakes.set('GET /api/runs/unlisted', (url) => { + unlistedAsked.push(url.searchParams.get('origin_kind')) + return { status: 200, json: { total: 3, by_origin: { eval: 2, reconcile: 1 } } } + }) + await page.goto(`${WEB}/runs`, { waitUntil: 'networkidle' }) + await page.waitForTimeout(300) + const note = page.locator('[data-runs-unlisted]') + const noteText = await note.innerText().catch(() => '') + check('列表底下写明另有几次系统运行未列出、各是什么', await note.count() === 1 + && noteText.includes('另有 3 次系统运行未列出') && noteText.includes('评测 2 次') && noteText.includes('口径对账 1 次'), noteText) + check('来源下拉的「全部」照实说不含系统运行', (await sel.locator('option').first().innerText()).includes('不含系统运行')) + await sel.selectOption('eval') + await settle() + check('点名了来源:不再问、不再写没列出', await page.locator('[data-runs-unlisted]').count() === 0 + && !unlistedAsked.includes('eval'), unlistedAsked.join(',')) + await sel.selectOption('') + await settle() + fakes.delete('GET /api/runs/unlisted') fakes.delete('GET /api/runs') if (base) { await page.goto(`${WEB}/runs/${base.id}`, { waitUntil: 'networkidle' }) From e82d7fa55d32a4e088316a14d7fc9a142527dd75 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Fri, 16 Oct 2026 10:19:50 +0800 Subject: [PATCH 10/39] =?UTF-8?q?=E9=9B=86=E6=88=90=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=E8=A1=A5=E9=BD=90=E9=94=99=E8=AF=AF=E7=A0=81=EF=BC=8C=E5=BC=80?= =?UTF-8?q?=E7=99=BB=E5=BD=95=E6=97=B6=E6=94=B6=E8=B5=B7=20/docs=E3=80=81/?= =?UTF-8?q?redoc=E3=80=81/openapi.json=EF=BC=88=E8=A6=81=E6=90=AD=E5=BB=BA?= =?UTF-8?q?=E8=80=85=E7=99=BB=E5=BD=95=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 外部系统只能按中文句子认是哪种错,文案一改就悄悄失灵;开了登录,接口说明仍然不用登录就能读,全部接口的结构对匿名的人 敞开(中间件只拦 /api/ 开头的路径)。 错误码(detail 文字一字不改,老前端照常显示;码都登记在 api/coded.py): - api/apps.py 13 处:应用不存在(app_not_found,同入口发起时的码;没有授权的人也回它)、版本、实例不存在,钩子没声明、 没生成密钥,岗位绑定、个人应用草稿、授权、页面参数不合规;派生时工作流、版本不存在(apps/derive.py 的 DeriveRefused 带上码);视图、入口、补算的拒绝原样带码,只有引擎不发起运行(LaunchRefused)没带码的几种补上 run_launch_refused - api/tasks.py 3 处:待办不存在、查看全部要搭建者(role_required);服务层没带码的拒绝按状态补上同一类的码 (404 task_not_found、403 task_forbidden、409 task_closed、422 task_invalid),没改 process/tasks.py - api/artifacts.py 5 处:没有权限、这类原文只给搭建者、不存在、内容和哈希对不上 - api/deps.py 的 _forbidden(run_forbidden)、运行和证据接口的角色拒绝(role_required)、个人运行只给拥有者 (run_private)、问数据(chat_forbidden);ensure_role / 路由级依赖走 auth.require,抽出 auth.refusal 一处判断, 没登录 401 auth_required、角色不够 403 role_required /docs:开登录时中间件把 /docs(含子路径)、/redoc、/openapi.json 当成 /api 一样拦,和路由级依赖用同一个 auth.refusal: 匿名 401,不到搭建者 403。关登录照旧(test_provenance_contract.py 的断言不受影响)。 测试 tests/test_integration_errors.py:这五个模块里不许再直接抛 HTTPException(静态检查);应用、待办、工件、角色、运行 拒绝逐类核对 code 和原 detail;/docs 关登录照旧 200,开登录匿名 401、使用者 403、搭建者 200。去掉中间件的改动回跑, 开登录时匿名打开 /docs 是 200,测试失败。 --- backend/app/api/apps.py | 41 +++--- backend/app/api/artifacts.py | 15 ++- backend/app/api/auth.py | 3 +- backend/app/api/coded.py | 44 +++++++ backend/app/api/deps.py | 23 ++-- backend/app/api/tasks.py | 23 ++-- backend/app/apps/derive.py | 12 +- backend/app/auth/__init__.py | 27 ++-- backend/app/auth/middleware.py | 17 ++- backend/tests/test_integration_errors.py | 159 +++++++++++++++++++++++ 10 files changed, 307 insertions(+), 57 deletions(-) create mode 100644 backend/tests/test_integration_errors.py diff --git a/backend/app/api/apps.py b/backend/app/api/apps.py index 68b04481..6d0b281f 100644 --- a/backend/app/api/apps.py +++ b/backend/app/api/apps.py @@ -12,14 +12,18 @@ import dataclasses import json from typing import Any -from fastapi import APIRouter, Depends, Header, HTTPException, Query +from fastapi import APIRouter, Depends, Header, Query from fastapi.responses import JSONResponse from pydantic import BaseModel, Field from sqlalchemy import func, select from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncSession -from app.api.coded import CodedHTTPException +from app.api.coded import ( + APP_BINDINGS_INVALID, APP_DRAFT_INVALID, APP_GRANTS_INVALID, APP_HOOK_NOT_DECLARED, APP_HOOK_NOT_FOUND, + APP_INSTANCE_NOT_FOUND, APP_NOT_FOUND, APP_PARAMS_INVALID, APP_VERSION_NOT_FOUND, RUN_LAUNCH_REFUSED, + CodedHTTPException, +) from app.api.deps import Principal, actor_of, current_principal from app.apps.access import ROLE_TEXT, app_role, at_least, grants_of from app.db.base import get_session @@ -60,11 +64,11 @@ def _principal(x_actor: str | None) -> Principal: async def _app(session: AsyncSession, app_id: str, principal: Principal, need: str) -> tuple[App, str]: app = await session.get(App, app_id) if app is None: - raise HTTPException(404, APP_GONE) + raise CodedHTTPException(404, APP_GONE, APP_NOT_FOUND) role = await app_role(session, principal, app) if role is None: - # 没有任何授权的人不该知道这个应用在不在 - raise HTTPException(404, APP_GONE) + # 没有任何授权的人不该知道这个应用在不在:码和不存在的一样 + raise CodedHTTPException(404, APP_GONE, APP_NOT_FOUND) if not at_least(role, need): raise CodedHTTPException(403, f"需要这个应用的「{ROLE_TEXT[need]}」权限", NO_ACCESS) return app, role @@ -314,7 +318,7 @@ async def derive(payload: DeriveIn, session: AsyncSession = Depends(get_session) try: return {"spec": await derive_app(session, payload.workflow_id, payload.version, name=payload.name)} except DeriveRefused as e: - raise HTTPException(e.status_code, e.message) from e + raise CodedHTTPException(e.status_code, e.message, e.code) from e @router.get("/{app_id}") @@ -359,13 +363,13 @@ async def update_app(app_id: str, payload: AppPatch, session: AsyncSession = Dep app, role = await _app(session, app_id, principal, "manage") _refuse_archived(app, "修改") if payload.draft_bindings is not None and (problem := _bindings_problem(payload.draft_bindings)): - raise HTTPException(422, problem) + raise CodedHTTPException(422, problem, APP_BINDINGS_INVALID) if app.scope == "personal" and payload.draft is not None: # 个人应用的草稿就是在用的那一份:只许改呈现,固定的蓝图、视图来源只由「存成我的应用」改 from app.blueprints.personal import draft_problem if problem := draft_problem(app.draft or {}, payload.draft): - raise HTTPException(422, problem) + raise CodedHTTPException(422, problem, APP_DRAFT_INVALID) for key in ("name", "description", "icon"): if (value := getattr(payload, key)) is not None: setattr(app, key, value) @@ -568,7 +572,7 @@ async def get_version(app_id: str, version: int, session: AsyncSession = Depends row = (await session.execute(select(AppVersion).where( AppVersion.app_id == app.id, AppVersion.version == version))).scalar_one_or_none() if row is None: - raise HTTPException(404, f"「{app.name}」没有 v{version}") + raise CodedHTTPException(404, f"「{app.name}」没有 v{version}", APP_VERSION_NOT_FOUND) # 岗位绑定是这一个部署里的人和组:管理者才看得到 return {**_version_row(row), "spec": row.spec, "bindings": row.bindings if role == "manage" else {"slots": {}}} @@ -611,7 +615,7 @@ async def put_grants(app_id: str, payload: GrantsIn, session: AsyncSession = Dep _refuse_personal(app, "授权给别人") wanted, problem = await grants_problem(session, [(i.principal_kind, i.principal, i.role) for i in payload.items]) if problem: - raise HTTPException(422, problem) + raise CodedHTTPException(422, problem, APP_GRANTS_INVALID) return {"items": await replace_grants(session, app, wanted, actor=actor_of(x_actor))} @@ -713,7 +717,8 @@ async def rotate_hook(app_id: str, name: str, session: AsyncSession = Depends(ge draft, published, rows = await _hook_state(session, app) declared = draft.get(name) or published.get(name) if declared is None: - raise HTTPException(404, f"草稿和当前发布版里都没有声明钩子「{name}」,请先在「钩子」一节里声明它") + raise CodedHTTPException(404, f"草稿和当前发布版里都没有声明钩子「{name}」,请先在「钩子」一节里声明它", + APP_HOOK_NOT_DECLARED) # 从读行锁到提交:两位管理者同时点「生成密钥」,后到的那次读得到前一次插入的行,只轮换它(见 process/hooks.py) async with hooks.hook_lock(app.id, name): row, secret, created = await hooks.rotate_hook(session, app.id, name, kind=declared["kind"], @@ -731,7 +736,7 @@ async def _hook_named(session: AsyncSession, app: App, name: str) -> AppHook: row = await hook_row(session, app.id, name) if row is None: - raise HTTPException(404, f"钩子「{name}」还没有生成密钥") + raise CodedHTTPException(404, f"钩子「{name}」还没有生成密钥", APP_HOOK_NOT_FOUND) return row @@ -778,9 +783,9 @@ def _params(raw: str | None) -> dict[str, Any]: try: value = json.loads(raw) except ValueError as e: - raise HTTPException(422, "页面参数需要是 JSON 对象") from e + raise CodedHTTPException(422, "页面参数需要是 JSON 对象", APP_PARAMS_INVALID) from e if not isinstance(value, dict): - raise HTTPException(422, "页面参数需要是 JSON 对象") + raise CodedHTTPException(422, "页面参数需要是 JSON 对象", APP_PARAMS_INVALID) return value @@ -788,14 +793,16 @@ async def _instance(session: AsyncSession, app: App, run_id: str, principal: Pri """实例要属于这个应用;只有 use 的人只能看自己发起的(关登录时都能看)。""" run = await session.get(Run, run_id) if run is None or run.app_id != app.id or run.origin_kind not in ("app_entry", "hook"): - raise HTTPException(404, "实例不存在") + raise CodedHTTPException(404, "实例不存在", APP_INSTANCE_NOT_FOUND) if principal.mode != "off" and not at_least(role, "observe") and run.started_by != principal.username: raise CodedHTTPException(403, "只能查看自己发起的实例", NO_ACCESS) return run -def _view_refused(e: Any) -> HTTPException: - return CodedHTTPException(e.status_code, e.message, e.code) if e.code else HTTPException(e.status_code, e.message) +def _view_refused(e: Any) -> CodedHTTPException: + """视图、入口、补算、发起运行的拒绝 → 带 code 的 HTTP 错误。只有引擎不发起运行(LaunchRefused)有不带码的几种(工作流 + 没发布、版本不存在、图没通过校验……),补上同一类的 run_launch_refused,detail 照旧说清是哪一种。""" + return CodedHTTPException(e.status_code, e.message, e.code or RUN_LAUNCH_REFUSED) @router.get("/{app_id}/pages/{page_id}") diff --git a/backend/app/api/artifacts.py b/backend/app/api/artifacts.py index e8099dbe..e210ffba 100644 --- a/backend/app/api/artifacts.py +++ b/backend/app/api/artifacts.py @@ -2,9 +2,12 @@ from __future__ import annotations from typing import Any -from fastapi import APIRouter, HTTPException +from fastapi import APIRouter from sqlalchemy import select +from app.api.coded import ( + ARTIFACT_FORBIDDEN, ARTIFACT_KIND_FORBIDDEN, ARTIFACT_NOT_FOUND, ARTIFACT_TAMPERED, CodedHTTPException, +) from app.api.deps import can_read_artifact, current_principal from app.core import artifact_store @@ -23,14 +26,14 @@ async def get_artifact(artifact_id: str) -> dict[str, Any]: principal = current_principal() restricted = principal.mode != "off" and not principal.at_least("builder") if principal.mode != "off" and not await can_read_artifact(principal, artifact_id): - raise HTTPException(403, "没有权限查看这件工件") + raise CodedHTTPException(403, "没有权限查看这件工件", ARTIFACT_FORBIDDEN) try: content = artifact_store.load(artifact_id) except ValueError as e: # 哈希对不上:明确报给调用方,这正是工件库存在的意义 - raise HTTPException(409, str(e)) from e + raise CodedHTTPException(409, str(e), ARTIFACT_TAMPERED) from e if content is None: - raise HTTPException(404, "工件不存在,可能已被删除") + raise CodedHTTPException(404, "工件不存在,可能已被删除", ARTIFACT_NOT_FOUND) if restricted: content = await _masked_for_user(principal, artifact_id, content) return {"id": artifact_id, "content": content} @@ -68,7 +71,7 @@ async def _masked_for_user(principal: Any, artifact_id: str, content: Any) -> An return await _masked_metrics(principal, artifact_id, content, rows) if _RECEIPT in kinds and isinstance(content, dict): return await _masked_receipt(content, rows) - raise HTTPException(403, "这类工件的原文只对搭建者及以上的角色开放") + raise CodedHTTPException(403, "这类工件的原文只对搭建者及以上的角色开放", ARTIFACT_KIND_FORBIDDEN) async def _masked_snapshot(content: dict[str, Any], *, endpoint: bool) -> dict[str, Any]: @@ -106,7 +109,7 @@ async def _masked_metrics(principal: Any, artifact_id: str, card: dict[str, Any] owner = (run_id, node_id or "") break if owner is None: - raise HTTPException(403, "没有权限查看这件工件") + raise CodedHTTPException(403, "没有权限查看这件工件", ARTIFACT_FORBIDDEN) sealed = await _sealed(owner[0]) masks = _Masks(sealed) metrics: list[Any] = [] diff --git a/backend/app/api/auth.py b/backend/app/api/auth.py index 60e58654..19fc9cf6 100644 --- a/backend/app/api/auth.py +++ b/backend/app/api/auth.py @@ -17,6 +17,7 @@ from pydantic import BaseModel, Field from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession +from app.api.coded import AUTH_REQUIRED from app.api.deps import Principal, current_principal, decode_actor, ensure_role from app.auth import ( MODES, SETTING_KEY, current_mode, env_mode, load_settings, remember_settings, stored_settings, user_can_chat, @@ -59,7 +60,7 @@ async def me(x_actor: str | None = Header(default=None), session: AsyncSession = return await principal_full(session, Principal(username=actor, display_name=actor, role="admin", mode="off")) if not principal.username: can_bootstrap = principal.mode == "local" and not await _local_admin_exists(session) - return JSONResponse(status_code=401, content={"detail": "请先登录", "code": "auth_required", + return JSONResponse(status_code=401, content={"detail": "请先登录", "code": AUTH_REQUIRED, "mode": principal.mode, "can_bootstrap": can_bootstrap}) return await principal_full(session, principal) diff --git a/backend/app/api/coded.py b/backend/app/api/coded.py index da319c42..27e7ea66 100644 --- a/backend/app/api/coded.py +++ b/backend/app/api/coded.py @@ -29,6 +29,50 @@ CATALOG_DRAFT_BUSY = "catalog_draft_busy" CATALOG_DRAFT_MODEL_UNAVAILABLE = "catalog_draft_model_unavailable" CATALOG_DRAFT_FINISHED = "catalog_draft_finished" +# ---------------------------------------------------------------- 集成接口(04 细案 A7) +# 外部系统用访问密钥调的应用、待办、工件接口,以及角色、运行读权限的拒绝:按 code 认是哪种错,detail 照旧是给人看的那句 + +#: 身份:没登录(401,中间件、auth.require 都回它)、角色不够(403,路由级依赖、接口内部的 ensure_role、开登录时的 /docs) +AUTH_REQUIRED = "auth_required" +ROLE_REQUIRED = "role_required" +#: 运行:这次运行读不了(403,api/deps 的证据、运行接口);别人的个人应用刷新、个人蓝图试运行只给拥有者(403) +RUN_FORBIDDEN = "run_forbidden" +RUN_PRIVATE = "run_private" +#: 问数据和助手:使用者没开问数据,或者这个接口不在问数据的范围里(403) +CHAT_FORBIDDEN = "chat_forbidden" + +#: 应用(api/apps.py)。app_not_found 同入口发起时的 apps/entries.APP_GONE;没有任何授权的人也回它(不该知道应用在不在) +APP_NOT_FOUND = "app_not_found" +APP_VERSION_NOT_FOUND = "app_version_not_found" +APP_INSTANCE_NOT_FOUND = "app_instance_not_found" +#: 草稿里、当前发布版里都没有声明这个钩子(404);声明了、还没生成密钥(404) +APP_HOOK_NOT_DECLARED = "app_hook_not_declared" +APP_HOOK_NOT_FOUND = "app_hook_not_found" +#: 改草稿、授权时写进来的不合规(422):岗位绑定、个人应用的草稿(只许改呈现)、授权的人和组 +APP_BINDINGS_INVALID = "app_bindings_invalid" +APP_DRAFT_INVALID = "app_draft_invalid" +APP_GRANTS_INVALID = "app_grants_invalid" +#: 页面参数不是 JSON 对象(422) +APP_PARAMS_INVALID = "app_params_invalid" +#: 发起实例、刷新视图、补算时引擎不发起运行,又没有更细的码(engine/launch.LaunchRefused 没带 code 的那几种) +RUN_LAUNCH_REFUSED = "run_launch_refused" +# 派生应用草稿时工作流、版本不存在(404)的码在 apps/derive.py(WORKFLOW_GONE、VERSION_GONE);入口、视图、补算的码 +# 在 apps/entries.py、apps/resolve.py、apps/backfill.py,同这里一样都是 app_ 开头 + +#: 待办(api/tasks.py)。服务层(process/tasks.py 的 TaskRefused)没带码的拒绝按状态补上同一类的码:404 不存在;403 +#: 不能这样做(不是管理员也不是应用的管理者,不能改派、催办);409 待办已经处理过、关闭了、刚被别人回复,或者所属的 +#: 运行不再等这条回复——刷新看最新状态;422 回复的内容不对。带了码的照用自己的(task_not_assigned、form_invalid……) +TASK_NOT_FOUND = "task_not_found" +TASK_FORBIDDEN = "task_forbidden" +TASK_CLOSED = "task_closed" +TASK_INVALID = "task_invalid" + +#: 工件(api/artifacts.py):没有权限(403)、这类工件的原文只给搭建者(403)、不存在(404)、内容和哈希对不上(409) +ARTIFACT_FORBIDDEN = "artifact_forbidden" +ARTIFACT_KIND_FORBIDDEN = "artifact_kind_forbidden" +ARTIFACT_NOT_FOUND = "artifact_not_found" +ARTIFACT_TAMPERED = "artifact_tampered" + #: 设置(api/settings.py):对外访问地址由环境变量 AGENTLAB_PUBLIC_URL 指定时,设置里那一项不能改成别的值(409) PUBLIC_URL_ENV_LOCKED = "public_url_env_locked" diff --git a/backend/app/api/deps.py b/backend/app/api/deps.py index 6fa03e2d..4c9913f2 100644 --- a/backend/app/api/deps.py +++ b/backend/app/api/deps.py @@ -26,9 +26,11 @@ from dataclasses import dataclass, field from typing import Any from urllib.parse import unquote -from fastapi import Depends, HTTPException +from fastapi import Depends from starlette.requests import HTTPConnection +from app.api.coded import CHAT_FORBIDDEN, ROLE_REQUIRED, RUN_FORBIDDEN, RUN_PRIVATE, CodedHTTPException + #: 登录方式 AUTH_MODES = ("off", "local", "oidc") #: 角色由高到低 @@ -140,7 +142,7 @@ async def can_read_artifact(principal: Principal, artifact_id: str) -> bool: def ensure_role(role: str) -> None: - """接口内部用:至少是 role,否则 401(没登录)/ 403(角色不够)。关登录时放行。 + """接口内部用:至少是 role,否则 401(没登录,auth_required)/ 403(角色不够,role_required)。关登录时放行。 用在要看请求内容才知道要什么角色的地方(比如 MCP 的 stdio 只有管理员能建)。""" from app.auth import require @@ -169,8 +171,8 @@ async def ensure_conversation_access(session: Any, conversation_id: str | None) return await check(session, conversation_id) -def _forbidden() -> HTTPException: - return HTTPException(403, "没有权限查看这次运行") +def _forbidden() -> CodedHTTPException: + return CodedHTTPException(403, "没有权限查看这次运行", RUN_FORBIDDEN) #: user 角色在自己的问数据运行上能读的(GET /api/runs/{id} 之后的部分):运行记录、事件、运行时的图,加上实时流 @@ -235,13 +237,13 @@ async def _run_scope(conn: HTTPConnection) -> None: return ensure_role("user") if not user_can_chat(): - raise HTTPException(403, _RUNS_FOR_BUILDERS) + raise CodedHTTPException(403, _RUNS_FOR_BUILDERS, ROLE_REQUIRED) method = conn.scope.get("method", "GET") run_id = conn.path_params.get("run_id") if run_id is None: if method == "POST" and conn.url.path.rstrip("/") == "/api/runs": return - raise HTTPException(403, _RUNS_FOR_BUILDERS) + raise CodedHTTPException(403, _RUNS_FOR_BUILDERS, ROLE_REQUIRED) run = await _load_run(run_id) rest = conn.url.path.rstrip("/").removeprefix(f"/api/runs/{run_id}") if method == "GET" and rest in _CHAT_RUN_READS: @@ -252,7 +254,7 @@ async def _run_scope(conn: HTTPConnection) -> None: raise _forbidden() if method == "POST" and rest == "/cancel": return - raise HTTPException(403, "没有权限:这项操作需要搭建者及以上的角色") + raise CodedHTTPException(403, "没有权限:这项操作需要搭建者及以上的角色", ROLE_REQUIRED) async def _load_run(run_id: str | None) -> Any: @@ -282,7 +284,7 @@ async def _refuse_private(conn: HTTPConnection, principal: Principal) -> None: from fastapi import WebSocketException, status raise WebSocketException(code=status.WS_1008_POLICY_VIOLATION) - raise HTTPException(403, PRIVATE_RUN) + raise CodedHTTPException(403, PRIVATE_RUN, RUN_PRIVATE) async def _private_run_scope(conn: HTTPConnection) -> None: @@ -310,7 +312,7 @@ async def _evidence_scope(conn: HTTPConnection) -> None: return ensure_role("user") if conn.scope.get("method", "GET") != "GET": - raise HTTPException(403, "没有权限:这项操作需要搭建者及以上的角色") + raise CodedHTTPException(403, "没有权限:这项操作需要搭建者及以上的角色", ROLE_REQUIRED) run_id = conn.path_params.get("run_id") if not run_id or not await can_read_run(principal, run_id): raise _forbidden() @@ -327,7 +329,8 @@ async def _chat_scope(conn: HTTPConnection) -> None: method = conn.scope.get("method", "GET") if user_can_chat() and (path.startswith("/api/conversations") or (method, path) in _CHAT_ROUTES): return - raise HTTPException(403, "没有权限:问数据和助手需要搭建者及以上的角色,或由管理员为使用者开启问数据") + raise CodedHTTPException(403, "没有权限:问数据和助手需要搭建者及以上的角色,或由管理员为使用者开启问数据", + CHAT_FORBIDDEN) #: user 角色开了问数据时能用的其余接口:问数据的生成流、读当前助手模型、问数据页选数据源用的数据源名单 diff --git a/backend/app/api/tasks.py b/backend/app/api/tasks.py index 8418558d..8089e350 100644 --- a/backend/app/api/tasks.py +++ b/backend/app/api/tasks.py @@ -1,17 +1,20 @@ """待办(BE-2):列表、计数、详情、回复、改派、催办。设计文档 §3.7、§3.10。 身份经 api/deps.current_principal 取;关登录时它没有名字,用自报的署名(X-Actor)补上,回复人一栏才写得出人。 -回复被拒回 403 / 404 / 409 / 422,{detail, code};表单不通过时另带 errors(同应用入口的 422)。 +回复被拒回 403 / 404 / 409 / 422,{detail, code},一律带 code(外部系统按它认);表单不通过时另带 errors(同应用入口的 +422)。服务层没带码的拒绝按状态补上同一类的码(api/coded.py 的 TASK_*)。 """ from __future__ import annotations from typing import Any, Literal -from fastapi import APIRouter, Header, HTTPException, Query +from fastapi import APIRouter, Header, Query from fastapi.responses import JSONResponse from pydantic import BaseModel, Field -from app.api.coded import CodedHTTPException +from app.api.coded import ( + ROLE_REQUIRED, TASK_CLOSED, TASK_FORBIDDEN, TASK_INVALID, TASK_NOT_FOUND, CodedHTTPException, +) from app.api.deps import current_principal, decode_actor from app.process import tasks as task_service from app.process.tasks import FormInvalid, TaskRefused @@ -23,12 +26,14 @@ def principal_of(x_actor: str | None) -> Any: return task_service.principal_with_actor(current_principal(), decode_actor(x_actor)) -def refused(e: TaskRefused) -> HTTPException | JSONResponse: +#: 服务层(process/tasks.py、process/actions.py、blueprints/personal.py)没带码的拒绝按状态归到哪一类 +_CODE_BY_STATUS = {404: TASK_NOT_FOUND, 403: TASK_FORBIDDEN, 409: TASK_CLOSED, 422: TASK_INVALID} + + +def refused(e: TaskRefused) -> CodedHTTPException | JSONResponse: if isinstance(e, FormInvalid): return JSONResponse({"detail": str(e), "code": e.code, "errors": e.errors}, status_code=422) - if e.code: - return CodedHTTPException(e.status_code, str(e), e.code) - return HTTPException(e.status_code, str(e)) + return CodedHTTPException(e.status_code, str(e), e.code or _CODE_BY_STATUS.get(e.status_code, TASK_CLOSED)) def _raise(e: TaskRefused) -> Any: @@ -53,7 +58,7 @@ async def list_tasks( ) -> dict[str, Any]: principal = principal_of(x_actor) if assigned == "all" and principal.mode != "off" and not principal.at_least("builder"): - raise HTTPException(403, "只能查看派给自己的待办") + raise CodedHTTPException(403, "只能查看派给自己的待办", ROLE_REQUIRED) statuses = None if status == "all" else _csv(status) or ["pending"] items = await task_service.list_tasks(principal, assigned=assigned, statuses=statuses, app_id=app_id, run_id=run_id, limit=limit) @@ -71,7 +76,7 @@ async def get_task(task_id: str, x_actor: str | None = Header(default=None)) -> principal = principal_of(x_actor) out = await task_service.detail(task_id, principal) if out is None or not task_service.visible(out, principal): - raise HTTPException(404, "待办不存在,所属的运行可能已被删除") + raise CodedHTTPException(404, "待办不存在,所属的运行可能已被删除", TASK_NOT_FOUND) return out diff --git a/backend/app/apps/derive.py b/backend/app/apps/derive.py index 069b9672..7fbb84d4 100644 --- a/backend/app/apps/derive.py +++ b/backend/app/apps/derive.py @@ -24,13 +24,19 @@ VIEW = "latest" INSTANCE_VIEW = "inst" ENTRY = "start" RECONCILER = "reconciler" +#: 派生不了时的机器码:工作流、版本不存在(404) +WORKFLOW_GONE = "workflow_not_found" +VERSION_GONE = "workflow_version_not_found" class DeriveRefused(Exception): - def __init__(self, status_code: int, message: str) -> None: + """派生不了。code 是给程序认的机器码,接口照它回。""" + + def __init__(self, status_code: int, message: str, code: str) -> None: super().__init__(message) self.status_code = status_code self.message = message + self.code = code def _safe(text: str) -> str: @@ -45,10 +51,10 @@ async def derive_app(session: AsyncSession, workflow_id: str, version: int, *, n workflow = await session.get(Workflow, workflow_id) if workflow is None: - raise DeriveRefused(404, "工作流不存在,可能已被删除") + raise DeriveRefused(404, "工作流不存在,可能已被删除", WORKFLOW_GONE) row = await version_row(session, workflow_id, version) if row is None: - raise DeriveRefused(404, f"「{workflow.name}」不存在 v{version} 版本") + raise DeriveRefused(404, f"「{workflow.name}」不存在 v{version} 版本", VERSION_GONE) state = PinState(alias=PIN, pin={"workflow_id": workflow_id, "version": version}, workflow=workflow, row=row) schedule = (await session.execute(select(Schedule).where( Schedule.workflow_id == workflow_id, Schedule.run_class == "formal", Schedule.enabled.is_(True)) diff --git a/backend/app/auth/__init__.py b/backend/app/auth/__init__.py index def0941d..361b5153 100644 --- a/backend/app/auth/__init__.py +++ b/backend/app/auth/__init__.py @@ -15,7 +15,9 @@ import logging import os from typing import Any -from fastapi import Depends, HTTPException +from fastapi import Depends + +from app.api.coded import AUTH_REQUIRED, ROLE_REQUIRED, CodedHTTPException logger = logging.getLogger(__name__) @@ -77,17 +79,24 @@ def remember_settings(value: dict[str, Any]) -> None: _loaded = True +def refusal(principal: Any, role: str) -> CodedHTTPException | None: + """principal 够不够 role:够(或关登录)是 None;不够交回该回的错——没登录 401(auth_required),角色不够 403 + (role_required)。路由级依赖、接口内部的 ensure_role、中间件拦 /docs 都用它,同一个判断、同一句话。""" + if principal.mode == "off": + return None + if not principal.username: + return CodedHTTPException(401, "请先登录", AUTH_REQUIRED) + if not principal.at_least(role): + return CodedHTTPException(403, f"没有权限:这项操作需要{ROLE_LABEL.get(role, role)}及以上的角色", ROLE_REQUIRED) + return None + + def require(role: str) -> None: - """当前请求至少是 role,否则 401(没登录)/ 403(角色不够)。关登录时放行。""" + """当前请求至少是 role,否则 401(没登录)/ 403(角色不够),都带 code。关登录时放行。""" from app.api.deps import current_principal - principal = current_principal() - if principal.mode == "off": - return - if not principal.username: - raise HTTPException(401, "请先登录") - if not principal.at_least(role): - raise HTTPException(403, f"没有权限:这项操作需要{ROLE_LABEL.get(role, role)}及以上的角色") + if (refused := refusal(current_principal(), role)) is not None: + raise refused def role_dependency(role: str) -> Any: diff --git a/backend/app/auth/middleware.py b/backend/app/auth/middleware.py index b3e44a2a..3a9cfaa8 100644 --- a/backend/app/auth/middleware.py +++ b/backend/app/auth/middleware.py @@ -9,6 +9,8 @@ - 会话 cookie → Principal(认证过的用户名、角色、组);没有会话是匿名(username=None),X-Actor 一律不看; - /api 下除了登录本身、健康检查、入站钩子,匿名一律 401。各路由的角色要求另由 main.py 的路由级依赖管,这里兜底: 哪个路由忘了挂依赖,也不会对没登录的人敞开; +- 接口说明(FastAPI 自带的 /docs、/redoc、/openapi.json)和 /api 一样拦:匿名 401,不到搭建者 403。它们是 FastAPI 自己 + 挂的路由,挂不上路由级依赖,所以角色也在这里判。以前只拦 /api/ 开头的,开了登录接口全貌也对匿名的人敞开; - 非 GET 的请求必须带 X-AgentLab-Request 头,带了 Origin 的必须是本站或登记的前端地址(CSRF)。自定义请求头跨站发不出来 (要过 CORS 预检,只放行配置里的前端地址),SameSite=Lax 的 cookie 也不随跨站的表单提交带过去; - WebSocket:没登录的握手前就关掉;登录了的交给路由,按运行判(api/deps.py 的 run_scope:user 角色只能连自己能读、 @@ -20,6 +22,7 @@ import json from http.cookies import SimpleCookie from typing import Any +from app.api.coded import AUTH_REQUIRED from app.core.config import settings #: 开登录时匿名也能访问的接口 @@ -27,6 +30,8 @@ _PUBLIC = frozenset({"/api/health", "/api/auth/me", "/api/auth/login", "/api/aut "/api/auth/oidc/callback", "/api/auth/jwks", "/api/auth/bootstrap"}) #: 只认签名、不走会话的入站钩子(也不查 CSRF:调用方是外部系统,没有浏览器的 Origin) _HOOKS = "/api/hooks/" +#: FastAPI 自带的接口说明(main.py 里 FastAPI() 的缺省地址):开登录时要搭建者 +_DOCS = ("/docs", "/redoc", "/openapi.json") _SAFE_METHODS = frozenset({"GET", "HEAD", "OPTIONS"}) CSRF_HEADER = "x-agentlab-request" @@ -48,6 +53,10 @@ def _cookie(headers: dict[str, str], name: str) -> str | None: return morsel.value if morsel else None +def _is_docs(path: str) -> bool: + return any(path == p or path.startswith(p + "/") for p in _DOCS) + + def _origin_ok(origin: str, headers: dict[str, str], scope: dict[str, Any]) -> bool: from app.core.public_url import origin_of, public_base_url @@ -93,7 +102,7 @@ class AuthMiddleware: await self.app(scope, receive, send) return from app.api.deps import reset_principal, set_principal - from app.auth import current_mode + from app.auth import current_mode, refusal # 登录设置在启动时读进缓存(main.py 的 _platform_startup),改设置的接口改完就刷新;这里不查库 mode = current_mode() @@ -131,7 +140,11 @@ class AuthMiddleware: return if (scope["type"] == "http" and is_api and not principal.username and path not in _PUBLIC and not path.startswith(_HOOKS)): - await _reply(send, 401, "请先登录", "auth_required") + await _reply(send, 401, "请先登录", AUTH_REQUIRED) + return + if scope["type"] == "http" and _is_docs(path) and (refused := refusal(principal, "builder")) is not None: + # 和路由级依赖(auth.require)同一个判断、同一句话:没登录 401,角色不够 403 + await _reply(send, refused.status_code, refused.detail, refused.code) return token = set_principal(principal) diff --git a/backend/tests/test_integration_errors.py b/backend/tests/test_integration_errors.py new file mode 100644 index 00000000..cd8c0262 --- /dev/null +++ b/backend/tests/test_integration_errors.py @@ -0,0 +1,159 @@ +"""集成接口的错误码与 /docs(04 细案 A7)。 + +外部系统按 code 认是哪种错,不按中文句子:应用(api/apps.py)、待办(api/tasks.py)、工件(api/artifacts.py)的错误, +和角色、运行读权限的拒绝(api/deps.py、auth.require)全部带 code;detail 还是原来那一句,老前端照常显示。 + +开了登录,/docs、/redoc、/openapi.json 和 /api 一样要登录、要搭建者:以前中间件只拦 /api/ 开头的路径,接口全貌对 +匿名的人敞开。关登录照旧(tests/test_provenance_contract.py 的断言不受影响)。 +""" +from __future__ import annotations + +import ast +import uuid +from pathlib import Path + +import pytest +from httpx import ASGITransport, AsyncClient + +from app.api import coded +from app.db.base import SessionLocal +from app.db.models_apps import App +from app.main import app as main_app +from tests.fakes import auth_env + +APP_DIR = Path(__file__).resolve().parents[1] / "app" +DOCS = ("/docs", "/redoc", "/openapi.json") + + +@pytest.fixture(autouse=True) +async def clean(): + await auth_env.reset() + try: + yield + finally: + await auth_env.reset() + + +def client() -> AsyncClient: + return AsyncClient(transport=ASGITransport(app=main_app), base_url="http://test") + + +async def new_app() -> str: + async with SessionLocal() as session: + row = App(name=f"报销-{uuid.uuid4().hex[:6]}") + session.add(row) + await session.commit() + return row.id + + +def test_no_bare_http_exception_left_in_the_integration_modules(): + """这几个模块里不许再直接抛 HTTPException(没有 code):一律 CodedHTTPException。""" + for rel in ("api/apps.py", "api/tasks.py", "api/artifacts.py", "api/deps.py", "auth/__init__.py"): + tree = ast.parse((APP_DIR / rel).read_text()) + bare = [node.lineno for node in ast.walk(tree) if isinstance(node, ast.Call) + and (getattr(node.func, "id", None) or getattr(node.func, "attr", None)) == "HTTPException"] + assert not bare, f"{rel} 第 {bare} 行抛的 HTTPException 没有 code" + + +def test_codes_agree_with_the_ones_already_in_use(): + from app.apps import entries + + assert coded.APP_NOT_FOUND == entries.APP_GONE, "入口发起时的「应用不存在」和这里同一个码" + + +async def test_app_errors_carry_codes_and_keep_their_detail(): + app_id = await new_app() + async with client() as c: + r = await c.get("/api/apps/nope-nope") + assert (r.status_code, r.json()) == (404, {"detail": "应用不存在,可能已被删除", "code": "app_not_found"}) + r = await c.get(f"/api/apps/{app_id}/versions/9") + assert r.status_code == 404 and r.json()["code"] == "app_version_not_found" and "没有 v9" in r.json()["detail"] + r = await c.get(f"/api/apps/{app_id}/pages/home", params={"params": "[1, 2]", "draft": 1}) + assert (r.status_code, r.json()) == (422, {"detail": "页面参数需要是 JSON 对象", "code": "app_params_invalid"}) + r = await c.patch(f"/api/apps/{app_id}", json={"draft_bindings": {"slots": {"财务": "不是列表"}}}) + assert r.status_code == 422 and r.json()["code"] == "app_bindings_invalid", r.text + r = await c.post(f"/api/apps/{app_id}/hooks/nope/rotate") + assert r.status_code == 404 and r.json()["code"] == "app_hook_not_declared" + r = await c.patch(f"/api/apps/{app_id}/hooks/nope", json={"enabled": False}) + assert r.status_code == 404 and r.json()["code"] == "app_hook_not_found", r.text + r = await c.post("/api/apps/derive", json={"workflow_id": "nope-nope", "version": 1}) + assert (r.status_code, r.json()["code"]) == (404, "workflow_not_found") + + +async def test_task_errors_carry_codes(): + async with client() as c: + r = await c.get("/api/tasks/nope-nope") + assert (r.status_code, r.json()) == (404, {"detail": "待办不存在,所属的运行可能已被删除", "code": "task_not_found"}) + # 服务层没带码的拒绝(这里是待办不存在)按状态补上同一类的码 + r = await c.post("/api/tasks/nope-nope/respond", json={"decision": "approve"}) + assert (r.status_code, r.json()["code"]) == (404, "task_not_found") + r = await c.post("/api/tasks/nope-nope/remind") + assert (r.status_code, r.json()["code"]) == (404, "task_not_found") + + +async def test_artifact_errors_carry_codes(): + async with client() as c: + r = await c.get(f"/api/artifacts/{'0' * 64}") + assert (r.status_code, r.json()) == (404, {"detail": "工件不存在,可能已被删除", "code": "artifact_not_found"}) + + +async def test_role_and_run_refusals_carry_codes(): + await auth_env.turn_on_local() + await auth_env.make_user("uma", "user") + uma = await auth_env.login("uma") + r = await uma.get("/api/workflows") + assert r.status_code == 403 and r.json() == {"detail": "没有权限:这项操作需要搭建者及以上的角色", + "code": "role_required"} + r = await uma.get("/api/tasks", params={"assigned": "all"}) + assert r.status_code == 403 and r.json()["code"] == "role_required" + r = await uma.get("/api/runs/nope-nope/evidence") + assert r.status_code == 403 and r.json() == {"detail": "没有权限查看这次运行", "code": "run_forbidden"} + r = await uma.get(f"/api/artifacts/{'0' * 64}") + assert r.status_code == 403 and r.json() == {"detail": "没有权限查看这件工件", "code": "artifact_forbidden"} + r = await uma.get("/api/runs") + assert r.status_code == 403 and r.json()["code"] == "role_required" + + +async def test_ensure_role_inside_an_endpoint_carries_codes(): + from app.api.deps import Principal, ensure_role, reset_principal, set_principal + + for principal, status, code in ((Principal(username=None, role="", mode="local"), 401, "auth_required"), + (Principal(username="bob", role="builder", verified=True, mode="local"), 403, + "role_required")): + token = set_principal(principal) + try: + with pytest.raises(coded.CodedHTTPException) as e: + ensure_role("admin") + finally: + reset_principal(token) + assert (e.value.status_code, e.value.code) == (status, code) + assert "管理员" in e.value.detail + + +# ---------------------------------------------------------------- /docs + + +async def test_docs_stay_open_when_login_is_off(): + async with client() as c: + for path in DOCS: + assert (await c.get(path)).status_code == 200, path + + +async def test_docs_need_a_builder_once_login_is_on(): + await auth_env.turn_on_local() + await auth_env.make_user("uma", "user") + await auth_env.make_user("bob", "builder") + async with client() as anonymous: + for path in DOCS: + r = await anonymous.get(path) + assert r.status_code == 401 and r.json()["code"] == "auth_required", path + # 子路径(Swagger 的 OAuth 回跳页)一样 + assert (await anonymous.get("/docs/oauth2-redirect")).status_code == 401 + uma = await auth_env.login("uma") + for path in DOCS: + r = await uma.get(path) + assert r.status_code == 403 and r.json()["code"] == "role_required", path + bob = await auth_env.login("bob") + for path in DOCS: + assert (await bob.get(path)).status_code == 200, path + assert "/api/apps/{app_id}" in (await bob.get("/openapi.json")).json()["paths"] From a59409cc85f6be3f12b732d619aff8933a234117 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Fri, 16 Oct 2026 10:24:22 +0800 Subject: [PATCH 11/39] =?UTF-8?q?=E4=BD=BF=E7=94=A8=E8=80=85=E9=80=89?= =?UTF-8?q?=E4=BA=BA=EF=BC=88=E5=90=8E=E7=AB=AF=EF=BC=89=EF=BC=9A=E6=8C=89?= =?UTF-8?q?=E7=94=A8=E9=80=94=E7=BB=99=E5=80=99=E9=80=89=E6=B1=A0=EF=BC=8C?= =?UTF-8?q?=E5=BC=80=E7=99=BB=E5=BD=95=E6=97=B6=E5=85=A5=E5=8F=A3=E8=A1=A8?= =?UTF-8?q?=E5=8D=95=E3=80=81=E5=BE=85=E5=8A=9E=E8=A1=A8=E5=8D=95=E7=9A=84?= =?UTF-8?q?=E9=80=89=E4=BA=BA=E6=8E=A7=E4=BB=B6=E6=8C=89=E5=90=8C=E4=B8=80?= =?UTF-8?q?=E4=B8=AA=E6=B1=A0=E6=A0=B8=E5=AF=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 为什么:使用者在选人控件里没有候选(名录只给搭建者),服务端也不核对。填错的名字原样派下去,待办给了一个不存在的人, 没人能处理、也不报错(05-已知遗留 (b))。名录又不能整份放开给使用者(只读与最小可见)。 做了什么: - auth/directory 新增候选池:form 用途缺省是「能用这个应用的人」——使用及以上的授权(人;组列出组本身并展开成员), 加上管理它的人(管理员、建应用的人);应用授权给了所有人时池是全部启用的人(open),不带前缀 q 不列(needs_query)。 字段可以声明 x-principal: {pool: app | group:<组> | slot:<岗位>}(slot 按这次用到的应用版本的 bindings 展开)。 review 用途是启用的搭建者和管理员(同 personal._review_targets)。停用的人一律不列。 - GET /api/directory/candidates(任何登录的人;main.py 路由清单只加一行):purpose=form 带 app_id+entry 或 task_id、field, 调用的人不能用这个应用 403;只回 {users:[{username, display_name}], groups, truncated, needs_query?},不给停用、角色、 邮箱、组成员;q 按用户名或显示名开头筛,limit 最多 50。/api/directory 仍只给搭建者。 - 开了登录时 start_entry 和待办表单提交按同一个池核对,不在池里 422,errors 的 code=principal(FORM_ERROR_CODES 加一项), 话写成「「x」不在可选的人里,请从候选中选择」(单点登录另说要先登录一次才进名录)。直接生效(遗留-Q4 按推荐)。 关登录照旧手填、不核对;不属于应用的待办(试运行、画布)没有池,不核对。 - apps/forms:x-principal 进 _FIELD_KEYS,validate_form_schema 查写法(只能写在选人控件上、只能是三种池)。 搭建者没有授权时按 apps/access 只能「查看全部实例」,不算池里的人——池按「能用」算,这一点写进了 app_pool 的说明。 测试:test_directory_candidates(只给名字、停用的不列、前缀筛、没使用权 403、名录仍 403;授权给所有人要前缀、截断;review 只出搭建者和管理员;入口表单不在池 422、停用的 422、组名和池里的人放行;待办表单同一个池;声明 group:<组> 收窄;关登录 任何名字照收;x-principal 写法校验)。 --- backend/app/api/identity.py | 53 ++++ backend/app/apps/entries.py | 7 + backend/app/apps/forms.py | 11 +- backend/app/apps/spec.py | 5 +- backend/app/auth/directory.py | 178 +++++++++++++- backend/app/main.py | 1 + backend/app/process/tasks.py | 18 ++ backend/tests/test_directory_candidates.py | 266 +++++++++++++++++++++ 8 files changed, 535 insertions(+), 4 deletions(-) create mode 100644 backend/tests/test_directory_candidates.py diff --git a/backend/app/api/identity.py b/backend/app/api/identity.py index 01759cd5..845f9b64 100644 --- a/backend/app/api/identity.py +++ b/backend/app/api/identity.py @@ -27,6 +27,9 @@ router = APIRouter(prefix="/api", tags=["identity"]) #: 人和组的名录(GET /api/directory):只读、给搭建者及以上(main.py 挂 require_role("builder")),岗位绑定、应用授权的 #: 选人控件据此给候选、核对填的名字在不在。只有用户名、显示名、是否停用和组名,不含邮箱、角色、登录时间 directory_router = APIRouter(prefix="/api", tags=["identity"]) +#: 使用者选人的候选(GET /api/directory/candidates,05-已知遗留 (b)):任何登录的人都能调(main.py 挂 require_role("user")), +#: 按用途给候选池,只有用户名和显示名 +candidates_router = APIRouter(prefix="/api", tags=["identity"]) def _iso(value: Any) -> str | None: @@ -224,6 +227,56 @@ async def directory(session: AsyncSession = Depends(get_session)) -> dict[str, A "groups": [{"name": g.name, "label": g.label} for g in known.groups.values()]} +@candidates_router.get("/directory/candidates") +async def candidates(purpose: str, app_id: str | None = None, entry: str | None = None, task_id: str | None = None, + field: str | None = None, q: str | None = None, limit: int = 20, + session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """使用者选人时的候选:{users: [{username, display_name}], groups: [{name, label}], truncated, needs_query?}。 + + 不给是否停用(停用的直接不列)、角色、邮箱、组成员。purpose=form:表单里的选人控件,带 app_id 和 entry(入口表单) + 或 task_id(待办表单)、field(哪个控件),池按字段的 x-principal 算、缺省是能用这个应用的人,要求调用的人能用这个 + 应用(否则 403);应用授权给了所有人时不带前缀 q 不列(needs_query)。purpose=review:个人应用提交审核,启用的搭建者 + 和管理员。q 按用户名或显示名的开头筛,limit 最多 50。开了登录时提交的值按同一个池核对(auth/directory.principal_errors)。""" + from app.api.coded import CodedHTTPException + from app.api.deps import current_principal + from app.apps.access import app_role, at_least + from app.apps.tree import entry_of, resolve_form + from app.auth.directory import PURPOSES, field_pool, principal_fields, review_pool + from app.db.models_apps import App, AppVersion + from app.db.models_process import Task + + if purpose not in PURPOSES: + raise CodedHTTPException(422, f"候选的用途只能是 {' / '.join(PURPOSES)}", "candidates_purpose") + if purpose == "review": + return (await review_pool(session)).pick(q, limit) | {"groups": []} + principal = current_principal() + bindings: dict[str, Any] | None = None + form: Any = None + task = await session.get(Task, task_id) if task_id else None + app = await session.get(App, task.app_id if task is not None else app_id) if (task or app_id) else None + if app is None or (task_id and task is None): + raise CodedHTTPException(404, "应用或待办不存在", "candidates_not_found") + if not at_least(await app_role(session, principal, app), "use"): + raise CodedHTTPException(403, "你没有这个应用的使用权限,看不到它的候选人", "candidates_forbidden") + if task is not None: + form = task.form + if task.app_version is not None: + bindings = await session.scalar(select(AppVersion.bindings).where( + AppVersion.app_id == app.id, AppVersion.version == task.app_version)) + else: + version = (await session.execute(select(AppVersion).where( + AppVersion.app_id == app.id, AppVersion.version == app.published_version))).scalar_one_or_none() \ + if app.published_version else None + spec = version.spec if version is not None else {} + found = entry_of(spec or {}, entry or "") if entry else None + form = resolve_form(spec or {}, found.get("form")) if found else None + bindings = version.bindings if version is not None else None + target = principal_fields(form).get(field or "") + if target is None: + raise CodedHTTPException(404, "表单里没有这个选人控件", "candidates_not_found") + return (await field_pool(session, app, bindings, target)).pick(q, limit) + + # ---------------------------------------------------------------- 审计 diff --git a/backend/app/apps/entries.py b/backend/app/apps/entries.py index 56bd7c69..76b6980a 100644 --- a/backend/app/apps/entries.py +++ b/backend/app/apps/entries.py @@ -113,6 +113,13 @@ async def start_entry(app_id: str, entry_id: str, values: dict[str, Any], princi form = resolve_form(spec, entry.get("form")) or {"type": "object", "properties": {}} if errors := validate_form(form, values if isinstance(values, dict) else None): raise FormRejected(errors) + # 选人控件:开了登录时按候选池核对(auth/directory.py,05-已知遗留 (b))。以前原样传下去,填错的名字会把待办 + # 派给一个不存在的人——没人能处理,也不报错 + from app.auth.directory import principal_errors + + if errors := await principal_errors(session, app, version.bindings, form, values, + getattr(principal, "mode", "off")): + raise FormRejected(errors) # 表单里的文件(docs/design/normal-user-apps.md N9):核对是提交人在这个入口传的、还没用过,先占住;表单值换成 # 登记的那一份引用(连同 sha256 进运行输入、随封存固定)。发起成功绑到这次运行,发起不成放回去 values, claimed = await _claim_files(form, values, principal, app.id, entry_id) diff --git a/backend/app/apps/forms.py b/backend/app/apps/forms.py index fc4f5ff1..d0f63b67 100644 --- a/backend/app/apps/forms.py +++ b/backend/app/apps/forms.py @@ -28,6 +28,8 @@ _FIELD_KEYS = frozenset({ "minLength", "maxLength", "maxItems", "format", "items", "x-widget", # 文件控件的限制:accept(类型或扩展名的列表)、max_mb(docs/design/normal-user-apps.md §3.8) "x-file", + # 选人控件的候选池:{pool: app | group:<组> | slot:<岗位>}(auth/directory.py,05-已知遗留 (b)) + "x-principal", }) #: 文件控件只认这几项写法 _FILE_FIELD_KEYS = frozenset({"type", "title", "description", "readOnly", "x-widget", "x-file"}) @@ -88,6 +90,13 @@ def _field_problems(field: Any) -> list[str]: return out + _file_field_problems(field) if "x-file" in field: out.append("不是文件控件,不能写文件的限制") + if "x-principal" in field: + from app.auth.directory import pool_problem + + if field.get("x-widget") != "principal": + out.append("不是选人控件,不能写候选范围") + elif problem := pool_problem(field["x-principal"]): + out.append(problem) if kind not in FORM_TYPES: out.append(f"的类型只能是 {' / '.join(FORM_TYPES)}") if "format" in field and field["format"] not in FORM_FORMATS: @@ -153,7 +162,7 @@ def _clean_schema(form: dict[str, Any]) -> dict[str, Any]: continue props[name] = {k: v for k, v in field.items() if k in _FIELD_KEYS and k not in ("format", "x-widget", "x-enum-labels", "title", - "description", "readOnly", "default")} + "description", "readOnly", "default", "x-principal")} return {"type": "object", "properties": props, "required": [r for r in form.get("required") or [] if r in props]} diff --git a/backend/app/apps/spec.py b/backend/app/apps/spec.py index a64b7df0..e8f3edfd 100644 --- a/backend/app/apps/spec.py +++ b/backend/app/apps/spec.py @@ -47,9 +47,10 @@ FILE_FIELD_TYPE = "object" FILE_REF_KEYS = ("file_id", "name", "size", "sha256", "type") #: 表单字段能用的 format。不支持 pattern:JS 和 Python 的正则语义不同,会出现前端放行、后端拒收 FORM_FORMATS = ("date", "date-time", "email", "uri") -#: 表单校验错误的 code。file:文件控件的值不对(没上传、不是本人传的、已经用过、类型或大小不符,app/files/store.py) +#: 表单校验错误的 code。file:文件控件的值不对(没上传、不是本人传的、已经用过、类型或大小不符,app/files/store.py); +#: principal:开了登录时选人控件的值不在候选池里(auth/directory.principal_errors) FORM_ERROR_CODES = ("required", "type", "enum", "minimum", "maximum", "min_length", "max_length", "max_items", "format", - "file") + "file", "principal") class _Strict(BaseModel): diff --git a/backend/app/auth/directory.py b/backend/app/auth/directory.py index 6f689ec1..d981f9bc 100644 --- a/backend/app/auth/directory.py +++ b/backend/app/auth/directory.py @@ -1,6 +1,10 @@ """人和组的名录:选人控件的候选(GET /api/directory)、应用检查核对岗位绑定和授权、改授权时核对填的名字,同一处取、 同一个口径(停用的人也在名录里,标 disabled:已经绑着他的岗位、授权要能认出「这个人停用了」,不是「没有这个人」)。 +使用者选人(表单里的选人控件、个人应用提交审核)不拿整份名录,拿按用途算的候选池(candidates,GET +/api/directory/candidates):只有这件事能选的人,只给用户名和显示名,停用的不列;开了登录时提交的值按同一个池核对 +(docs/design/next-round/05-已知遗留.md (b))。名录整份仍只给搭建者。 + 名录里对不上怎么算看登录方式(同前端 PrincipalPicker): - 账号密码登录:人和组都在本系统里建,名录里没有就是填错了(或人被删了),停用的人处理不了待办——都算错; - 单点登录:人和组第一次登录后才进名录,名录里没有的照实提醒、不拒收;停用的人名录里认得出,照样算错; @@ -10,7 +14,7 @@ from __future__ import annotations from collections.abc import Iterable from dataclasses import dataclass, field -from typing import Literal +from typing import Any, Literal from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession @@ -95,3 +99,175 @@ def mismatch(directory: Directory, mode: str, kind: str, name: str) -> Mismatch return Mismatch(kind, name, problem, False, f"{what}名录里还没有:单点登录的人和组第一次登录后才进名录,名字要和身份提供方里的一致才生效") return Mismatch(kind, name, problem, True, f"{what}名录里没有") + + +# -------------------------------------------------------------------------- +# 候选池:使用者选人时能选谁(05-已知遗留 (b)) +# -------------------------------------------------------------------------- + +#: 候选池的用途:form 表单里的选人控件(入口表单、待办表单);review 个人应用提交审核选审核的人 +PURPOSES = ("form", "review") +#: 一次最多给几位候选 +CANDIDATES_MAX = 50 +#: 表单字段声明候选池(x-principal.pool):app 能用这个应用的人(缺省);group:<组> 这个组的成员;slot:<岗位> 这个岗位 +#: 绑定的人和组(按应用版本的 bindings) +POOL_APP = "app" +POOL_GROUP = "group:" +POOL_SLOT = "slot:" + + +def pool_problem(value: Any) -> str | None: + """x-principal 的写法对不对:只能是 {pool: app | group:<组> | slot:<岗位>}。没问题为 None。""" + if not isinstance(value, dict) or set(value) != {"pool"} or not isinstance(value.get("pool"), str): + return "的候选范围只能写 {\"pool\": \"app\"}、{\"pool\": \"group:组名\"} 或 {\"pool\": \"slot:岗位\"}" + pool = value["pool"].strip() + if pool == POOL_APP: + return None + for prefix in (POOL_GROUP, POOL_SLOT): + if pool.startswith(prefix) and pool[len(prefix):].strip(): + return None + return "的候选范围只能是 app(能用这个应用的人)、group:组名 或 slot:岗位" + + +def pool_of(field: Any) -> str: + """表单字段声明的候选池;没声明、写法不对的按缺省(能用这个应用的人)。""" + setting = field.get("x-principal") if isinstance(field, dict) else None + if isinstance(setting, dict) and pool_problem(setting) is None: + return setting["pool"].strip() + return POOL_APP + + +@dataclass +class Pool: + """一个候选池。open:池就是全部启用的人(应用授权给了所有人)——不带前缀不列,免得等于发通讯录。""" + users: dict[str, DirectoryUser] = field(default_factory=dict) + groups: dict[str, DirectoryGroup] = field(default_factory=dict) + open: bool = False + + def has(self, name: str) -> bool: + """提交的这个名字在不在池里(用户名或组名)。""" + return name in self.users or name in self.groups + + def pick(self, q: str | None, limit: int = CANDIDATES_MAX) -> dict[str, Any]: + """接口给的那一份:{users: [{username, display_name}], groups: [{name, label}], truncated};按前缀筛 + (用户名或显示名,不分大小写)。池是 open 的、又没带前缀:什么都不给,needs_query 为真。""" + prefix = (q or "").strip().lower() + if self.open and not prefix: + return {"users": [], "groups": [], "truncated": False, "needs_query": True} + limit = max(1, min(int(limit or CANDIDATES_MAX), CANDIDATES_MAX)) + + def hit(*names: str) -> bool: + return not prefix or any(n.lower().startswith(prefix) for n in names if n) + + users = [u for u in sorted(self.users.values(), key=lambda u: u.username) if hit(u.username, u.display_name)] + groups = [g for g in sorted(self.groups.values(), key=lambda g: g.name) if hit(g.name, g.label)] + return {"users": [{"username": u.username, "display_name": u.display_name} for u in users[:limit]], + "groups": [{"name": g.name, "label": g.label} for g in groups[:limit]], + "truncated": len(users) > limit or len(groups) > limit} + + +async def _members(session: AsyncSession, group: str) -> dict[str, DirectoryUser]: + from app.db.models import Group, GroupMember, User + + rows = (await session.execute( + select(User).join(GroupMember, GroupMember.user_id == User.id).join(Group, Group.id == GroupMember.group_id) + .where(Group.name == group, User.disabled.is_(False)))).scalars() + return {u.username: DirectoryUser(u.username, u.display_name or u.username, False) for u in rows} + + +async def _people(session: AsyncSession, items: Iterable[tuple[str, str]]) -> Pool: + """一组(种类, 名字)展开成池:人按名录认(停用的、名录里没有的不列),组列出组本身、再展开成员。""" + from app.db.models import Group, User + + pool = Pool() + items = list(items) + names = sorted({n for k, n in items if k == "user" and n}) + if names: + for u in (await session.execute(select(User).where(User.username.in_(names), User.disabled.is_(False))))\ + .scalars(): + pool.users[u.username] = DirectoryUser(u.username, u.display_name or u.username, False) + groups = sorted({n for k, n in items if k == "group" and n}) + if groups: + for g in (await session.execute(select(Group).where(Group.name.in_(groups)))).scalars(): + pool.groups[g.name] = DirectoryGroup(g.name, g.label or g.name) + pool.users.update(await _members(session, g.name)) + return pool + + +async def _everyone(session: AsyncSession) -> Pool: + from app.db.models import Group, User + + pool = Pool(open=True) + for u in (await session.execute(select(User).where(User.disabled.is_(False)))).scalars(): + pool.users[u.username] = DirectoryUser(u.username, u.display_name or u.username, False) + for g in (await session.execute(select(Group))).scalars(): + pool.groups[g.name] = DirectoryGroup(g.name, g.label or g.name) + return pool + + +async def app_pool(session: AsyncSession, app: Any) -> Pool: + """能用这个应用的人:使用及以上的授权(人、组展开成员),加上管理它的人(管理员、建它的人)。授权给了所有人的,池就是 + 全部启用的人(open)。按 apps/access 的口径:搭建者没有授权时只能查看全部实例,不算「能用」的候选。""" + from app.apps.access import RANK, grants_of + from app.db.models import User + + grants = [g for g in await grants_of(session, app.id) if RANK.get(g.role, 0) >= RANK["use"]] + if any(g.principal_kind == "everyone" for g in grants): + return await _everyone(session) + admins = (await session.execute(select(User.username).where(User.role == "admin"))).scalars() + items = [(g.principal_kind, g.principal) for g in grants] + [("user", a) for a in admins] + if getattr(app, "created_by", None): + items.append(("user", app.created_by)) + return await _people(session, items) + + +async def field_pool(session: AsyncSession, app: Any, bindings: dict[str, Any] | None, field: Any) -> Pool: + """表单里一个选人控件的候选池(按字段的 x-principal,缺省能用这个应用的人)。bindings 是这次用到的应用版本的绑定。""" + pool = pool_of(field) + if pool.startswith(POOL_GROUP): + name = pool[len(POOL_GROUP):].strip() + return Pool(users=await _members(session, name)) + if pool.startswith(POOL_SLOT): + bound = ((bindings or {}).get("slots") or {}).get(pool[len(POOL_SLOT):].strip()) + items = [(str(b.get("kind") or "user"), str(b.get("name") or "")) for b in bound or [] if isinstance(b, dict)] + if any(k == "everyone" for k, _ in items): + return await _everyone(session) + return await _people(session, items) + return await app_pool(session, app) + + +async def review_pool(session: AsyncSession) -> Pool: + """个人应用提交审核能交给谁:启用的搭建者和管理员(同 blueprints/personal._review_targets 的规则)。""" + from app.db.models import User + + rows = (await session.execute(select(User).where(User.role.in_(("builder", "admin")), + User.disabled.is_(False)))).scalars() + return Pool(users={u.username: DirectoryUser(u.username, u.display_name or u.username, False) for u in rows}) + + +def principal_fields(form: Any) -> dict[str, dict[str, Any]]: + """表单里的选人控件:{字段名: 字段}。""" + props = form.get("properties") if isinstance(form, dict) else None + return {name: f for name, f in (props or {}).items() if isinstance(f, dict) and f.get("x-widget") == "principal"} + + +#: 选人控件的值不在候选池里时表单错误的 code(apps/spec.FORM_ERROR_CODES 的 principal) +PRINCIPAL_ERROR_CODE = "principal" + + +async def principal_errors(session: AsyncSession, app: Any, bindings: dict[str, Any] | None, form: Any, + values: Any, mode: str) -> list[dict[str, str]]: + """开了登录时按候选池核对表单里选人控件的值:不在池里的按表单错误交回(code=principal),话写成「「x」不在可选的人里」。 + 关登录不核对:没有用户表,签批记的是各自填的署名。没填的不在这里报(必填由表单校验报)。""" + if mode == "off" or not isinstance(values, dict): + return [] + out: list[dict[str, str]] = [] + for name, f in principal_fields(form).items(): + value = values.get(name) + if value in (None, "") or not isinstance(value, str): + continue + pool = await field_pool(session, app, bindings, f) + if not pool.has(value.strip()): + hint = ":单点登录的人要先登录一次才进名录" if mode == "oidc" else ",请从候选中选择" + out.append({"path": name, "code": PRINCIPAL_ERROR_CODE, "message": f"「{value.strip()}」不在可选的人里{hint}"}) + return out diff --git a/backend/app/main.py b/backend/app/main.py index 657c917a..b2647a1b 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -238,6 +238,7 @@ for router, guard in ( (identity_api.router, _ADMIN), # 选人用的名录(用户名、显示名、是否停用、组名):岗位绑定、应用授权由搭建者做,只读 (identity_api.directory_router, _BUILDER), + (identity_api.candidates_router, _USER), # 能力登记、端点:搭建者只读,改动在接口里要管理员 (capabilities_api.router, _BUILDER), # 普通用户建应用:个人应用的接口共用 /api/apps 前缀,要在 apps_api 之前登记;蓝图助手共用 /api/copilot 前缀, diff --git a/backend/app/process/tasks.py b/backend/app/process/tasks.py index 6ec167af..818b4f76 100644 --- a/backend/app/process/tasks.py +++ b/backend/app/process/tasks.py @@ -406,6 +406,22 @@ def _check_hash(card: dict[str, Any], given: Any, *, approving: bool) -> None: status_code=422, code="payload_hash_mismatch") +async def _check_principals(session: AsyncSession, task: Task, principal: Any, values: dict[str, Any]) -> None: + """待办表单里的选人控件:开了登录时按候选池核对(同入口表单,auth/directory.principal_errors)。不属于应用的待办 + (试运行、画布)没有池,不核对。""" + from app.auth.directory import principal_errors + + if not task.app_id or getattr(principal, "mode", "off") == "off": + return + app = await session.get(App, task.app_id) + if app is None: + return + bindings = await session.scalar(select(AppVersion.bindings).where( + AppVersion.app_id == task.app_id, AppVersion.version == task.app_version)) if task.app_version else None + if errors := await principal_errors(session, app, bindings, task.form, values, principal.mode): + raise FormInvalid(errors) + + def _validated(form: dict[str, Any], values: dict[str, Any]) -> dict[str, Any]: from app.apps.forms import validate_form @@ -491,6 +507,8 @@ async def respond(task_id: str, principal: Any, *, decision: Any, values: dict[s rows = (await _assignments(session, [task.id])).get(task.id, []) mine = _pick(task, rows, principal) response, verdict = _engine_response(task, approval, decision, values, note, payload_hash) + if task.form and verdict in ("approve", "submit") and values: + await _check_principals(session, task, principal, values) response = await _bind_files(session, task, principal, response) now = utcnow() marked = await session.execute( diff --git a/backend/tests/test_directory_candidates.py b/backend/tests/test_directory_candidates.py new file mode 100644 index 00000000..c8fe3a36 --- /dev/null +++ b/backend/tests/test_directory_candidates.py @@ -0,0 +1,266 @@ +"""使用者选人:按用途给的候选池 + 开登录时服务端核对(docs/design/next-round/05-已知遗留.md (b))。 + +以前使用者在入口表单、待办表单的选人控件里没有候选(名录只给搭建者),服务端也不核对:填错的名字会把待办派给一个不存在 +的人,没人能处理,也不报错。现在: +- GET /api/directory/candidates 按用途给候选池,只有用户名和显示名,停用的不列;form 用途缺省是能用这个应用的人(使用 + 及以上的授权展开组成员,加上管理员和建应用的人),字段可以用 x-principal 声明别的池;应用授权给了所有人时不带前缀不列; + 调用的人不能用这个应用时 403;review 用途是启用的搭建者和管理员; +- 开了登录时,入口表单和待办表单的选人控件按同一个池核对,不在池里 422(errors 的 code=principal);关登录照旧手填; +- /api/directory 仍然只给搭建者。 + +名字都是虚构的。 +""" +from __future__ import annotations + +import asyncio +import uuid + +import pytest +from sqlalchemy import delete, select + +from app.apps.forms import validate_form_schema +from app.core.artifact_store import graph_hash +from app.db.base import SessionLocal, utcnow +from app.db.models import App, AppGrant, Group, GroupMember, Run, Task, User, Workflow, WorkflowVersion +from app.engine.runner import run_manager +from tests.fakes import auth_env + + +@pytest.fixture(autouse=True) +async def engine_up(): + await run_manager.setup() + yield + await run_manager.shutdown() + + +def node(nid, ntype, label=None, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": label or nid, "config": config}} + + +def chain(*nodes): + return {"nodes": list(nodes), "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]} + + +PICK = {"type": "string", "title": "审批人", "x-widget": "principal"} + +GRAPH = chain( + node("start", "input", "开始", fields=[{"name": "approver"}, {"name": "reviewer"}]), + node("review", "human", "复核", mode="input", title="请指定复核人", + form={"type": "object", "properties": {"reviewer": {**PICK, "title": "复核人"}}}), + node("out", "output", "成果", fields=[{"name": "r", "value": "{{ input.approver }}"}])) + + +async def _workflow() -> tuple[str, str]: + async with SessionLocal() as session: + wf = Workflow(name=f"选人-{uuid.uuid4().hex[:6]}", graph=GRAPH, status="published", published_version=1, + version=1) + session.add(wf) + await session.flush() + session.add(WorkflowVersion(workflow_id=wf.id, version=1, graph=GRAPH, graph_hash=graph_hash(GRAPH), + level="published", published_at=utcnow())) + await session.commit() + return wf.id, graph_hash(GRAPH) + + +def _spec(wf: str, hash_: str, approver: dict | None = None) -> dict: + return { + "format": "agentlab.app/1", "name": "选人", + "pins": [{"alias": "main", "workflow_id": wf, "version": 1, "graph_hash": hash_, "use": "entry"}], + "entries": [{"id": "apply", "label": "发起", "pin": "main", "form": {"$ref": "#/forms/apply"}, + "map": {"approver": "form.approver"}}], + "forms": {"apply": {"type": "object", "properties": {"approver": approver or PICK}}}, + "pages": [{"id": "home", "title": "首页", "body": [ + {"id": "new", "type": "entry_button", "props": {"entry": "apply"}}]}], + } + + +async def _new_app(root, spec: dict) -> str: + """root 建一个应用并发布,授权使用给 uma、停用的 dan 和组「财务」。""" + res = await root.post("/api/apps", json={"name": f"选人-{uuid.uuid4().hex[:4]}", "spec": spec}) + assert res.status_code == 201, res.text + app_id = res.json()["id"] + published = (await root.post(f"/api/apps/{app_id}/publish", json={})).json() + assert published["ok"], published + async with SessionLocal() as session: + session.add_all([AppGrant(app_id=app_id, principal_kind="user", principal="uma", role="use"), + AppGrant(app_id=app_id, principal_kind="user", principal="dan", role="use"), + AppGrant(app_id=app_id, principal_kind="group", principal="财务", role="use")]) + await session.commit() + return app_id + + +@pytest.fixture +async def world(): + """开账号密码登录:管理员 root(建应用)、搭建者 bob、使用者 uma(授权使用)、ulf(没授权)、停用的 dan(授权使用); + 组「财务」(fay、fin),整组授权使用。""" + await auth_env.reset() + root = await auth_env.turn_on_local("root") + try: + await auth_env.make_user("bob", "builder") + await auth_env.make_user("uma", "user") + await auth_env.make_user("ulf", "user") + await auth_env.make_user("dan", "user", disabled=True) + fay = await auth_env.make_user("fay", "user") + fin = await auth_env.make_user("fin", "user") + async with SessionLocal() as session: + for username, shown in (("uma", "乌玛"), ("fay", "费伊"), ("bob", "鲍勃")): + user = (await session.execute(select(User).where(User.username == username))).scalar_one() + user.display_name = shown + group = Group(name="财务", label="财务组", source="local") + session.add(group) + await session.flush() + session.add_all([GroupMember(group_id=group.id, user_id=fay.id), GroupMember(group_id=group.id, user_id=fin.id)]) + await session.commit() + wf, hash_ = await _workflow() + app_id = await _new_app(root, _spec(wf, hash_)) + yield {"root": root, "app_id": app_id, "wf": wf, "hash": hash_} + finally: + await root.aclose() + async with SessionLocal() as session: + for model in (AppGrant, App): + await session.execute(delete(model)) + await session.commit() + await auth_env.reset() + + +def _params(app_id: str, **more) -> dict: + return {"purpose": "form", "app_id": app_id, "entry": "apply", "field": "approver", **more} + + +async def test_users_get_the_people_who_can_use_the_app_and_only_their_names(world): + uma = await auth_env.login("uma") + r = await uma.get("/api/directory/candidates", params=_params(world["app_id"])) + assert r.status_code == 200, r.text + body = r.json() + assert [u["username"] for u in body["users"]] == ["fay", "fin", "root", "uma"] + assert all(set(u) == {"username", "display_name"} for u in body["users"]), "不给停用、角色、邮箱" + assert {u["username"]: u["display_name"] for u in body["users"]}["uma"] == "乌玛" + assert body["groups"] == [{"name": "财务", "label": "财务组"}] and body["truncated"] is False + # 前缀按用户名或显示名筛 + assert [u["username"] for u in (await uma.get("/api/directory/candidates", + params=_params(world["app_id"], q="费"))).json()["users"]] == ["fay"] + # 名录整份仍然只给搭建者 + assert (await uma.get("/api/directory")).status_code == 403 + # 不能用这个应用的人拿不到它的候选 + ulf = await auth_env.login("ulf") + assert (await ulf.get("/api/directory/candidates", params=_params(world["app_id"]))).status_code == 403 + for c in (uma, ulf): + await c.aclose() + + +async def test_an_app_granted_to_everyone_needs_a_prefix(world): + async with SessionLocal() as session: + session.add(AppGrant(app_id=world["app_id"], principal_kind="everyone", principal="", role="use")) + await session.commit() + ulf = await auth_env.login("ulf") + body = (await ulf.get("/api/directory/candidates", params=_params(world["app_id"]))).json() + assert body["users"] == [] and body["needs_query"] is True + found = (await ulf.get("/api/directory/candidates", params=_params(world["app_id"], q="u"))).json() + assert [u["username"] for u in found["users"]] == ["ulf", "uma"], "停用的 dan 不列" + capped = (await ulf.get("/api/directory/candidates", params=_params(world["app_id"], q="f", limit=1))).json() + assert len(capped["users"]) == 1 and capped["truncated"] is True + await ulf.aclose() + + +async def test_review_candidates_are_builders_and_admins(world): + uma = await auth_env.login("uma") + body = (await uma.get("/api/directory/candidates", params={"purpose": "review"})).json() + assert [u["username"] for u in body["users"]] == ["bob", "root"] and body["groups"] == [] + assert (await uma.get("/api/directory/candidates", params={"purpose": "nope"})).status_code == 422 + await uma.aclose() + + +async def test_starting_with_a_name_outside_the_pool_is_refused(world): + uma = await auth_env.login("uma") + url = f"/api/apps/{world['app_id']}/entries/apply" + bad = await uma.post(url, json={"values": {"approver": "wangwu"}}) + assert bad.status_code == 422, bad.text + [error] = bad.json()["errors"] + assert (error["path"], error["code"]) == ("approver", "principal") + assert error["message"].startswith("「wangwu」不在可选的人里") + disabled = await uma.post(url, json={"values": {"approver": "dan"}}) + assert disabled.status_code == 422 and disabled.json()["errors"][0]["code"] == "principal" + for ok in ("fay", "财务"): + good = await uma.post(url, json={"values": {"approver": ok}}) + assert good.status_code == 201, good.text + await uma.aclose() + + +async def test_a_task_form_is_checked_against_the_same_pool(world): + uma = await auth_env.login("uma") + started = await uma.post(f"/api/apps/{world['app_id']}/entries/apply", json={"values": {"approver": "fay"}}) + assert started.status_code == 201, started.text + run_id = started.json()["run_id"] + task = None + for _ in range(200): + async with SessionLocal() as session: + task = (await session.execute(select(Task).where(Task.run_id == run_id, Task.status == "pending")))\ + .scalars().first() + if task is not None: + break + await asyncio.sleep(0.05) + assert task is not None + cands = await uma.get("/api/directory/candidates", params={"purpose": "form", "task_id": task.id, + "field": "reviewer"}) + assert cands.status_code == 200 and "fin" in [u["username"] for u in cands.json()["users"]] + root = world["root"] + bad = await root.post(f"/api/tasks/{task.id}/respond", json={"decision": "submit", "values": {"reviewer": "wangwu"}}) + assert bad.status_code == 422, bad.text + assert bad.json()["errors"][0]["code"] == "principal" + good = await root.post(f"/api/tasks/{task.id}/respond", json={"decision": "submit", "values": {"reviewer": "fin"}}) + assert good.status_code == 200, good.text + await uma.aclose() + + +async def test_a_declared_pool_narrows_the_candidates(world): + """x-principal 声明了 group:财务:候选只有组里的人,提交组外的人也拒。""" + app_id = await _new_app(world["root"], _spec(world["wf"], world["hash"], + {**PICK, "x-principal": {"pool": "group:财务"}})) + uma = await auth_env.login("uma") + body = (await uma.get("/api/directory/candidates", params=_params(app_id))).json() + assert [u["username"] for u in body["users"]] == ["fay", "fin"] and body["groups"] == [] + refused = await uma.post(f"/api/apps/{app_id}/entries/apply", json={"values": {"approver": "uma"}}) + assert refused.status_code == 422 and refused.json()["errors"][0]["code"] == "principal" + await uma.aclose() + + +async def test_login_off_takes_any_name(): + await auth_env.reset() + wf, hash_ = await _workflow() + from httpx import ASGITransport, AsyncClient + + from app.main import app as asgi + + async with AsyncClient(transport=ASGITransport(app=asgi), base_url="http://test") as c: + res = await c.post("/api/apps", json={"name": f"关登录-{uuid.uuid4().hex[:4]}", "spec": _spec(wf, hash_)}) + app_id = res.json()["id"] + assert (await c.post(f"/api/apps/{app_id}/publish", json={})).json()["ok"] + started = await c.post(f"/api/apps/{app_id}/entries/apply", json={"values": {"approver": "随手填的名字"}}) + assert started.status_code == 201, started.text + async with SessionLocal() as session: + await session.execute(delete(Run).where(Run.id == started.json()["run_id"])) + await session.execute(delete(App).where(App.id == app_id)) + await session.commit() + + +def test_x_principal_is_checked_in_the_form_schema(): + ok = {"type": "object", "properties": {"a": {**PICK, "x-principal": {"pool": "slot:finance"}}}} + assert validate_form_schema(ok) == [] + bad = {"type": "object", "properties": { + "a": {**PICK, "x-principal": {"pool": "everyone"}}, + "b": {"type": "string", "x-principal": {"pool": "app"}}, + "c": {**PICK, "x-principal": "app"}}} + problems = validate_form_schema(bad) + assert len(problems) == 3, problems + assert any("「b」不是选人控件" in p for p in problems) + + +async def test_users_table_is_untouched_by_candidates(world): + """候选接口只读:不建用户、不改名录。""" + async with SessionLocal() as session: + before = len((await session.execute(select(User.id))).all()) + uma = await auth_env.login("uma") + await uma.get("/api/directory/candidates", params=_params(world["app_id"], q="zz")) + await uma.aclose() + async with SessionLocal() as session: + assert len((await session.execute(select(User.id))).all()) == before From ee7b8b2f97ecab1aba081d4fd3a2d0cbdae94d62 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Fri, 16 Oct 2026 10:25:28 +0800 Subject: [PATCH 12/39] =?UTF-8?q?=E5=89=96=E6=9E=90=E4=B8=8E=E6=8E=A2?= =?UTF-8?q?=E6=9F=A5=E7=9A=84=E6=96=B9=E8=A8=80=E4=BF=AE=E6=AD=A3=EF=BC=88?= =?UTF-8?q?C1=EF=BC=89=E5=92=8C=E6=B2=A1=E6=9C=89=E7=9C=9F=E5=BA=93?= =?UTF-8?q?=E6=97=B6=E7=9A=84=E6=96=B9=E8=A8=80=E9=AA=8C=E8=AF=81=EF=BC=88?= =?UTF-8?q?C2=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 为什么:剖析是全站唯一主动查业务库的地方,以前只在守卫放行和字符串片段两层上测过,从没在真库上跑过。 本机四个容器(PG 16、MySQL 8.4、MariaDB 11、Oracle Free 23)实测,旧的 IN 列表写法在 PostgreSQL 上整数子列 对文字主键直接报 operator does not exist: character varying = integer,关系一条都核对不了。 C1 方言修正: - IN 列表的字面量按被指向列的类型族写(key_literals):父列文字写 '12',父列数字只收整数写法的值,父列是时间、 对不上的不拼并在备注里写「两端类型不同」;子表是 CHAR(n) 时去掉补齐的尾随空格。 - 列类型改存按方言编译的完整写法(VARCHAR2(20 CHAR)、NUMBER(10, 0)、ENUM('a','b')、TIMESTAMP WITH TIME ZONE), 编译不了退回 str()。SQLAlchemy 把 Oracle 的 VARCHAR2 反射成通用 VARCHAR(容器实测),照数据字典写成 VARCHAR2。 ENUM / SET 另记 enum_values;Oracle 小写带引号的表和列记 quote_name;各索引(含主键)的首列记进 index_first_columns(下一波取值观察判断能不能对大表发窗口查询)。新探查的快照哈希会变,旧运行封存的不受影响。 - catalog._type_family 认 ENUM / SET 是文字、INTERVAL 不是数字,去掉引号里的取值再认;新增 column_family (有 enum_values 的是文字,PostgreSQL 枚举编译出来只有类型名);is_date_column 不再把 ENUM('date') 认成日期列。 - 码值候选:ENUM / SET 不看列名直接算候选,ENUM 列定义里这次没出现的取值也写进码值(含义待填);VARCHAR2(20 CHAR) 的长度按 20 算;CHAR 码值去尾随空格;MySQL 上写明按排序规则分组、比较(可能不分大小写)。 - 统计信息估出的行数不超过整表统计上限时再跑一次有上限的计数确认,统计过期的大表不再被当成小表做整表统计。 - 日期列的最小值、最大值只看 0001-01-02 到 9999-12-31 之间的值,零日期、占位日期、PG 的 ±infinity 不计入, 照实写「另有 N 个无效日期未计入」,日期范围里带 invalid 个数(界面显示)。 - 新增取值观察用的方言方法 date_literal / day_trunc / sample_from,以及按日行数、时间窗口里的取值分布、块采样三类 查询的拼法(窗口上端截在 9999-12-31 之前,不让 infinity 多出一天)。 - 说明文字不再说「抽样」:大表看的是「按存储顺序取前 N 行」;设置项「抽样键值数」改叫「核对键值数」。 C2 验证: - tests/test_profile_dialect_golden.py:五种方言 × 全部剖析和观察查询的完整 SQL 钉在 tests/fixtures/dialect/, sqlglot 逐条解析、过只读守卫,再用 qualify + annotate_types 核对 IN 两边类型族一致(附旧写法的对照)。 - scripts/dialect-it.sh(不进 CI):本机起四个容器,建虚构夹具(保留字列、Oracle 小写带引号的表、CHAR 码值、 MySQL ENUM、零日期 / infinity / 占位日期、做旧的统计信息、超上限的大表、带索引的日期列),用 AgentLab 自己的 探查和剖析代码跑全部查询,断言零报错、结论和手算一致。四种库都通过;MariaDB 插入时自己更新行数估算,统计信息 做不旧,照实打印。把 IN 列表改回旧写法回跑,PG 上照样报出上面那条错误。跑完删容器、留镜像。 --- backend/app/data/catalog.py | 40 +- backend/app/data/catalog_profile.py | 346 +++++++++++++--- backend/app/data/introspect.py | 86 +++- backend/tests/fixtures/dialect/mariadb.sql | 27 ++ backend/tests/fixtures/dialect/mysql.sql | 27 ++ backend/tests/fixtures/dialect/oracle.sql | 27 ++ backend/tests/fixtures/dialect/postgresql.sql | 27 ++ backend/tests/fixtures/dialect/sqlite.sql | 27 ++ backend/tests/test_catalog_draft_facts.py | 2 +- backend/tests/test_catalog_profile.py | 4 +- backend/tests/test_catalog_profile_units.py | 12 +- backend/tests/test_catalog_semantics.py | 2 +- backend/tests/test_profile_dialect_golden.py | 178 ++++++++ backend/tests/test_profile_dialect_units.py | 296 +++++++++++++ frontend/src/lib/terms.ts | 14 +- frontend/src/pages/catalog/ProfileDialog.tsx | 5 + .../src/pages/catalog/ProfileSettings.tsx | 2 +- frontend/src/types.ts | 3 +- scripts/check-catalog-profile.mjs | 6 +- scripts/dialect-it.sh | 62 +++ scripts/dialect_it.py | 388 ++++++++++++++++++ 21 files changed, 1499 insertions(+), 82 deletions(-) create mode 100644 backend/tests/fixtures/dialect/mariadb.sql create mode 100644 backend/tests/fixtures/dialect/mysql.sql create mode 100644 backend/tests/fixtures/dialect/oracle.sql create mode 100644 backend/tests/fixtures/dialect/postgresql.sql create mode 100644 backend/tests/fixtures/dialect/sqlite.sql create mode 100644 backend/tests/test_profile_dialect_golden.py create mode 100644 backend/tests/test_profile_dialect_units.py create mode 100755 scripts/dialect-it.sh create mode 100644 scripts/dialect_it.py diff --git a/backend/app/data/catalog.py b/backend/app/data/catalog.py index 6ca00287..678f249b 100644 --- a/backend/app/data/catalog.py +++ b/backend/app/data/catalog.py @@ -1365,7 +1365,9 @@ def is_date_column(name: str, type_name: str | None) -> bool: 数据剖析(表里只有一个日期类列时提议它作业务日期)和 SQL 检查(按业务日期以外的时间列统计)共用这一个判断。 以前两边各写一套:剖析认的「唯一日期列」,SQL 检查可能认为还有别的时间列,反过来也一样,两边对不上。 """ - t = (type_name or "").upper() + t = _type_words(type_name) + if t.startswith(("ENUM", "SET")) or "INTERVAL" in t: + return False if "DATE" in t or "TIMESTAMP" in t: return True if t and not any(k in t for k in _TEXT_TYPES): @@ -1400,11 +1402,29 @@ def _name_index(tables: Mapping[str, Any]) -> dict[tuple[str, str], set[str]]: return index +#: 类型写法里带引号的部分(ENUM('int','date') 的取值):认类型时去掉,取值里的词不能让一列变成数字或时间 +_QUOTED_IN_TYPE = re.compile(r"'(?:[^']|'')*'") + + +def _type_words(type_name: str | None) -> str: + """认类型用的写法:大写、去掉引号里的取值。探查存的是按方言编译的完整写法(ENUM('paid','refund')、 + VARCHAR2(20 CHAR)、TIMESTAMP WITH TIME ZONE),不再是 SQLAlchemy 的通用写法。""" + return _QUOTED_IN_TYPE.sub("''", (type_name or "").upper()).strip() + + def _type_family(type_name: str | None) -> str | None: - """列类型的大类:数字、文字、时间;认不出来返回 None(不据此排除)。""" - t = (type_name or "").upper() + """列类型的大类:数字、文字、时间;认不出来返回 None(不据此排除)。 + + 认的是按方言编译的完整写法:MySQL 的 ENUM / SET 是文字(取值列表里的词不算),Oracle 的 INTERVAL DAY TO SECOND + 是时长、不算数字(以前按子串「INT」认成了数字)。PostgreSQL 的枚举编译出来是类型名本身,只看类型认不出来, + 要看列上的取值列表(column_family)。""" + t = _type_words(type_name) if not t: return None + if t.startswith(("ENUM", "SET")): + return "text" + if "INTERVAL" in t: + return None if any(k in t for k in ("CHAR", "TEXT", "CLOB", "STRING", "UUID")): return "text" if any(k in t for k in ("INT", "NUMBER", "NUMERIC", "DECIMAL", "SERIAL", "REAL", "FLOAT", "DOUBLE")): @@ -1414,6 +1434,16 @@ def _type_family(type_name: str | None) -> str | None: return None +def column_family(col: Mapping[str, Any] | None) -> str | None: + """表结构里一列的类型大类:有取值列表(探查记下的 enum_values:MySQL 的 ENUM / SET、PostgreSQL 的枚举)的是 + 文字,其余按类型写法认(_type_family)。""" + if not isinstance(col, Mapping): + return None + if col.get("enum_values"): + return "text" + return _type_family(col.get("type")) + + def _id_stem(column: str) -> str | None: m = _SNAKE_ID.match(column) or _CAMEL_ID.match(column) return m.group("stem") if m else None @@ -1469,8 +1499,8 @@ def infer_name_relations(schema_cache: Mapping[str, Any] | None, table: str, *, pk = list(target_meta.get("primary_key") or []) if len(pk) != 1: continue - pk_type = next((c.get("type") for c in target_meta.get("columns") or [] if c.get("name") == pk[0]), None) - mine, theirs = _type_family(col.get("type")), _type_family(pk_type) + pk_col = next((c for c in target_meta.get("columns") or [] if c.get("name") == pk[0]), None) + mine, theirs = column_family(col), column_family(pk_col) if mine and theirs and mine != theirs: continue out.append(_relation(table, [name], target, pk, "name", cardinality=_cardinality(meta, [name]))) diff --git a/backend/app/data/catalog_profile.py b/backend/app/data/catalog_profile.py index 8eee4488..cc9b1062 100644 --- a/backend/app/data/catalog_profile.py +++ b/backend/app/data/catalog_profile.py @@ -41,7 +41,7 @@ import re import time from collections.abc import Awaitable, Callable from dataclasses import asdict, dataclass, field -from datetime import datetime, timezone +from datetime import date as _date, datetime, timezone from decimal import Decimal from typing import Any @@ -99,7 +99,7 @@ _NUMBER_FIELDS: dict[str, tuple[str, float, float, bool]] = { "table_max_queries": ("每张表最多查询数", 1, 200, True), "table_max_s": ("每张表时长上限(秒)", 10, 600, False), "query_timeout_s": ("单条查询时限(秒)", 1, 60, False), - "sample_size": ("抽样键值数", 10, 10_000, True), + "sample_size": ("核对键值数", 10, 10_000, True), "max_scan_rows": ("整表统计行数上限", 0, 10_000_000, True), } _ENABLED_LABEL = "开启数据剖析" @@ -241,16 +241,82 @@ def sql_literal(value: Any, kind: str | None) -> str | None: return "'" + value.replace("'", "''") + "'" +#: 定长文字类型(CHAR(2)、NCHAR、bpchar、CHARACTER(2)):PostgreSQL、Oracle 取回的值带尾随空格补齐 +_FIXED_CHAR = re.compile(r"^\s*(?:N?CHAR|BPCHAR|CHARACTER)\b(?!\s+VARYING)", re.IGNORECASE) + + +def is_fixed_char(type_name: str | None) -> bool: + """定长文字列(CHAR(n)):值的尾随空格是补齐出来的,不是取值的一部分(SQL 比较时也不算)。""" + return bool(type_name) and bool(_FIXED_CHAR.match(str(type_name))) + + +def key_literals(values: list[Any], kind: str | None, *, parent: str | None, + child_type: str | None) -> tuple[list[str], int]: + """从子表取回的键值 → 去父表里对的 IN 列表字面量(去重、保持顺序):(字面量, 因为两端类型不同没拼的个数)。 + + 字面量按**被指向列**的类型族写(catalog.column_family),不是按子表取回的值: + - 父列是文字、子列是数:写成 '123'。PostgreSQL 上 varchar 列 IN (123) 直接报「operator does not exist」; + - 父列是数、子列是文字:只收整数写法的值('0012' 也照 12 写),别的('A1')计入不同类型、不拼。Oracle 上 + 数字列 IN ('A1') 会把字面量转成数报 ORA-01722,MySQL 会把 'A1' 转成 0 去比、对上不该对上的行; + - 父列是时间:键不核对(各家时间字面量写法不同、精度不同),全部计入不同类型; + - 父列类型认不出:照子表取回的值的类型写(sql_literal,以前的做法)。 + 子表是定长文字(CHAR(n))时先去掉尾随空格:PG、Oracle 取回的 'A ' 拿去和 VARCHAR 父列比就对不上,而按 CHAR 的 + 语义尾随空格本来就不算。不安全的值(反斜杠、「:名字」、控制字符、太长)照 sql_literal 的规矩跳过,不算类型不同。 + """ + fixed = is_fixed_char(child_type) + out: list[str] = [] + mismatched = 0 + for value in values: + if fixed and isinstance(value, str): + value = value.rstrip(" ") + if parent == "time": + mismatched += 1 + continue + if parent == "text": + if isinstance(value, bool): + continue + if isinstance(value, (int, float, Decimal)): + lit = sql_literal(value, "number") + lit = None if lit is None else "'" + lit + "'" + else: + lit = sql_literal(value, "text") + elif parent == "number": + if isinstance(value, str): + text = value.strip() + if not _INT_TEXT.fullmatch(text): + mismatched += 1 + continue + lit = str(int(text)) + else: + lit = sql_literal(value, "number") + else: + lit = sql_literal(value, kind) + if lit is not None: + out.append(lit) + return list(dict.fromkeys(out)), mismatched + + _DIALECT_MODULES = {"sqlite": sqlite, "postgres": postgresql, "postgresql": postgresql, "mysql": mysql, "mariadb": mysql, "oracle": oracle} +#: 日期列的最小值、最大值只看这个区间里的值(含下限、不含上限):0000-00-00(MySQL 关了严格模式写进去的零日期)、 +#: 0001-01-01、9999-12-31(「没有」「永久」的占位日期)、PostgreSQL 的 -infinity / infinity 都落在区间外。区间外的 +#: 值照实数出来(「有 N 个无效日期未计入」),不让一个占位日期把业务日期的范围撑成两千年。两端整天排除:带时刻的 +#: 9999-12-31 23:59:59、文字存的「0001-01-01 00:00:00」也算占位 +VALID_DATE_FROM = "0001-01-02" +VALID_DATE_BEFORE = "9999-12-31" +_ISO_DAY = re.compile(r"\d{4}-\d{2}-\d{2}") + + class SqlDialect: - """剖析 SQL 的方言差异:标识符怎么加引号、怎么只取前 n 行、统计信息在哪个系统视图里。只拼字符串,不连库。 + """剖析 SQL 的方言差异:标识符怎么加引号、怎么只取前 n 行、统计信息在哪个系统视图里、日期字面量和按天截断 + 怎么写、有没有块采样。只拼字符串,不连库。 标识符交给 SQLAlchemy 各方言的 identifier_preparer:需要时才加引号(memberTags 加、visits 不加), MySQL 用反引号。Oracle 尤其不能一律加双引号:探查拿到的表名是 SQLAlchemy 规整过的小写,库里其实是 - 大写,"visits" 加了引号就成了另一张不存在的表;不加引号 Oracle 自己转大写才对得上。 + 大写,"visits" 加了引号就成了另一张不存在的表;不加引号 Oracle 自己转大写才对得上。反过来,库里真是小写的 + 对象(建表时写了 "visits")探查时记了 quote_name,这种要强制加引号、不转大写(table、column、stats_sql)。 """ def __init__(self, kind: str | None) -> None: @@ -262,13 +328,64 @@ class SqlDialect: self._prep = self._dialect.identifier_preparer self.oracle = self.kind == "oracle" - def quote(self, name: str) -> str: - return self._prep.quote(name) + def quote(self, name: str, *, force: bool = False) -> str: + """标识符。force:探查记了 quote_name 的名字,原样加引号(Oracle 小写带引号的对象)。""" + return self._prep.quote_identifier(name) if force else self._prep.quote(name) def table(self, meta: dict[str, Any] | None, name: str) -> str: """写进 FROM 的表名:探查时指定了 schema 的带上它(Oracle 只读账号名下没有对象,全靠 schema 前缀)。""" schema = (meta or {}).get("schema") - return f"{self._prep.quote_schema(schema)}.{self.quote(name)}" if schema else self.quote(name) + quoted = self.quote(name, force=(meta or {}).get("quote_name") is True) + return f"{self._prep.quote_schema(schema)}.{quoted}" if schema else quoted + + def column(self, meta: dict[str, Any] | None, name: str) -> str: + """写进 SQL 的列名:表结构里这一列记了 quote_name 的强制加引号。""" + cols = (meta or {}).get("columns") or [] + entry = next((c for c in cols if isinstance(c, dict) and c.get("name") == name), None) + return self.quote(name, force=bool(entry and entry.get("quote_name") is True)) + + def date_literal(self, value: Any, *, text: bool = False) -> str: + """日期字面量(取值观察的时间窗口、日期列的有效区间用)。PostgreSQL、MySQL、Oracle 都认 DATE 'YYYY-MM-DD'; + SQLite 没有日期类型,日期存成文字,写字符串。text:列是文字存的日期(VARCHAR),按字符串比——PostgreSQL 上 + varchar 和 date 比较会直接报错。只收真实存在的日期(date 或 YYYY-MM-DD),别的抛 ValueError:拼进 SQL 的 + 只能是我们自己算出来的日子。""" + if isinstance(value, datetime): + value = value.date() + if isinstance(value, _date): + day = value.isoformat() + elif isinstance(value, str) and _ISO_DAY.fullmatch(value): + try: + day = _date.fromisoformat(value).isoformat() + except ValueError: + raise ValueError(f"不是有效的日期:{value}") from None + else: + raise ValueError(f"不是有效的日期:{value!r}") + if text or self.kind == "sqlite": + return f"'{day}'" + return f"DATE '{day}'" + + def day_trunc(self, column: str) -> str: + """时间列截到天(按日行数用)。PostgreSQL CAST(x AS DATE);MySQL DATE(x);Oracle 必须用 TRUNC(x):Oracle 的 + DATE 带时分秒,CAST(ts AS DATE) 截不掉;SQLite 的日期是文字,取前 10 个字符。""" + if self.kind in ("postgres", "postgresql"): + return f"CAST({column} AS DATE)" + if self.kind in ("mysql", "mariadb"): + return f"DATE({column})" + if self.oracle: + return f"TRUNC({column})" + return f"substr({column}, 1, 10)" + + def sample_from(self, table: str, percent: float) -> str | None: + """块采样的 FROM:PostgreSQL TABLESAMPLE SYSTEM (p)、Oracle SAMPLE BLOCK (p),按数据块随机取约 p% 的行; + MySQL、SQLite 没有块采样,返回 None(调用方退回「按存储顺序的前 N 行」,并照实写口径)。""" + p = float(percent) + if not (0 < p < 100): + raise ValueError(f"采样比例要在 0 到 100 之间:{percent}") + if self.kind in ("postgres", "postgresql"): + return f"{table} TABLESAMPLE SYSTEM ({p:g})" + if self.oracle: + return f"{table} SAMPLE BLOCK ({p:g})" + return None def limit(self, sql: str, n: int) -> str: """只取前 n 行。Oracle 11g 没有 LIMIT 也没有 FETCH FIRST,一律套一层 ROWNUM。""" @@ -304,7 +421,8 @@ class SqlDialect: if self.oracle: denorm = self._dialect.denormalize_name owner = sql_literal(denorm(schema), "text") if schema else "SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA')" - table = sql_literal(denorm(name), "text") + # 小写带引号的对象(quote_name)在数据字典里就是小写,不能转成大写去找 + table = sql_literal(name if (meta or {}).get("quote_name") is True else denorm(name), "text") if not (owner and table): return None return f"SELECT NUM_ROWS FROM ALL_TABLES WHERE OWNER = {owner} AND TABLE_NAME = {table}" @@ -335,8 +453,49 @@ class SqlDialect: f"WHERE {column} IS NOT NULL GROUP BY {column} ORDER BY cnt DESC, {column}") return self.limit(sql, n) - def min_max_sql(self, table: str, columns: list[str]) -> str: - return f"SELECT {', '.join(f'MIN({c}), MAX({c})' for c in columns)} FROM {table}" + def min_max_sql(self, table: str, columns: list[tuple[str, bool]]) -> str: + """日期列的最小值、最大值和无效日期的个数:每列三个值(MIN、MAX、区间外的个数)。columns 是 (写进 SQL 的列, + 是不是文字存的日期)。只看有效区间(VALID_DATE_FROM ≤ x < VALID_DATE_BEFORE)里的值,见 VALID_DATE_FROM。 + 整表统计,只对小表用。""" + parts = [] + for c, text in columns: + lo = self.date_literal(VALID_DATE_FROM, text=text) + hi = self.date_literal(VALID_DATE_BEFORE, text=text) + valid = f"{c} >= {lo} AND {c} < {hi}" + parts.append(f"MIN(CASE WHEN {valid} THEN {c} END), MAX(CASE WHEN {valid} THEN {c} END), " + f"SUM(CASE WHEN {c} < {lo} OR {c} >= {hi} THEN 1 ELSE 0 END)") + return f"SELECT {', '.join(parts)} FROM {table}" + + # ---- 取值观察(下一轮第 3 波用):窗口、按日行数、块采样。这里先把各方言的写法定下来,金样和容器集成测试都覆盖 + + def _window(self, column: str, since: Any, text: bool) -> str: + """时间窗口的条件:since 起(含),并且早于占位日期 9999-12-31——PostgreSQL 的 infinity、Oracle 的 9999-12-31 + 都大于任何一天,不截掉上端就会在「最近 35 天」里多出一天。""" + return (f"{column} >= {self.date_literal(since, text=text)} " + f"AND {column} < {self.date_literal(VALID_DATE_BEFORE, text=text)}") + + def day_volume_sql(self, table: str, column: str, since: Any, *, text: bool = False) -> str: + """最近若干天按天计数:since 起(含)每天几行。只在日期列打头有索引时对大表发(走索引的范围扫描)。""" + day = self.day_trunc(column) + return (f"SELECT {day} AS d, COUNT(*) AS n FROM {table} " + f"WHERE {self._window(column, since, text)} GROUP BY {day} ORDER BY d") + + def window_value_counts_sql(self, table: str, column: str, date_column: str, since: Any, n: int, *, + text: bool = False) -> str: + """时间窗口里的取值分布:业务日期 since 起(含)的行里,出现最多的 n 个取值和各自的行数。""" + sql = (f"SELECT {column}, COUNT(*) AS cnt FROM {table} " + f"WHERE {self._window(date_column, since, text)} AND {column} IS NOT NULL " + f"GROUP BY {column} ORDER BY cnt DESC, {column}") + return self.limit(sql, n) + + def sample_value_counts_sql(self, table: str, column: str, percent: float, n: int) -> str | None: + """块采样里的取值分布;没有块采样的方言返回 None。""" + source = self.sample_from(table, percent) + if source is None: + return None + sql = (f"SELECT {column}, COUNT(*) AS cnt FROM {source} WHERE {column} IS NOT NULL " + f"GROUP BY {column} ORDER BY cnt DESC, {column}") + return self.limit(sql, n) # ========================================================================== @@ -581,8 +740,10 @@ async def estimate_size(run: Callable[..., Awaitable[QueryResult]], dialect: Sql """先读统计信息;读不到(SQLite、没分析过的表)再数到 max_scan_rows + 1 行为止。 统计信息为 0 或负数(PostgreSQL 没分析过的表是 -1,MySQL 的估算值可能是 0)当作不知道:一张其实很大的表 - 要是被当成空表,后面就会对它做整表统计。返回 (行数, 途中没查成的几条)。这张表的查询次数、时长用完(或者任务止损) -时抛 _Stop。 + 要是被当成空表,后面就会对它做整表统计。统计信息说的行数不超过 max_scan_rows 时也再数一次(多一条有上限的 + 计数):统计信息可能是很久以前的(表后来涨到了几千万行,ANALYZE 没再跑过),照它把大表当小表,就会对它做 + 整表的 COUNT(DISTINCT)、MIN/MAX。数过的结果是准数(count),数到上限还没数完就按大表算(at_least)。 + 返回 (行数, 途中没查成的几条)。这张表的查询次数、时长用完(或者任务止损)时抛 _Stop。 """ skips: list[_Skip] = [] sql = dialect.stats_sql(meta, name) @@ -592,7 +753,9 @@ async def estimate_size(run: Callable[..., Awaitable[QueryResult]], dialect: Sql except _Skip as e: skips.append(e) else: - if rows is not None and rows > 0: + if rows is not None and rows > max(max_scan_rows, 0): + return TableSize(rows, "stats"), skips + if rows is not None and rows > 0 and max_scan_rows <= 0: return TableSize(rows, "stats"), skips if max_scan_rows <= 0: return TableSize(None, "unknown"), skips @@ -752,6 +915,8 @@ class _CodesOutcome: rows: int note: str complete: bool = False + #: 列定义(ENUM)列出的取值:这次没出现的也写进码值(含义待填),码值是「可能的取值」 + declared: list[str] = field(default_factory=list) @dataclass @@ -901,6 +1066,11 @@ def _column(meta: dict[str, Any], name: str) -> str | None: if isinstance(c, dict) and str(c.get("name", "")).lower() == lowered), None) +def _column_meta(meta: dict[str, Any], name: str) -> dict[str, Any] | None: + """表结构里这一列(类型、取值列表、quote_name);name 是表结构里的写法(_column 的返回值)。""" + return next((c for c in meta.get("columns") or [] if isinstance(c, dict) and c.get("name") == name), None) + + def _constraint(meta: dict[str, Any], column: str) -> str | None: """这一列单独是主键或某个唯一约束时返回「主键」/「唯一约束」,否则 None。""" wanted = [column.lower()] @@ -950,7 +1120,7 @@ async def _unique(cx: _TableCx, table: str, meta: dict[str, Any], column: str, if not size.within(cx.settings.max_scan_rows): return None, "too_large" d = cx.dialect - res = await cx.run(d.unique_check_sql(d.table(meta, table), d.quote(column))) + res = await cx.run(d.unique_check_sql(d.table(meta, table), d.column(meta, column))) row = res.rows[0] if res.rows else [None, None] total, distinct = _as_int(row[0]), _as_int(row[1]) if total is None or distinct is None: @@ -969,20 +1139,38 @@ def _fk_note(child: tuple[bool | None, str]) -> str: return f"{catalog.PROFILE_NOTE_PREFIX}外键约束已保证引用完整,未抽样核对覆盖率;子表一侧{basis}:{shape}" +#: MySQL 上比较文字按列的排序规则:常见的排序规则不分大小写(_ci),PAD SPACE 的还忽略尾随空格。覆盖率、唯一性、 +#: 码值的分组都按它算,写进说明,人才知道「A01」和「a01」在这里算同一个值 +MYSQL_COLLATION_NOTE = "MySQL 按列的排序规则比较文字(常见的排序规则不分大小写),覆盖率和唯一性按它计算" +MYSQL_CODES_COLLATION_NOTE = "MySQL 按列的排序规则分组(常见的排序规则不分大小写),大小写不同的写法可能并成一个取值" + + +def _collation_note(dialect: SqlDialect, family: str | None, note: str) -> str | None: + """MySQL、MariaDB 上文字列(或类型认不出的列)的排序规则说明;别的方言、数字列没有。""" + return note if dialect.kind in ("mysql", "mariadb") and family in ("text", None) else None + + def _relation_note(*, sample: int, matched: int, coverage: float, scan_cap: int | None, skipped_values: int, parent: tuple[bool | None, str], child: tuple[bool | None, str] | None, - holds: bool) -> str: - """关系的剖析备注:抽样规模、覆盖率、父键和子键唯一的依据。人据此判断这条结论有多可靠。 - 不写日期:什么时候剖析的看这一项的 updated_at(界面按本地时间显示)。""" + holds: bool, mismatched: int = 0, collation: str | None = None) -> str: + """关系的剖析备注:取了多少个键值、看了多少行、覆盖率、父键和子键唯一的依据。人据此判断这条结论有多可靠。 + 不写日期:什么时候剖析的看这一项的 updated_at(界面按本地时间显示)。 + + 不说「抽样」:DISTINCT … LIMIT 取的是数据库先给出的那几个值,大表上是按存储顺序的前若干行里的值,不是随机抽的 + (多半是最老的数据)。mismatched:和被指向列类型不同、没拼进核对的键值个数(key_literals)。""" parts: list[str] = [] low = coverage < catalog.PROFILE_VERIFY_COVERAGE if low: - parts.append(f"抽样覆盖率 {_percent(coverage)},可能不是这条关系") - scope = f"(只看前 {scan_cap} 行)" if scan_cap else "" - sampled = f"子表抽样 {sample} 个不同键值{scope},父表对上 {matched} 个" + parts.append(f"覆盖率 {_percent(coverage)},可能不是这条关系") + scope = f"(按存储顺序只看前 {scan_cap} 行)" if scan_cap else "" + sampled = f"子表取了 {sample} 个不同键值{scope},父表对上 {matched} 个" parts.append(sampled if low else f"{sampled},覆盖率 {_percent(coverage)}") if skipped_values: parts.append(f"另有 {skipped_values} 个键值的类型不支持核对,未计入") + if mismatched: + parts.append(f"另有 {mismatched} 个键值和被指向列的类型不同(两端类型不同),未计入") + if collation: + parts.append(collation) unique, how = parent if unique is True: parts.append(f"被指向列是{how}" if how in ("主键", "唯一约束") else "被指向列经计数核对唯一") @@ -1029,18 +1217,25 @@ async def _check_relation(cx: _TableCx, table: str, rel: dict[str, Any], out: _T size = await cx.size_with_data(table, result) scan_cap = cx.scan_cap(size) child_table, parent_table = d.table(meta, table), d.table(pmeta, parent) - res = await cx.run(d.distinct_sample_sql(child_table, d.quote(col), s.sample_size, scan_cap=scan_cap), + res = await cx.run(d.distinct_sample_sql(child_table, d.column(meta, col), s.sample_size, scan_cap=scan_cap), max_rows=s.sample_size) kind = (res.column_types or {}).get(res.columns[0]) if res.columns else None values = [row[0] for row in res.rows if row] - literals = list(dict.fromkeys(lit for v in values if (lit := sql_literal(v, kind)) is not None)) + child_col, parent_col = _column_meta(meta, col), _column_meta(pmeta, pcol) + parent_family = catalog.column_family(parent_col) + literals, mismatched = key_literals(values, kind, parent=parent_family, + child_type=(child_col or {}).get("type")) if not values: raise _Skip("no_data", f"{table}.{col} 没有非空值,无法核对") + if not literals and mismatched: + their = {"number": "数字", "text": "文字", "time": "日期时间"}.get(parent_family or "", "其他类型") + raise _Skip("unsupported", f"两端类型不同:{parent}.{pcol} 是{their},{table}.{col} 取到的值对不上这种类型," + "无法核对") if not literals: raise _Skip("unsupported", f"{table}.{col} 的取值类型不支持核对(只核对整数和文字)") matched = 0 for start in range(0, len(literals), IN_CHUNK): - sql = d.match_count_sql(parent_table, d.quote(pcol), literals[start:start + IN_CHUNK]) + sql = d.match_count_sql(parent_table, d.column(pmeta, pcol), literals[start:start + IN_CHUNK]) matched += _as_int(_first(await cx.run(sql))) or 0 matched = min(matched, len(literals)) coverage = round(matched / len(literals), 4) @@ -1051,9 +1246,11 @@ async def _check_relation(cx: _TableCx, table: str, rel: dict[str, Any], out: _T if child_unique is not None: cardinality = "one_to_one" if child_unique[0] is True else "many_to_one" holds = coverage >= catalog.PROFILE_VERIFY_COVERAGE and cardinality is not None + unsafe = len(values) - mismatched - len(literals) note = _relation_note(sample=len(literals), matched=matched, coverage=coverage, scan_cap=scan_cap, - skipped_values=len(values) - len(literals), parent=parent_unique, child=child_unique, - holds=holds) + skipped_values=max(unsafe, 0), parent=parent_unique, child=child_unique, holds=holds, + mismatched=mismatched, + collation=_collation_note(d, parent_family, MYSQL_COLLATION_NOTE)) checked = child_unique is not None and child_unique[0] is not None out.relations.append(_RelationOutcome(rid=str(rel["id"]), target=_relation_target(rel), coverage=coverage, cardinality=cardinality, holds=holds, sample=len(literals), @@ -1076,17 +1273,27 @@ _NOT_CODE_TAILS = frozenset({"id", "no", "num", "number", "name", "desc", "descr "at", "date", "time", "on", "amount", "price", "count", "qty", "rate", "url"}) #: 像日期的取值:2026-07-01、2026/07/01,后面可以跟时间 _DATE_TEXT = re.compile(r"^\d{4}[-/]\d{2}[-/]\d{2}") +#: 日期列最小值、最大值不计入的值(见 VALID_DATE_FROM)在说明里的说法 +INVALID_DATES = "零日期 0000-00-00、0001-01-01、9999-12-31 这类占位日期或无穷大" -def _code_type_ok(type_name: str | None) -> bool: - """整数,或者短文字(定长不超过 MAX_CODE_LEN;SQLite 的 TEXT 不带长度,也算)。""" - t = (type_name or "").upper() +def _is_enum(type_name: str | None, enum_values: Any = None) -> bool: + """取值列表有限的列:MySQL 的 ENUM / SET(类型写法以它们开头),或者探查记了取值列表(PostgreSQL 的枚举)。""" + return bool(enum_values) or catalog._type_words(type_name).startswith(("ENUM", "SET")) + + +def _code_type_ok(type_name: str | None, enum_values: Any = None) -> bool: + """ENUM / SET(取值列表有限,直接算码值候选)、整数,或者短文字(定长不超过 MAX_CODE_LEN;SQLite 的 TEXT 不带 + 长度,也算)。类型是按方言编译的完整写法:VARCHAR2(20 CHAR) 的长度按 20 算。""" + if _is_enum(type_name, enum_values): + return True + t = catalog._type_words(type_name) if not t or any(k in t for k in ("CLOB", "BLOB", "BINARY", "JSON", "DATE", "TIME", "INTERVAL", "REAL", "FLOAT", "DOUBLE", "BOOL")): return False if "INT" in t: return True - size = re.search(r"\(\s*(\d+)\s*(?:,\s*(\d+)\s*)?\)", t) + size = re.search(r"\(\s*(\d+)(?:\s+(?:CHAR|BYTE))?\s*(?:,\s*(\d+)\s*)?\)", t) if any(k in t for k in ("NUMBER", "NUMERIC", "DECIMAL")): return size is None or not size.group(2) or int(size.group(2)) == 0 if any(k in t for k in ("CHAR", "TEXT", "STRING")): @@ -1100,9 +1307,9 @@ def _looks_like_code(name: str) -> bool: def _code_columns(meta: dict[str, Any], notes: dict[str, Any]) -> list[str]: - """要取值分布的列:目录里度量类型记为状态的、剖析以前提过码值候选的(再核一次)、或者类型和命名像状态、 - 类型、渠道的整数或短文字列。主键、唯一约束列、关系里的键列不算;码值人工确认或驳回过的不再取。 - 被遮罩的列也列出来,由检查记为「被遮罩」跳过——要让人看得见为什么它没有码值。""" + """要取值分布的列:目录里度量类型记为状态的、剖析以前提过码值候选的(再核一次)、ENUM / SET 列(取值列表有限, + 不看名字)、或者类型和命名像状态、类型、渠道的整数或短文字列。主键、唯一约束列、关系里的键列不算;码值人工确认 + 或驳回过的不再取。被遮罩的列也列出来,由检查记为「被遮罩」跳过——要让人看得见为什么它没有码值。""" pk = [c.lower() for c in meta.get("primary_key") or []] unique = {str(g[0]).lower() for g in meta.get("unique") or [] if isinstance(g, list) and len(g) == 1} keys = {str(c).lower() for r in notes.get("relations") or [] if isinstance(r, dict) for c in r.get("columns") or []} @@ -1117,23 +1324,28 @@ def _code_columns(meta: dict[str, Any], notes: dict[str, Any]) -> list[str]: status_measure = (isinstance(measure, dict) and measure.get("value") == "status" and measure.get("status") != "rejected") profiled = isinstance(codes, dict) and codes.get("source") == "profile" - if not (status_measure or profiled or _looks_like_code(name)): + enum = _is_enum(col.get("type"), col.get("enum_values")) + if not (status_measure or profiled or enum or _looks_like_code(name)): continue lowered = name.lower() - if (pk == [lowered]) or lowered in unique or lowered in keys or not _code_type_ok(col.get("type")): + if (pk == [lowered]) or lowered in unique or lowered in keys or not _code_type_ok(col.get("type"), + col.get("enum_values")): continue out.append(name) return out -def _code_key(value: Any) -> str | None: - """取值分布里的一个取值 → 码值(文字)。只认整数和不太长的文字。""" +def _code_key(value: Any, *, fixed: bool = False) -> str | None: + """取值分布里的一个取值 → 码值(文字)。只认整数和不太长的文字。fixed:定长文字列(CHAR(n)),去掉补齐的尾随 + 空格(PG、Oracle 取回的是 'A ',WHERE status = 'A' 照样对得上,码值写 'A')。""" if value is None or isinstance(value, bool): return None if isinstance(value, int): return str(value) if isinstance(value, float): return str(int(value)) if value.is_integer() else None + if isinstance(value, str) and fixed: + value = value.rstrip(" ") if isinstance(value, str) and value.strip() and len(value) <= MAX_CODE_LEN and not _CONTROL.search(value): return value return None @@ -1151,29 +1363,48 @@ async def _check_codes(cx: _TableCx, table: str, column: str, out: _TableOutcome if cx.is_masked(column): raise _Skip("masked", f"{table}.{column} 在数据源设置中被遮罩,不取值", carry=False) d, meta = cx.dialect, cx.tables[table] + col_meta = _column_meta(meta, column) or {} size = await cx.size_with_data(table, result) scan_cap = cx.scan_cap(size) - res = await cx.run(d.value_counts_sql(d.table(meta, table), d.quote(column), CODES_MAX + 1, scan_cap=scan_cap), + res = await cx.run(d.value_counts_sql(d.table(meta, table), d.column(meta, column), CODES_MAX + 1, + scan_cap=scan_cap), max_rows=CODES_MAX + 1) if not res.rows: raise _Skip("no_data", f"{table}.{column} 没有非空值,无法取值分布") if len(res.rows) > CODES_MAX: raise _Skip("high_cardinality", f"{table}.{column} 的取值超过 {CODES_MAX} 个,不作为码值候选", carry=False) - values: list[tuple[str, int]] = [] + fixed = is_fixed_char(col_meta.get("type")) + enum = _is_enum(col_meta.get("type"), col_meta.get("enum_values")) + counts: dict[str, int] = {} for row in res.rows: - key = _code_key(row[0]) + key = _code_key(row[0], fixed=fixed) if key is None: raise _Skip("unsupported", f"{table}.{column} 的取值不是整数或短文字,不作为码值候选", carry=False) - values.append((key, _as_int(row[1]) or 0)) - if len(values) >= 3 and all(n == 1 for _, n in values): - # 每个取值只出现一次(票种编码 T01…T09):这是一列标识,不是状态码 + counts[key] = counts.get(key, 0) + (_as_int(row[1]) or 0) + values = list(counts.items()) + if len(values) >= 3 and all(n == 1 for _, n in values) and not enum: + # 每个取值只出现一次(票种编码 T01…T09):这是一列标识,不是状态码。ENUM 的取值列表是列定义给的,不按这条判 raise _Skip("high_cardinality", f"{table}.{column} 的取值各不相同,像标识而不是码值,不作为码值候选", carry=False) rows = sum(n for _, n in values) - scope = f"按前 {scan_cap} 行统计" if scan_cap else "统计全表" + # 不说「抽样」:大表看的是按存储顺序的前若干行(多半是最老的数据),不是随机抽的 + scope = f"按存储顺序取前 {scan_cap} 行统计" if scan_cap else "统计全表" spread = "、".join(f"{k}({_share(n, rows)})" for k, n in values) - note = f"{NOTE_PREFIX}{scope},{rows} 行非空值共 {len(values)} 个取值:{spread}。" - out.codes.append(_CodesOutcome(column=column, values=values, rows=rows, note=note, complete=scan_cap is None)) + extra: list[str] = [] + # ENUM 的取值列表(SET 的是成员,取值是成员的组合,不算):这次没出现的也是可能的取值 + is_set = catalog._type_words(col_meta.get("type")).startswith("SET") + declared = [] if is_set else [str(v) for v in col_meta.get("enum_values") or []] + unseen = [v for v in declared if v not in counts] + if unseen: + extra.append(f"列定义里另有 {len(unseen)} 个取值这次没有出现:{'、'.join(unseen[:CODES_MAX])}") + family = "text" if enum else catalog.column_family(col_meta) + collation = _collation_note(d, family, MYSQL_CODES_COLLATION_NOTE) + if collation: + extra.append(collation) + tail = "".join(f";{x}" for x in extra) + note = f"{NOTE_PREFIX}{scope},{rows} 行非空值共 {len(values)} 个取值:{spread}{tail}。" + out.codes.append(_CodesOutcome(column=column, values=values, rows=rows, note=note, complete=scan_cap is None, + declared=unseen)) def _is_date_column(col: dict[str, Any]) -> bool: @@ -1190,20 +1421,31 @@ async def _check_dates(cx: _TableCx, table: str, columns: list[str], total: int, if not size.within(cx.settings.max_scan_rows): raise _Skip("too_large", f"{table} 的行数超过整表统计的行数上限({cx.settings.max_scan_rows} 行)," "不取日期列的最小值和最大值") - res = await cx.run(d.min_max_sql(d.table(meta, table), [d.quote(c) for c in columns])) + # 文字存的日期(SQLite 的 TEXT、业务库里的 VARCHAR 日期)按字符串和有效区间比;日期类型的按日期字面量比 + texts = [catalog.column_family(_column_meta(meta, c)) != "time" for c in columns] + res = await cx.run(d.min_max_sql(d.table(meta, table), [(d.column(meta, c), t) for c, t in zip(columns, texts)])) row = res.rows[0] if res.rows else [] + invalid: dict[str, int] = {} for i, column in enumerate(columns): - low, high = (row[2 * i], row[2 * i + 1]) if len(row) >= 2 * i + 2 else (None, None) + low, high, bad = (row[3 * i], row[3 * i + 1], row[3 * i + 2]) if len(row) >= 3 * i + 3 else (None, None, 0) + invalid[column] = _as_int(bad) or 0 if low is None or high is None or not (_DATE_TEXT.match(str(low)) and _DATE_TEXT.match(str(high))): continue - result.date_ranges.append({"column": column, "min": str(low), "max": str(high)}) + found: dict[str, Any] = {"column": column, "min": str(low), "max": str(high)} + if invalid[column]: + found["invalid"] = invalid[column] + result.date_ranges.append(found) if total != 1: return if not result.date_ranges: + if invalid.get(columns[0]): + raise _Skip("no_data", f"{table}.{columns[0]} 只有 {invalid[columns[0]]} 个无效日期({INVALID_DATES})," + "没有可识别的日期值") raise _Skip("no_data", f"{table}.{columns[0]} 没有可识别的日期值") found = result.date_ranges[0] + dropped = (f"另有 {found['invalid']} 个无效日期({INVALID_DATES})未计入," if found.get("invalid") else "") note = (f"{NOTE_PREFIX}表里只有这一个日期类列,取值从 {str(found['min'])[:19]} 到 " - f"{str(found['max'])[:19]},可作为业务日期的参考。") + f"{str(found['max'])[:19]},{dropped}可作为业务日期的参考。") out.business_date = _DateOutcome(column=found["column"], low=found["min"], high=found["max"], note=note) @@ -1241,6 +1483,8 @@ def _codes_value(c: _CodesOutcome, item: dict[str, Any] | None) -> dict[str, str 沿用它的含义;它写了、数据里没出现的取值也留着——码值是「可能的取值」,没出现不等于不存在。""" old = item.get("value") if isinstance(item, dict) and isinstance(item.get("value"), dict) else {} value = {key: str(old.get(key) or "") for key, _ in c.values} + for key in c.declared: + value.setdefault(key, str(old.get(key) or "")) for key, meaning in old.items(): if key not in value and isinstance(meaning, str) and meaning.strip(): value[key] = meaning @@ -1744,8 +1988,8 @@ def ensure_enabled(row: Any, settings: ProfileSettings | None = None) -> Profile #: 有效行过滤核对的说法(写进这一项的 note;前面是模型给的依据) FILTER_CHECKED = "剖析核对:排除 {share} 的行(共 {rows} 行)" -FILTER_SAMPLED = "剖析核对:按前 {rows} 行统计,排除 {share} 的行" -FILTER_SAMPLED_EDGE = "剖析核对:按前 {rows} 行统计,{what},请人工核对" +FILTER_SAMPLED = "剖析核对:按存储顺序取前 {rows} 行统计,排除 {share} 的行" +FILTER_SAMPLED_EDGE = "剖析核对:按存储顺序取前 {rows} 行统计,{what},请人工核对" FILTER_UNCHECKED = "没有核对排除比例({why})" FILTER_NOT_ENABLED = "数据源没有开数据剖析" diff --git a/backend/app/data/introspect.py b/backend/app/data/introspect.py index ea8875be..30fbfda2 100644 --- a/backend/app/data/introspect.py +++ b/backend/app/data/introspect.py @@ -165,6 +165,74 @@ def _unique_groups(constraints: list[dict[str, Any]], indexes: list[dict[str, An return groups +def type_text(col_type: Any, dialect: Any) -> str: + """反射到的列类型 → 存进表结构的写法:按这个数据源的方言编译的完整写法(VARCHAR2(20 CHAR)、NUMBER(10, 0)、 + ENUM('paid','refund')、TIMESTAMP WITH TIME ZONE),编译不了的退回 str()。 + + 以前存的是 str(类型),即 SQLAlchemy 的通用写法:Oracle 的 VARCHAR2、NUMBER 丢了长度和精度,MySQL 的 ENUM 丢了 + 取值列表,PostgreSQL 的 TIMESTAMP 丢了时区。类型写不全,表结构前后比对就看不出类型变了,ENUM 状态列也进不了 + 码值候选。这一改会让新探查的表结构快照哈希变化(提示词里的类型写法变长),旧运行封存的快照不受影响。""" + if col_type is None: + return "" + try: + compiled = str(col_type.compile(dialect=dialect)) + except Exception: # noqa: BLE001 - 编译不了(NullType 等)就用通用写法,不让一列拖垮整次探查 + return str(col_type) + if getattr(dialect, "name", "") == "oracle" and compiled.startswith("VARCHAR("): + # SQLAlchemy 把 Oracle 的 VARCHAR2 反射成通用的 VARCHAR,按 Oracle 方言编译出来是 VARCHAR(10 CHAR)(容器集成 + # 测试实测)。Oracle 的数据字典里只有 VARCHAR2(建表写 VARCHAR 也存成 VARCHAR2),照数据字典写 + compiled = "VARCHAR2" + compiled[len("VARCHAR"):] + return compiled + + +def _enum_values(col_type: Any) -> list[str] | None: + """ENUM(MySQL、PostgreSQL 的枚举,都是 sqltypes.Enum 的子类)的取值列表,MySQL SET 的成员列表;别的类型 None。""" + from sqlalchemy.dialects.mysql import SET + from sqlalchemy.types import Enum + + if isinstance(col_type, Enum): + values = list(col_type.enums or []) + elif isinstance(col_type, SET): + values = list(col_type.values or []) + else: + return None + return [str(v) for v in values] + + +def _forced_quote(name: Any) -> bool: + """名字必须原样加引号:Oracle 反射回来的全小写对象(建表时写了 "visits")是 quoted_name(quote=True)。 + 它存进 JSON 就成了普通字符串,剖析拼 SQL 时不加引号,Oracle 把它转成大写 VISITS,查的是另一个不存在的对象。""" + return getattr(name, "quote", None) is True + + +def column_entry(col: Mapping[str, Any], dialect: Any) -> dict[str, Any]: + """反射到的一列 → 表结构里的一列:名字、按方言编译的类型、可空、注释;ENUM / SET 另记取值列表(enum_values), + 要强制加引号的名字另记 quote_name。只在有的时候写这两个键:其余列和以前一字不差。""" + entry: dict[str, Any] = { + "name": str(col["name"]), + "type": type_text(col.get("type"), dialect), + "nullable": bool(col.get("nullable", True)), + "comment": col.get("comment") or None, + } + values = _enum_values(col.get("type")) + if values: + entry["enum_values"] = values + if _forced_quote(col.get("name")): + entry["quote_name"] = True + return entry + + +def index_first_columns(indexes: list[dict[str, Any]], pk: list[str]) -> list[str]: + """哪些列打头有索引:各索引(含主键)的第一列,去重、按名字排序。取值观察看大表的最新业务日期、按日行数时, + 只在业务日期列打头有索引时才发窗口查询(按它走索引,不扫全表)。表达式索引(列名是 None)不算。""" + first = {str(pk[0])} if pk else set() + for ix in indexes: + cols = ix.get("column_names") or [] + if cols and cols[0] is not None: + first.add(str(cols[0])) + return sorted(first) + + async def introspect(source: Any, *, schema: str | None = None) -> dict[str, Any]: """探查整个库的结构,结果直接存进 DataSource.schema_cache。 @@ -266,24 +334,20 @@ async def introspect(source: Any, *, schema: str | None = None) -> dict[str, Any # 只读账号名下没有这些对象,全靠跨 schema 授权 "qualified": f"{target_schema}.{name}" if target_schema else name, "schema": target_schema, - "columns": [ - { - "name": c["name"], - "type": str(c.get("type") or ""), - "nullable": bool(c.get("nullable", True)), - "comment": c.get("comment") or None, - } - for c in columns - ], + "columns": [column_entry(c, sync_conn.dialect) for c in columns], "primary_key": pk, "comment": comment, "is_view": name in views, } + if _forced_quote(name): + tables[name]["quote_name"] = True if name in fk_ok: tables[name]["foreign_keys"] = _foreign_keys(fk_by.get(key) or [], target_schema, default_schema) if name in uq_ok: tables[name]["unique"] = _unique_groups(uq_by.get(key) or [], (ix_by.get(key) or []) if name in ix_ok else [], pk) + if name in ix_ok: + tables[name]["index_first_columns"] = index_first_columns(ix_by.get(key) or [], pk) payload: dict[str, Any] = {"tables": tables, "truncated": truncated, "total": len(names) + len(views)} if failures and not tables: @@ -471,6 +535,10 @@ def describe_table(source: Any, table: str, *, catalog: Mapping[str, Mapping[str marks.append("非空") if col.get("comment"): marks.append(col["comment"]) + values = col.get("enum_values") + if values and "(" not in str(col.get("type") or ""): + # PostgreSQL 的枚举类型写法只有类型名(order_status):取值列表另外写出来,模型才知道能写哪些值 + marks.append("取值:" + "、".join(str(v) for v in values[:30]) + ("…" if len(values) > 30 else "")) suffix = f" {'、'.join(marks)}" if marks else "" lines.append(f" {col['name']} {col['type']}{suffix}") if catalog: diff --git a/backend/tests/fixtures/dialect/mariadb.sql b/backend/tests/fixtures/dialect/mariadb.sql new file mode 100644 index 00000000..0130c4d9 --- /dev/null +++ b/backend/tests/fixtures/dialect/mariadb.sql @@ -0,0 +1,27 @@ +-- 剖析与取值观察的 SQL 金样:mariadb(tests/test_profile_dialect_golden.py 生成,改动要人审) +-- stats +SELECT TABLE_ROWS FROM information_schema.TABLES WHERE TABLE_SCHEMA = 'sales' AND TABLE_NAME = 'orders'; +-- stats_quoted +SELECT TABLE_ROWS FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'visits'; +-- bounded +SELECT COUNT(*) AS n FROM (SELECT 1 AS one FROM sales.orders LIMIT 100001) s; +-- distinct +SELECT DISTINCT store_code FROM sales.orders WHERE store_code IS NOT NULL LIMIT 2000; +-- distinct_capped +SELECT DISTINCT store_code FROM (SELECT store_code FROM sales.orders WHERE store_code IS NOT NULL LIMIT 100000) s LIMIT 2000; +-- match +SELECT COUNT(DISTINCT code) AS n FROM stores WHERE code IN ('12', '7'); +-- unique +SELECT COUNT(`level`) AS n, COUNT(DISTINCT `level`) AS d FROM `visits`; +-- value_counts +SELECT status, COUNT(*) AS cnt FROM sales.orders WHERE status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status LIMIT 21; +-- value_counts_capped +SELECT status, COUNT(*) AS cnt FROM (SELECT status FROM sales.orders WHERE status IS NOT NULL LIMIT 100000) s WHERE status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status LIMIT 21; +-- min_max +SELECT MIN(CASE WHEN paid_at >= DATE '0001-01-02' AND paid_at < DATE '9999-12-31' THEN paid_at END), MAX(CASE WHEN paid_at >= DATE '0001-01-02' AND paid_at < DATE '9999-12-31' THEN paid_at END), SUM(CASE WHEN paid_at < DATE '0001-01-02' OR paid_at >= DATE '9999-12-31' THEN 1 ELSE 0 END), MIN(CASE WHEN biz_day >= '0001-01-02' AND biz_day < '9999-12-31' THEN biz_day END), MAX(CASE WHEN biz_day >= '0001-01-02' AND biz_day < '9999-12-31' THEN biz_day END), SUM(CASE WHEN biz_day < '0001-01-02' OR biz_day >= '9999-12-31' THEN 1 ELSE 0 END) FROM sales.orders; +-- date_window +SELECT status, COUNT(*) AS cnt FROM sales.orders WHERE paid_at >= DATE '2026-08-27' AND paid_at < DATE '9999-12-31' AND status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status LIMIT 21; +-- day_volume +SELECT DATE(paid_at) AS d, COUNT(*) AS n FROM sales.orders WHERE paid_at >= DATE '2026-08-27' AND paid_at < DATE '9999-12-31' GROUP BY DATE(paid_at) ORDER BY d; +-- sample +-- (这种方言没有这类查询) diff --git a/backend/tests/fixtures/dialect/mysql.sql b/backend/tests/fixtures/dialect/mysql.sql new file mode 100644 index 00000000..1b046e36 --- /dev/null +++ b/backend/tests/fixtures/dialect/mysql.sql @@ -0,0 +1,27 @@ +-- 剖析与取值观察的 SQL 金样:mysql(tests/test_profile_dialect_golden.py 生成,改动要人审) +-- stats +SELECT TABLE_ROWS FROM information_schema.TABLES WHERE TABLE_SCHEMA = 'sales' AND TABLE_NAME = 'orders'; +-- stats_quoted +SELECT TABLE_ROWS FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'visits'; +-- bounded +SELECT COUNT(*) AS n FROM (SELECT 1 AS one FROM sales.orders LIMIT 100001) s; +-- distinct +SELECT DISTINCT store_code FROM sales.orders WHERE store_code IS NOT NULL LIMIT 2000; +-- distinct_capped +SELECT DISTINCT store_code FROM (SELECT store_code FROM sales.orders WHERE store_code IS NOT NULL LIMIT 100000) s LIMIT 2000; +-- match +SELECT COUNT(DISTINCT code) AS n FROM stores WHERE code IN ('12', '7'); +-- unique +SELECT COUNT(`level`) AS n, COUNT(DISTINCT `level`) AS d FROM `visits`; +-- value_counts +SELECT status, COUNT(*) AS cnt FROM sales.orders WHERE status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status LIMIT 21; +-- value_counts_capped +SELECT status, COUNT(*) AS cnt FROM (SELECT status FROM sales.orders WHERE status IS NOT NULL LIMIT 100000) s WHERE status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status LIMIT 21; +-- min_max +SELECT MIN(CASE WHEN paid_at >= DATE '0001-01-02' AND paid_at < DATE '9999-12-31' THEN paid_at END), MAX(CASE WHEN paid_at >= DATE '0001-01-02' AND paid_at < DATE '9999-12-31' THEN paid_at END), SUM(CASE WHEN paid_at < DATE '0001-01-02' OR paid_at >= DATE '9999-12-31' THEN 1 ELSE 0 END), MIN(CASE WHEN biz_day >= '0001-01-02' AND biz_day < '9999-12-31' THEN biz_day END), MAX(CASE WHEN biz_day >= '0001-01-02' AND biz_day < '9999-12-31' THEN biz_day END), SUM(CASE WHEN biz_day < '0001-01-02' OR biz_day >= '9999-12-31' THEN 1 ELSE 0 END) FROM sales.orders; +-- date_window +SELECT status, COUNT(*) AS cnt FROM sales.orders WHERE paid_at >= DATE '2026-08-27' AND paid_at < DATE '9999-12-31' AND status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status LIMIT 21; +-- day_volume +SELECT DATE(paid_at) AS d, COUNT(*) AS n FROM sales.orders WHERE paid_at >= DATE '2026-08-27' AND paid_at < DATE '9999-12-31' GROUP BY DATE(paid_at) ORDER BY d; +-- sample +-- (这种方言没有这类查询) diff --git a/backend/tests/fixtures/dialect/oracle.sql b/backend/tests/fixtures/dialect/oracle.sql new file mode 100644 index 00000000..c338ead9 --- /dev/null +++ b/backend/tests/fixtures/dialect/oracle.sql @@ -0,0 +1,27 @@ +-- 剖析与取值观察的 SQL 金样:oracle(tests/test_profile_dialect_golden.py 生成,改动要人审) +-- stats +SELECT NUM_ROWS FROM ALL_TABLES WHERE OWNER = 'SALES' AND TABLE_NAME = 'ORDERS'; +-- stats_quoted +SELECT NUM_ROWS FROM ALL_TABLES WHERE OWNER = SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA') AND TABLE_NAME = 'visits'; +-- bounded +SELECT COUNT(*) AS n FROM (SELECT * FROM (SELECT 1 AS one FROM sales.orders) WHERE ROWNUM <= 100001) s; +-- distinct +SELECT * FROM (SELECT DISTINCT store_code FROM sales.orders WHERE store_code IS NOT NULL) WHERE ROWNUM <= 2000; +-- distinct_capped +SELECT * FROM (SELECT DISTINCT store_code FROM (SELECT * FROM (SELECT store_code FROM sales.orders WHERE store_code IS NOT NULL) WHERE ROWNUM <= 100000) s) WHERE ROWNUM <= 2000; +-- match +SELECT COUNT(DISTINCT code) AS n FROM stores WHERE code IN ('12', '7'); +-- unique +SELECT COUNT("level") AS n, COUNT(DISTINCT "level") AS d FROM "visits"; +-- value_counts +SELECT * FROM (SELECT status, COUNT(*) AS cnt FROM sales.orders WHERE status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status) WHERE ROWNUM <= 21; +-- value_counts_capped +SELECT * FROM (SELECT status, COUNT(*) AS cnt FROM (SELECT * FROM (SELECT status FROM sales.orders WHERE status IS NOT NULL) WHERE ROWNUM <= 100000) s WHERE status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status) WHERE ROWNUM <= 21; +-- min_max +SELECT MIN(CASE WHEN paid_at >= DATE '0001-01-02' AND paid_at < DATE '9999-12-31' THEN paid_at END), MAX(CASE WHEN paid_at >= DATE '0001-01-02' AND paid_at < DATE '9999-12-31' THEN paid_at END), SUM(CASE WHEN paid_at < DATE '0001-01-02' OR paid_at >= DATE '9999-12-31' THEN 1 ELSE 0 END), MIN(CASE WHEN biz_day >= '0001-01-02' AND biz_day < '9999-12-31' THEN biz_day END), MAX(CASE WHEN biz_day >= '0001-01-02' AND biz_day < '9999-12-31' THEN biz_day END), SUM(CASE WHEN biz_day < '0001-01-02' OR biz_day >= '9999-12-31' THEN 1 ELSE 0 END) FROM sales.orders; +-- date_window +SELECT * FROM (SELECT status, COUNT(*) AS cnt FROM sales.orders WHERE paid_at >= DATE '2026-08-27' AND paid_at < DATE '9999-12-31' AND status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status) WHERE ROWNUM <= 21; +-- day_volume +SELECT TRUNC(paid_at) AS d, COUNT(*) AS n FROM sales.orders WHERE paid_at >= DATE '2026-08-27' AND paid_at < DATE '9999-12-31' GROUP BY TRUNC(paid_at) ORDER BY d; +-- sample +SELECT * FROM (SELECT status, COUNT(*) AS cnt FROM sales.orders SAMPLE BLOCK (1) WHERE status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status) WHERE ROWNUM <= 21; diff --git a/backend/tests/fixtures/dialect/postgresql.sql b/backend/tests/fixtures/dialect/postgresql.sql new file mode 100644 index 00000000..1416da24 --- /dev/null +++ b/backend/tests/fixtures/dialect/postgresql.sql @@ -0,0 +1,27 @@ +-- 剖析与取值观察的 SQL 金样:postgresql(tests/test_profile_dialect_golden.py 生成,改动要人审) +-- stats +SELECT c.reltuples FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = 'sales' AND c.relname = 'orders'; +-- stats_quoted +SELECT c.reltuples FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = current_schema() AND c.relname = 'visits'; +-- bounded +SELECT COUNT(*) AS n FROM (SELECT 1 AS one FROM sales.orders LIMIT 100001) s; +-- distinct +SELECT DISTINCT store_code FROM sales.orders WHERE store_code IS NOT NULL LIMIT 2000; +-- distinct_capped +SELECT DISTINCT store_code FROM (SELECT store_code FROM sales.orders WHERE store_code IS NOT NULL LIMIT 100000) s LIMIT 2000; +-- match +SELECT COUNT(DISTINCT code) AS n FROM stores WHERE code IN ('12', '7'); +-- unique +SELECT COUNT("level") AS n, COUNT(DISTINCT "level") AS d FROM "visits"; +-- value_counts +SELECT status, COUNT(*) AS cnt FROM sales.orders WHERE status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status LIMIT 21; +-- value_counts_capped +SELECT status, COUNT(*) AS cnt FROM (SELECT status FROM sales.orders WHERE status IS NOT NULL LIMIT 100000) s WHERE status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status LIMIT 21; +-- min_max +SELECT MIN(CASE WHEN paid_at >= DATE '0001-01-02' AND paid_at < DATE '9999-12-31' THEN paid_at END), MAX(CASE WHEN paid_at >= DATE '0001-01-02' AND paid_at < DATE '9999-12-31' THEN paid_at END), SUM(CASE WHEN paid_at < DATE '0001-01-02' OR paid_at >= DATE '9999-12-31' THEN 1 ELSE 0 END), MIN(CASE WHEN biz_day >= '0001-01-02' AND biz_day < '9999-12-31' THEN biz_day END), MAX(CASE WHEN biz_day >= '0001-01-02' AND biz_day < '9999-12-31' THEN biz_day END), SUM(CASE WHEN biz_day < '0001-01-02' OR biz_day >= '9999-12-31' THEN 1 ELSE 0 END) FROM sales.orders; +-- date_window +SELECT status, COUNT(*) AS cnt FROM sales.orders WHERE paid_at >= DATE '2026-08-27' AND paid_at < DATE '9999-12-31' AND status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status LIMIT 21; +-- day_volume +SELECT CAST(paid_at AS DATE) AS d, COUNT(*) AS n FROM sales.orders WHERE paid_at >= DATE '2026-08-27' AND paid_at < DATE '9999-12-31' GROUP BY CAST(paid_at AS DATE) ORDER BY d; +-- sample +SELECT status, COUNT(*) AS cnt FROM sales.orders TABLESAMPLE SYSTEM (1) WHERE status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status LIMIT 21; diff --git a/backend/tests/fixtures/dialect/sqlite.sql b/backend/tests/fixtures/dialect/sqlite.sql new file mode 100644 index 00000000..8b10405f --- /dev/null +++ b/backend/tests/fixtures/dialect/sqlite.sql @@ -0,0 +1,27 @@ +-- 剖析与取值观察的 SQL 金样:sqlite(tests/test_profile_dialect_golden.py 生成,改动要人审) +-- stats +-- (这种方言没有这类查询) +-- stats_quoted +-- (这种方言没有这类查询) +-- bounded +SELECT COUNT(*) AS n FROM (SELECT 1 AS one FROM sales.orders LIMIT 100001) s; +-- distinct +SELECT DISTINCT store_code FROM sales.orders WHERE store_code IS NOT NULL LIMIT 2000; +-- distinct_capped +SELECT DISTINCT store_code FROM (SELECT store_code FROM sales.orders WHERE store_code IS NOT NULL LIMIT 100000) s LIMIT 2000; +-- match +SELECT COUNT(DISTINCT code) AS n FROM stores WHERE code IN ('12', '7'); +-- unique +SELECT COUNT("level") AS n, COUNT(DISTINCT "level") AS d FROM "visits"; +-- value_counts +SELECT status, COUNT(*) AS cnt FROM sales.orders WHERE status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status LIMIT 21; +-- value_counts_capped +SELECT status, COUNT(*) AS cnt FROM (SELECT status FROM sales.orders WHERE status IS NOT NULL LIMIT 100000) s WHERE status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status LIMIT 21; +-- min_max +SELECT MIN(CASE WHEN paid_at >= '0001-01-02' AND paid_at < '9999-12-31' THEN paid_at END), MAX(CASE WHEN paid_at >= '0001-01-02' AND paid_at < '9999-12-31' THEN paid_at END), SUM(CASE WHEN paid_at < '0001-01-02' OR paid_at >= '9999-12-31' THEN 1 ELSE 0 END), MIN(CASE WHEN biz_day >= '0001-01-02' AND biz_day < '9999-12-31' THEN biz_day END), MAX(CASE WHEN biz_day >= '0001-01-02' AND biz_day < '9999-12-31' THEN biz_day END), SUM(CASE WHEN biz_day < '0001-01-02' OR biz_day >= '9999-12-31' THEN 1 ELSE 0 END) FROM sales.orders; +-- date_window +SELECT status, COUNT(*) AS cnt FROM sales.orders WHERE paid_at >= '2026-08-27' AND paid_at < '9999-12-31' AND status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status LIMIT 21; +-- day_volume +SELECT substr(paid_at, 1, 10) AS d, COUNT(*) AS n FROM sales.orders WHERE paid_at >= '2026-08-27' AND paid_at < '9999-12-31' GROUP BY substr(paid_at, 1, 10) ORDER BY d; +-- sample +-- (这种方言没有这类查询) diff --git a/backend/tests/test_catalog_draft_facts.py b/backend/tests/test_catalog_draft_facts.py index 37f3705b..c65eff02 100644 --- a/backend/tests/test_catalog_draft_facts.py +++ b/backend/tests/test_catalog_draft_facts.py @@ -402,7 +402,7 @@ async def test_a_big_table_is_checked_on_its_first_rows(scenic): found = await catalog_profile.check_valid_filter(source, "visits", source.schema_cache["tables"]["visits"], "status = 1") assert found.sampled and found.rows == 100 - assert found.note.startswith("剖析核对:按前 100 行统计,") + assert found.note.startswith("剖析核对:按存储顺序取前 100 行统计,") async def test_unchecked_filters_are_kept_with_the_reason(scenic, tmp_path): diff --git a/backend/tests/test_catalog_profile.py b/backend/tests/test_catalog_profile.py index fee64c5f..2a04fe76 100644 --- a/backend/tests/test_catalog_profile.py +++ b/backend/tests/test_catalog_profile.py @@ -240,7 +240,7 @@ async def test_profile_verifies_name_relations_and_proposes_codes_and_business_d assert rel["to_table"] == parent assert (rel["source"], rel["status"]) == ("profile", "verified") assert rel["coverage"] == 1.0 and rel["cardinality"] == "many_to_one" - assert "数据剖析" in rel["note"] and "抽样" in rel["note"] + assert "数据剖析" in rel["note"] and "子表取了" in rel["note"] and "抽样" not in rel["note"] # 子表一侧数过 COUNT 和 COUNT(DISTINCT)、确有重复:基数是用数据核实的(A6,一对多关联的检查据此给 error) assert rel["cardinality_checked"] is True # A7:说明里不写日期,时间看这一项的 updated_at(界面按本地时间显示) @@ -318,7 +318,7 @@ async def test_wrong_name_relation_stays_proposed_with_coverage_in_note(client, rel = _relation((await _notes(client, sid, "parking_records"))["notes"], "lot_id") assert rel["status"] == "proposed" and rel["source"] == "profile" assert rel["coverage"] == 0.15 - assert "抽样覆盖率 15%" in rel["note"] and "可能不是这条关系" in rel["note"] and "20 个" in rel["note"] + assert "覆盖率 15%" in rel["note"] and "可能不是这条关系" in rel["note"] and "20 个" in rel["note"] finding = _table(body, "parking_records")["findings"][0] assert finding["status"] == "proposed" and finding["coverage"] == 0.15 # 两个日期类列:不提议业务日期,取值范围只在剖析结果里 diff --git a/backend/tests/test_catalog_profile_units.py b/backend/tests/test_catalog_profile_units.py index 4fa1472f..22933754 100644 --- a/backend/tests/test_catalog_profile_units.py +++ b/backend/tests/test_catalog_profile_units.py @@ -40,7 +40,7 @@ def test_settings_default_off_and_parse_leniently(): ({"table_max_s": 9}, "每张表时长上限"), ({"table_max_s": 601}, "每张表时长上限"), ({"query_timeout_s": 61}, "单条查询时限"), - ({"sample_size": 5}, "抽样键值数"), + ({"sample_size": 5}, "核对键值数"), ({"max_scan_rows": -1}, "整表统计行数上限"), ({"enabled": "true"}, "开启数据剖析"), ({"table_max_queries": True}, "每张表最多查询数"), @@ -143,8 +143,16 @@ def test_profile_sql_passes_readonly_guard_on_every_dialect(kind): d.unique_check_sql(tbl, col), d.value_counts_sql(tbl, col, 21, scan_cap=None), d.value_counts_sql(tbl, col, 21, scan_cap=100_000), - d.min_max_sql(tbl, [col]), + d.min_max_sql(tbl, [(col, False)]), + d.min_max_sql(tbl, [(col, True)]), + d.day_volume_sql(tbl, col, "2026-08-01"), + d.window_value_counts_sql(tbl, col, col, "2026-08-01", 21), ] + sampled = d.sample_value_counts_sql(tbl, col, 1, 21) + if kind in ("postgresql", "oracle"): + sqls.append(sampled) + else: + assert sampled is None # MySQL、SQLite 没有块采样 for sql in sqls: assert guard.check(sql, readonly=True, dialect=kind) == sql, sql diff --git a/backend/tests/test_catalog_semantics.py b/backend/tests/test_catalog_semantics.py index 25eb9aad..5501a4d8 100644 --- a/backend/tests/test_catalog_semantics.py +++ b/backend/tests/test_catalog_semantics.py @@ -365,7 +365,7 @@ async def test_profile_marks_codes_complete_only_when_the_whole_table_was_read(c capped = await make_source(scenic_db, options=_on(max_scan_rows=0, sample_size=10), draft=["visits"]) await _profile(client, capped, ["visits"]) codes = (await _notes(client, capped, "visits"))["notes"]["columns"]["status"]["codes"] - assert "按前 10 行统计" in codes["note"] and "complete" not in codes + assert "按存储顺序取前 10 行统计" in codes["note"] and "complete" not in codes # ========================================================================== diff --git a/backend/tests/test_profile_dialect_golden.py b/backend/tests/test_profile_dialect_golden.py new file mode 100644 index 00000000..2eb0615c --- /dev/null +++ b/backend/tests/test_profile_dialect_golden.py @@ -0,0 +1,178 @@ +"""剖析 SQL 的方言金样(下一轮 03 细案 C2):没有真库时挡住方言写法的回退。 + +三层,各挡各的: + +1. **金样**:每种方言(mysql、mariadb、postgresql、oracle、sqlite)× 每类查询(统计信息、有上限的计数、取不同键值、 + 对键值、唯一性、取值分布、日期最小最大、时间窗口里的取值分布、按日行数、块采样),完整 SQL 钉在 + tests/fixtures/dialect/<方言>.sql,人审过一次;以后拼法有任何改动都会出 diff。要更新金样: + ``UPDATE_DIALECT_GOLDEN=1 pytest tests/test_profile_dialect_golden.py``,再逐行看 git diff。 +2. **sqlglot 逐条解析**:按方言读一遍(mariadb 按 mysql 读),挡住语法写坏了(少括号、关键字拼错)。 +3. **IN 两边类型族一致**:用 sqlglot 的 qualify + annotate_types,配上表结构里的类型,核对 IN 列表的字面量和被指向 + 列是同一类型族(PostgreSQL 上 varchar 父列配整数子列时,字面量必须带引号)。 + +挡不住的照实写:sqlglot 不知道真库的统计信息是不是过期、零日期驱动怎么返回、Oracle 小写对象是否存在、排序规则 +怎么比——这些要靠 scripts/dialect-it.sh 在本机容器里跑真库(不进 CI)。 +""" +from __future__ import annotations + +import os +from pathlib import Path + +import pytest +import sqlglot +from sqlglot import exp +from sqlglot.optimizer.annotate_types import annotate_types +from sqlglot.optimizer.qualify import qualify + +from app.data import catalog, guard +from app.data.catalog_profile import IN_CHUNK, SqlDialect, key_literals + +GOLDEN = Path(__file__).parent / "fixtures" / "dialect" +DIALECTS = ("mysql", "mariadb", "postgresql", "oracle", "sqlite") +#: sqlglot 的方言名 +READ = {"mysql": "mysql", "mariadb": "mysql", "postgresql": "postgres", "oracle": "oracle", "sqlite": "sqlite"} + +#: 夹具表结构:一张带 schema 的订单表、一张门店表(编码是文字),Oracle 的一张小写带引号的表和一个小写带引号的列 +META = { + "orders": {"schema": "sales", "columns": [ + {"name": "id", "type": "INTEGER"}, {"name": "store_code", "type": "INTEGER"}, + {"name": "status", "type": "CHAR(2)"}, {"name": "paid_at", "type": "TIMESTAMP"}, + {"name": "biz_day", "type": "VARCHAR(10)"}, {"name": "level", "type": "INTEGER"}], + "primary_key": ["id"]}, + "stores": {"schema": None, "columns": [{"name": "code", "type": "VARCHAR(10)"}], "primary_key": ["code"]}, + "visits": {"schema": None, "quote_name": True, "columns": [ + {"name": "id", "type": "INTEGER"}, {"name": "level", "type": "INTEGER", "quote_name": True}], + "primary_key": ["id"]}, +} + + +def _queries(kind: str) -> list[tuple[str, str | None]]: + """一种方言的全部剖析、观察查询:(名字, SQL)。没有这种查询(SQLite 没有统计信息、MySQL 没有块采样)的 SQL 为 None。""" + d = SqlDialect(kind) + orders, stores, visits = d.table(META["orders"], "orders"), d.table(META["stores"], "stores"), \ + d.table(META["visits"], "visits") + status, code = d.column(META["orders"], "status"), d.column(META["stores"], "code") + level = d.column(META["visits"], "level") + keys, _ = key_literals([12, 7], "number", parent="text", child_type="INTEGER") + out: list[tuple[str, str | None]] = [ + ("stats", d.stats_sql(META["orders"], "orders")), + ("stats_quoted", d.stats_sql(META["visits"], "visits")), + ("bounded", d.bounded_count_sql(orders, 100_001)), + ("distinct", d.distinct_sample_sql(orders, d.column(META["orders"], "store_code"), 2000, scan_cap=None)), + ("distinct_capped", d.distinct_sample_sql(orders, d.column(META["orders"], "store_code"), 2000, + scan_cap=100_000)), + ("match", d.match_count_sql(stores, code, keys)), + ("unique", d.unique_check_sql(visits, level)), + ("value_counts", d.value_counts_sql(orders, status, 21, scan_cap=None)), + ("value_counts_capped", d.value_counts_sql(orders, status, 21, scan_cap=100_000)), + ("min_max", d.min_max_sql(orders, [(d.column(META["orders"], "paid_at"), False), + (d.column(META["orders"], "biz_day"), True)])), + ("date_window", d.window_value_counts_sql(orders, status, d.column(META["orders"], "paid_at"), + "2026-08-27", 21)), + ("day_volume", d.day_volume_sql(orders, d.column(META["orders"], "paid_at"), "2026-08-27")), + ("sample", d.sample_value_counts_sql(orders, status, 1, 21)), + ] + return out + + +def _render(kind: str) -> str: + lines = [f"-- 剖析与取值观察的 SQL 金样:{kind}(tests/test_profile_dialect_golden.py 生成,改动要人审)"] + for name, sql in _queries(kind): + lines.append(f"-- {name}") + lines.append(f"{sql};" if sql is not None else "-- (这种方言没有这类查询)") + return "\n".join(lines) + "\n" + + +@pytest.mark.parametrize("kind", DIALECTS) +def test_profile_sql_matches_the_golden_file(kind): + path = GOLDEN / f"{kind}.sql" + text = _render(kind) + if os.environ.get("UPDATE_DIALECT_GOLDEN") == "1": + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + assert path.read_text(encoding="utf-8") == text, f"{kind} 的剖析 SQL 和金样不一致:看 diff,确认无误后重新生成金样" + + +@pytest.mark.parametrize("kind", DIALECTS) +def test_every_profile_query_parses_and_passes_the_guard(kind): + for name, sql in _queries(kind): + if sql is None: + continue + parsed = sqlglot.parse(sql, read=READ[kind], error_level=sqlglot.ErrorLevel.RAISE) + assert len(parsed) == 1 and parsed[0] is not None, (kind, name) + assert guard.check(sql, readonly=True, dialect=kind) == sql, (kind, name) + + +@pytest.mark.parametrize("kind", ["postgresql", "oracle", "mysql"]) +def test_observation_queries_use_the_right_day_and_sample_functions(kind): + """按天截断和块采样各家写法不同,解析成语法树再看用的是哪个函数,不只比字符串。""" + sql = dict(_queries(kind)) + day = sqlglot.parse_one(sql["day_volume"], read=READ[kind]) + if kind == "oracle": + assert any(isinstance(n, (exp.DateTrunc, exp.TimestampTrunc, exp.Anonymous)) or n.key == "trunc" + for n in day.walk()), sql["day_volume"] + assert "TRUNC(paid_at)" in sql["day_volume"] + elif kind == "postgresql": + assert any(isinstance(n, exp.Cast) for n in day.walk()) + else: + assert "DATE(paid_at)" in sql["day_volume"] + if kind == "mysql": + assert sql["sample"] is None + else: + sampled = sqlglot.parse_one(sql["sample"], read=READ[kind]) + assert any(isinstance(n, exp.TableSample) for n in sampled.walk()), sql["sample"] + + +# ---------------------------------------------------------------- IN 两边的类型族 + + +_FAMILY_OF = [(exp.DataType.TEXT_TYPES, "text"), (exp.DataType.NUMERIC_TYPES, "number"), + (exp.DataType.TEMPORAL_TYPES, "time")] + + +def _sg_family(t: exp.DataType | None) -> str | None: + if t is None: + return None + for group, name in _FAMILY_OF: + if t.this in group: + return name + return None + + +def _in_families(sql: str, kind: str, schema: dict[str, dict[str, str]]) -> list[tuple[str | None, set[str | None]]]: + """sql 里每个 IN:(被比较的列的类型族, 字面量的类型族集合),按 annotate_types 配上表结构的类型推出来。""" + tree = qualify(sqlglot.parse_one(sql, read=READ[kind]), schema=schema, dialect=READ[kind], + validate_qualify_columns=False) + tree = annotate_types(tree, schema=schema, dialect=READ[kind]) + return [(_sg_family(node.this.type), {_sg_family(x.type) for x in node.expressions}) + for node in tree.find_all(exp.In)] + + +@pytest.mark.parametrize("kind", DIALECTS) +@pytest.mark.parametrize("parent_type, child_values, child_type", [ + ("VARCHAR(10)", [12, 7, 30], "INTEGER"), # 父列文字、子列数:字面量要带引号 + ("INTEGER", ["12", "0007", "A1"], "VARCHAR(10)"), # 父列数、子列文字:只收整数写法,写成数字 + ("VARCHAR(10)", ["A ", "BC"], "CHAR(2)"), # 定长子列:去掉补齐的空格 + ("INTEGER", [1, 2], "INTEGER"), +]) +def test_in_list_literals_have_the_parent_columns_family(kind, parent_type, child_values, child_type): + d = SqlDialect(kind) + parent = {"name": "code", "type": parent_type} + literals, _ = key_literals(child_values, None, parent=catalog.column_family(parent), child_type=child_type) + assert literals + sql = d.match_count_sql("stores", d.quote("code"), literals[:IN_CHUNK]) + found = _in_families(sql, kind, {"stores": {"code": parent_type}}) + assert found, sql + for column_family, literal_families in found: + assert literal_families == {column_family}, (kind, sql, column_family, literal_families) + + +def test_the_old_way_would_mismatch_on_postgres(): + """对照:以前按子表取回的值写字面量,varchar 父列配整数子列在 sqlglot 推出的类型上就两边不同——PostgreSQL 真库上 + 这是「operator does not exist: character varying = integer」。""" + from app.data.catalog_profile import sql_literal + + old = [lit for v in (12, 7) if (lit := sql_literal(v, "number"))] + sql = SqlDialect("postgresql").match_count_sql("stores", "code", old) + [(column_family, literal_families)] = _in_families(sql, "postgresql", {"stores": {"code": "VARCHAR(10)"}}) + assert column_family == "text" and literal_families == {"number"} diff --git a/backend/tests/test_profile_dialect_units.py b/backend/tests/test_profile_dialect_units.py new file mode 100644 index 00000000..db01b746 --- /dev/null +++ b/backend/tests/test_profile_dialect_units.py @@ -0,0 +1,296 @@ +"""剖析与探查的方言修正(下一轮 03 细案 C1):不连库的部分。 + +- 列类型按方言编译的完整写法:类型族、日期列、码值候选都认得 ENUM / SET、VARCHAR2(20 CHAR)、INTERVAL 这些写法; +- IN 列表的字面量按被指向列的类型族生成(PG 上文字列配整数子列会直接报错,Oracle 会把父列转成数报 ORA-01722); +- Oracle 小写带引号的对象(quote_name)强制加引号、不转大写; +- 统计信息估出的行数不超过整表统计上限时再数一次确认; +- 日期列的最小值、最大值排除零日期和哨兵日期,照实写排除了几个; +- 取值观察要用的三个方言方法(date_literal / day_trunc / sample_from); +- 说明文字不再说「抽样」:大表看的是按存储顺序的前 N 行。 +""" +from __future__ import annotations + +import asyncio +import datetime as _dt +from typing import Any + +import pytest +from sqlalchemy import VARCHAR +from sqlalchemy.dialects import mysql, oracle, postgresql, sqlite +from sqlalchemy.sql.elements import quoted_name + +from app.data import catalog, catalog_profile, introspect +from app.data.catalog_profile import ProfileSettings, SqlDialect, TableSize +from app.data.engine import QueryResult + +DIALECTS = ("mysql", "mariadb", "postgresql", "oracle", "sqlite") + + +# ---------------------------------------------------------------- 类型的写法 + + +@pytest.mark.parametrize("type_name, family", [ + ("ENUM('paid','refund')", "text"), + ("ENUM('int','date')", "text"), # 取值里的词不能让它变成数字或时间 + ("SET('a','b')", "text"), + ("VARCHAR2(20 CHAR)", "text"), + ("NVARCHAR2(10)", "text"), + ("VARCHAR(20) COLLATE utf8mb4_bin", "text"), + ("NUMBER(10, 0)", "number"), + ("INTEGER(11)", "number"), + ("BIGINT UNSIGNED", "number"), + ("DOUBLE PRECISION", "number"), + ("TIMESTAMP WITH TIME ZONE", "time"), + ("TIMESTAMP(3)", "time"), + ("INTERVAL DAY(2) TO SECOND(6)", None), # 时长不是数也不是时刻;以前按「INT」认成了数字 + ("order_status", None), + ("", None), +]) +def test_type_family_reads_dialect_full_types(type_name, family): + assert catalog._type_family(type_name) == family + + +def test_type_family_of_a_column_uses_its_enum_values(): + """PostgreSQL 的枚举类型编译出来是类型名(order_status),只有取值列表能说明它是文字。""" + assert catalog.column_family({"name": "status", "type": "order_status", "enum_values": ["a", "b"]}) == "text" + assert catalog.column_family({"name": "n", "type": "INTEGER"}) == "number" + assert catalog.column_family({"name": "x"}) is None + + +def test_enum_values_never_make_a_date_column(): + assert not catalog.is_date_column("status", "ENUM('date','time')") + assert catalog.is_date_column("paid_at", "TIMESTAMP WITH TIME ZONE") + assert catalog.is_date_column("biz_dt", "VARCHAR2(10 CHAR)") # 文字存日期、名字像日期 + assert not catalog.is_date_column("gap", "INTERVAL DAY(2) TO SECOND(6)") + + +@pytest.mark.parametrize("col, ok", [ + ({"type": "VARCHAR2(20 CHAR)"}, True), + ({"type": "VARCHAR2(200 CHAR)"}, False), + ({"type": "CHAR(2)"}, True), + ({"type": "ENUM('paid','refund')"}, True), + ({"type": "SET('a','b')"}, True), + ({"type": "order_status", "enum_values": ["a", "b"]}, True), + ({"type": "NUMBER(10, 0)"}, True), + ({"type": "NUMBER(10, 2)"}, False), + ({"type": "TIMESTAMP(3)"}, False), + ({"type": "INTERVAL DAY(2) TO SECOND(6)"}, False), +]) +def test_code_type_accepts_enum_and_char_semantics(col, ok): + assert catalog_profile._code_type_ok(col.get("type"), col.get("enum_values")) is ok + + +def test_enum_columns_are_code_candidates_whatever_their_name(): + """ENUM 的取值列表有限,直接算码值候选:pay_way 这种名字不像码值的列以前被漏掉。""" + meta = {"columns": [{"name": "id", "type": "INTEGER"}, {"name": "pay_way", "type": "ENUM('cash','card')", + "enum_values": ["cash", "card"]}, + {"name": "pay_note", "type": "VARCHAR(20)"}], + "primary_key": ["id"]} + assert catalog_profile._code_columns(meta, {}) == ["pay_way"] + + +# ---------------------------------------------------------------- 探查:类型写法、取值列表、引号、索引首列 + + +@pytest.mark.parametrize("dialect, col_type, text", [ + (mysql.dialect(), mysql.ENUM("paid", "refund"), "ENUM('paid','refund')"), + (mysql.dialect(), mysql.TINYINT(display_width=1), "TINYINT(1)"), + (oracle.dialect(), oracle.VARCHAR2(20), "VARCHAR2(20 CHAR)"), + # Oracle 的 VARCHAR2 反射回来是通用的 VARCHAR(容器集成测试实测):照数据字典写成 VARCHAR2 + (oracle.dialect(), VARCHAR(10), "VARCHAR2(10 CHAR)"), + (oracle.dialect(), oracle.NUMBER(10, 0), "NUMBER(10, 0)"), + (postgresql.dialect(), postgresql.ENUM("a", "b", name="order_status"), "order_status"), + (sqlite.dialect(), sqlite.DATETIME(), "DATETIME"), +]) +def test_reflected_type_is_compiled_for_the_dialect(dialect, col_type, text): + assert introspect.type_text(col_type, dialect) == text + + +def test_type_that_cannot_compile_falls_back_to_str(): + from sqlalchemy.types import NullType + + assert introspect.type_text(NullType(), sqlite.dialect()) == "NULL" + assert introspect.type_text(None, sqlite.dialect()) == "" + + +def test_column_entry_records_enum_values_and_oracle_quoted_names(): + ora = oracle.dialect() + entry = introspect.column_entry({"name": quoted_name("level", quote=True), "type": oracle.VARCHAR2(10), + "nullable": True, "comment": None}, ora) + assert entry == {"name": "level", "type": "VARCHAR2(10 CHAR)", "nullable": True, "comment": None, + "quote_name": True} + plain = introspect.column_entry({"name": "status", "type": mysql.ENUM("a", "b"), "nullable": False}, mysql.dialect()) + assert plain["enum_values"] == ["a", "b"] and "quote_name" not in plain + members = introspect.column_entry({"name": "tags", "type": mysql.SET("x", "y")}, mysql.dialect()) + assert members["enum_values"] == ["x", "y"] and members["type"] == "SET('x','y')" + + +def test_index_first_columns_are_recorded(): + indexes = [{"column_names": ["paid_at", "id"]}, {"column_names": [None]}, {"column_names": ["store_id"]}, + {"column_names": ["paid_at"]}] + assert introspect.index_first_columns(indexes, ["id"]) == ["id", "paid_at", "store_id"] + + +# ---------------------------------------------------------------- IN 列表的字面量按被指向列的类型族 + + +def test_key_literals_follow_the_parent_family(): + lits, mismatched = catalog_profile.key_literals([12, 7], "number", parent="text", child_type="INTEGER") + assert lits == ["'12'", "'7'"] and mismatched == 0 + lits, mismatched = catalog_profile.key_literals(["12", "A1", "-3"], "text", parent="number", + child_type="VARCHAR(10)") + assert lits == ["12", "-3"] and mismatched == 1 + lits, mismatched = catalog_profile.key_literals([1, 2], "number", parent="time", child_type="INTEGER") + assert lits == [] and mismatched == 2 + # 父列类型不知道:照子表取回的值的类型写(以前的做法) + assert catalog_profile.key_literals([3, "x"], None, parent=None, child_type=None) == (["3", "'x'"], 0) + + +def test_char_keys_lose_their_padding(): + """PG、Oracle 的 CHAR(n) 取回来带尾随空格:拼成 'A ' 去和 VARCHAR 父列比就对不上。""" + lits, _ = catalog_profile.key_literals(["A ", "BC"], "text", parent="text", child_type="CHAR(2)") + assert lits == ["'A'", "'BC'"] + lits, _ = catalog_profile.key_literals(["A "], "text", parent="text", child_type="VARCHAR(2)") + assert lits == ["'A '"] + + +# ---------------------------------------------------------------- Oracle 小写带引号的对象 + + +def test_oracle_quote_name_forces_quotes_and_keeps_case(): + ora = SqlDialect("oracle") + meta = {"schema": "sales", "quote_name": True, + "columns": [{"name": "level", "type": "VARCHAR2(10 CHAR)", "quote_name": True}, + {"name": "amount", "type": "NUMBER(10, 2)"}]} + assert ora.table(meta, "visits") == 'sales."visits"' + assert ora.column(meta, "level") == '"level"' and ora.column(meta, "amount") == "amount" + stats = ora.stats_sql(meta, "visits") + assert "TABLE_NAME = 'visits'" in stats and "OWNER = 'SALES'" in stats + # 没有 quote_name 的照旧:不加引号,Oracle 自己转大写 + assert ora.table({"schema": None}, "visits") == "visits" + assert "TABLE_NAME = 'VISITS'" in ora.stats_sql({}, "visits") + + +# ---------------------------------------------------------------- 统计信息过期:再数一次确认 + + +def _result(value: Any) -> QueryResult: + return QueryResult(columns=["n"], rows=[[value]], row_count=1, truncated=False, elapsed_ms=1, sql="") + + +def _runner(answers: list[Any], seen: list[str]): + async def run(sql: str, **_kw: Any) -> QueryResult: + seen.append(sql) + return _result(answers.pop(0)) + return run + + +@pytest.mark.parametrize("stats, counted, expected", [ + (500, 500, TableSize(500, "count")), # 统计信息说小、数过也小:准数 + (500, 100_001, TableSize(None, "count", at_least=100_001)), # 统计信息过期:其实是大表 + (5_000_000, None, TableSize(5_000_000, "stats")), # 统计信息说大:不必再数 +]) +def test_small_stats_are_confirmed_with_a_bounded_count(stats, counted, expected): + seen: list[str] = [] + answers = [stats] + ([counted] if counted is not None else []) + size, skips = asyncio.run(catalog_profile.estimate_size(_runner(answers, seen), SqlDialect("postgresql"), + {"schema": None}, "visits", max_scan_rows=100_000)) + assert size == expected and skips == [] + assert len(seen) == (2 if counted is not None else 1) + if counted is not None: + assert "LIMIT 100001" in seen[1] + + +# ---------------------------------------------------------------- 日期列:排除零日期和哨兵日期 + + +@pytest.mark.parametrize("kind", DIALECTS) +def test_min_max_excludes_invalid_dates_and_counts_them(kind): + d = SqlDialect(kind) + sql = d.min_max_sql("t", [("paid_at", False), ("biz_day", True)]) + # 日期类型的列按日期字面量比,文字存的日期按字符串比(PG 上 varchar 和 date 比较直接报错) + lo, hi = d.date_literal("0001-01-02"), d.date_literal("9999-12-31") + assert f"paid_at >= {lo}" in sql and f"paid_at < {hi}" in sql + assert "biz_day >= '0001-01-02'" in sql and "biz_day < '9999-12-31'" in sql + assert sql.count("SUM(CASE WHEN") == 2 + + +def _dates_cx(row: list[Any], kind: str = "mysql"): + settings = ProfileSettings(enabled=True) + source = type("S", (), {"id": "s1", "name": "shop", "kind": kind, "options": {}})() + job = catalog_profile.ProfileContext(source=source, settings=settings, dialect=SqlDialect(kind), + runner=None, tables={"orders": {"columns": [ + {"name": "paid_at", "type": "DATETIME"}]}}, masked=set()) + job.sizes["orders"] = TableSize(10, "count") + result = catalog_profile.TableResult(table="orders") + cx = catalog_profile._TableCx(job, "orders", result, catalog_profile.TableHooks()) + + async def run(sql: str, **_kw: Any) -> QueryResult: + return QueryResult(columns=["a", "b", "c"], rows=[row], row_count=1, truncated=False, elapsed_ms=1, sql=sql) + cx.run = run # type: ignore[method-assign] + return cx, result + + +def test_dates_note_says_how_many_invalid_values_were_left_out(): + cx, result = _dates_cx(["2026-01-03 08:00:00", "2026-09-30 21:00:00", 4]) + out = catalog_profile._TableOutcome() + asyncio.run(catalog_profile._check_dates(cx, "orders", ["paid_at"], 1, out, result)) + assert result.date_ranges == [{"column": "paid_at", "min": "2026-01-03 08:00:00", "max": "2026-09-30 21:00:00", + "invalid": 4}] + assert "另有 4 个无效日期" in out.business_date.note and "0000-00-00" in out.business_date.note + + +def test_only_invalid_dates_is_no_data(): + cx, result = _dates_cx([None, None, 9]) + with pytest.raises(catalog_profile._Skip) as e: + asyncio.run(catalog_profile._check_dates(cx, "orders", ["paid_at"], 1, catalog_profile._TableOutcome(), + result)) + assert e.value.reason == "no_data" and "9 个无效日期" in e.value.detail + + +# ---------------------------------------------------------------- 取值观察要用的方言方法 + + +@pytest.mark.parametrize("kind, literal, day, sample", [ + ("mysql", "DATE '2026-09-01'", "DATE(paid_at)", None), + ("mariadb", "DATE '2026-09-01'", "DATE(paid_at)", None), + ("postgresql", "DATE '2026-09-01'", "CAST(paid_at AS DATE)", "orders TABLESAMPLE SYSTEM (1.5)"), + ("oracle", "DATE '2026-09-01'", "TRUNC(paid_at)", "orders SAMPLE BLOCK (1.5)"), + ("sqlite", "'2026-09-01'", "substr(paid_at, 1, 10)", None), +]) +def test_observation_dialect_methods(kind, literal, day, sample): + d = SqlDialect(kind) + assert d.date_literal(_dt.date(2026, 9, 1)) == literal == d.date_literal("2026-09-01") + assert d.date_literal("2026-09-01", text=True) == "'2026-09-01'" + assert d.day_trunc("paid_at") == day + assert d.sample_from("orders", 1.5) == sample + + +@pytest.mark.parametrize("bad", ["2026-9-1", "2026-09-01'; --", "0000-00-00", 20260901, None]) +def test_date_literal_refuses_anything_but_a_real_date(bad): + with pytest.raises(ValueError): + SqlDialect("postgresql").date_literal(bad) + + +@pytest.mark.parametrize("bad", [0, -1, 100, 101, float("nan")]) +def test_sample_percent_must_be_between_0_and_100(bad): + with pytest.raises(ValueError): + SqlDialect("postgresql").sample_from("orders", bad) + + +# ---------------------------------------------------------------- 说明文字 + + +def test_notes_say_storage_order_not_sampling(): + note = catalog_profile._relation_note(sample=2000, matched=2000, coverage=1.0, scan_cap=100_000, skipped_values=0, + mismatched=0, parent=(True, "主键"), child=(False, "count"), holds=True, + collation=None) + assert "按存储顺序只看前 100000 行" in note and "抽样" not in note + assert catalog_profile.FILTER_SAMPLED.format(rows="100", share="1%").startswith("剖析核对:按存储顺序取前 100 行统计") + + +def test_mysql_notes_mention_the_collation(): + note = catalog_profile._relation_note(sample=10, matched=10, coverage=1.0, scan_cap=None, skipped_values=0, + mismatched=0, parent=(True, "主键"), child=(False, "count"), holds=True, + collation=catalog_profile.MYSQL_COLLATION_NOTE) + assert "排序规则" in note and "不分大小写" in note diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index 51fb6dba..ed0661cf 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -2771,7 +2771,7 @@ export const SCHEMA_PARTIAL_TEXT = { // - 剖析是后台任务:逐张表进行,每张表剖析完就写进目录;查询一条一条地发,可以随时停止。关掉对话框照常进行。 // 目录只在整张表结束时写一次:停止、关停服务时正在做的那张来得及收尾,做完的项照常写进去;进程被杀时那张表没来得及收尾, // 这次做完的项一项都没写进去。说停下以后写进了什么,按逐表的状态照实说(cancelledNote / interruptedNote),不一律说「已写进」。 -// - 预算按表算:「每张表最多查询数」「每张表时长上限」用完只停这张表,下一张表重新计数;另有单条查询时限、抽样键值数、 +// - 预算按表算:「每张表最多查询数」「每张表时长上限」用完只停这张表,下一张表重新计数;另有单条查询时限、核对键值数、 // 整表统计行数上限。标签和服务端校验报错里的叫法一致(catalog_profile._NUMBER_FIELDS),保存被拒时按机读码认出是哪一项。 // - 没有整次总量上限:任务自己在连续失败、多数超时时停下(止损),写明原因。 // - 「没查成」≠ 结论错了:因为次数、时长、超时、出错没查成的项保留剖析以前的结论,可以「重试没做完的表」。 @@ -2783,7 +2783,7 @@ export const PROFILE_FIELD_LABEL: Record<'enabled' | CatalogProfileNumberKey, st table_max_queries: '每张表最多查询数', table_max_s: '每张表时长上限', query_timeout_s: '单条查询时限', - sample_size: '抽样键值数', + sample_size: '核对键值数', max_scan_rows: '整表统计行数上限', } @@ -2799,7 +2799,7 @@ export const PROFILE_FIELD_HINT: Record = { table_max_queries: '剖析一张表最多发出的查询条数,用完只停这张表,下一张表重新计数', table_max_s: '剖析一张表最多用的时长,到时只停这张表,下一张表重新计时', query_timeout_s: '超过即中断这条查询;不超过数据源自身的查询时限,也不超过这张表剩下的时长', - sample_size: '核对一条关联关系时,从本表抽取的不同键值个数', + sample_size: '核对一条关联关系时,从本表取的不同键值个数:取的是数据库先给出的那些,不是随机抽样', max_scan_rows: '不超过此行数的表才做去重计数、最小值和最大值等整表统计', } @@ -2863,7 +2863,7 @@ export const PROFILE_TEXT = { budgetPerTable: (queries: number, secs: number) => `每张表最多 ${formatNumber(queries)} 条查询、${formatNumber(secs)} 秒,用完只停这张表,下一张表重新计数`, budgetQuery: (secs: number) => `单条查询不超过 ${formatNumber(secs)} 秒`, - budgetSample: (n: number) => `每条关联关系抽样 ${formatNumber(n)} 个键值`, + budgetSample: (n: number) => `每条关联关系取 ${formatNumber(n)} 个不同键值核对`, budgetScan: (n: number) => (n > 0 ? `行数超过 ${formatNumber(n)} 的表不做整表统计` : '不做整表统计'), /** 上限的估算:表数 × 每张表的上限 */ budgetTotal: (tables: number, queries: number, span: string) => @@ -3002,7 +3002,7 @@ export const PROFILE_TEXT = { relationLowCoverage: '覆盖率不足,可能不是这条关系', relationNotUnique: '被指向列未核实唯一', coverage: (pct: string) => `覆盖率 ${pct}`, - sampled: (sample: number, matched: number) => `抽样 ${formatNumber(sample)} 个键值,对上 ${formatNumber(matched)} 个`, + sampled: (sample: number, matched: number) => `取了 ${formatNumber(sample)} 个不同键值,对上 ${formatNumber(matched)} 个`, cardinalityUnknown: '基数未核实', codeRows: (n: number) => `${formatNumber(n)} 行`, codesSummary: (values: number, rows: number) => `${formatNumber(values)} 个取值,共 ${formatNumber(rows)} 行`, @@ -3012,10 +3012,12 @@ export const PROFILE_TEXT = { dateProposal: (col: string) => `提议按 ${col} 作为业务日期`, dateSpan: (min: string, max: string) => `取值从 ${min} 到 ${max}`, dateRanges: '日期列的取值范围', + /** 日期列的最小值、最大值不计入的值(catalog_profile.VALID_DATE_FROM):零日期、0001-01-01、9999-12-31、无穷大 */ + dateInvalid: (n: number) => `另有 ${formatNumber(n)} 个无效日期未计入(零日期或占位日期)`, skippedTitle: (n: number) => `跳过 ${formatNumber(n)} 项`, moreSkipped: (n: number) => `…另有 ${formatNumber(n)} 项`, queriesTitle: (n: number) => `看发过的查询(${formatNumber(n)} 条)`, - queriesHint: 'SQL 里的取值都换成了「?」:抽样到的键值不保存在 AgentLab 中;遮罩的列不出现在任何查询里', + queriesHint: 'SQL 里的取值都换成了「?」:取到的键值不保存在 AgentLab 中;遮罩的列不出现在任何查询里', queryKind: { row_estimate: '行数估算', relation: '关系', codes: '码值', date: '日期' } as Record, queryOutcome: { ok: '成功', timeout: '超时', error: '失败', rejected: '未通过只读守卫' } as Record, queryRows: (n: number) => `${formatNumber(n)} 行`, diff --git a/frontend/src/pages/catalog/ProfileDialog.tsx b/frontend/src/pages/catalog/ProfileDialog.tsx index 321ad9c3..b14641fc 100644 --- a/frontend/src/pages/catalog/ProfileDialog.tsx +++ b/frontend/src/pages/catalog/ProfileDialog.tsx @@ -780,6 +780,10 @@ export function ProfileTaskBody({ detail: d, mode, filled = {}, focusAfter = nul {PT.dateProposal(date.column)} ,{PT.dateSpan(date.min.slice(0, 19), date.max.slice(0, 19))} + {(() => { + const invalid = d.date_ranges.find((r) => r.column === date.column)?.invalid + return invalid ? ,{PT.dateInvalid(invalid)} : null + })()} )} @@ -788,6 +792,7 @@ export function ProfileTaskBody({ detail: d, mode, filled = {}, focusAfter = nul {d.date_ranges.map((r) => (
  • {r.column}:{PT.dateSpan(r.min.slice(0, 19), r.max.slice(0, 19))} + {r.invalid ? `,${PT.dateInvalid(r.invalid)}` : null}
  • ))} diff --git a/frontend/src/pages/catalog/ProfileSettings.tsx b/frontend/src/pages/catalog/ProfileSettings.tsx index ddd9a3ec..91f13ee1 100644 --- a/frontend/src/pages/catalog/ProfileSettings.tsx +++ b/frontend/src/pages/catalog/ProfileSettings.tsx @@ -14,7 +14,7 @@ import type { ProfileForm } from './profile' // 数据剖析的设置:开关、五项预算、风险说明。数据源编辑框里是其中一段;导入表格的源没有编辑框,数据目录页在 // 剖析被拒(未开启)时也要就地打开它,所以另有一个只改这一项的弹窗(options 的其余键原样带上)。 // 剖析会对业务库发查询,开关默认关着;开启后五项预算才展开,风险说明写的是此刻填的数。 -// 预算按表算:每张表最多查询数、每张表时长上限(用完只停这张表),另有单条查询时限、抽样键值数、整表统计行数上限; +// 预算按表算:每张表最多查询数、每张表时长上限(用完只停这张表),另有单条查询时限、核对键值数、整表统计行数上限; // 没有整次剖析的总量上限(任务自己在连续失败、多数超时时停下)。 // =========================================================================== diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 0bd204f6..f44637d7 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -3528,7 +3528,8 @@ export interface CatalogProfileTaskDetail extends CatalogProfileTaskRow { findings: CatalogProfileFinding[] skipped: CatalogProfileSkip[] /** 日期类列的取值范围 */ - date_ranges: { column: string; min: string; max: string }[] + /** invalid:零日期、占位日期这类不计入最小值和最大值的个数(没有时不给) */ + date_ranges: { column: string; min: string; max: string; invalid?: number }[] query_log: CatalogProfileQuery[] } diff --git a/scripts/check-catalog-profile.mjs b/scripts/check-catalog-profile.mjs index 66efe68e..5c7f6cde 100644 --- a/scripts/check-catalog-profile.mjs +++ b/scripts/check-catalog-profile.mjs @@ -695,7 +695,7 @@ await section('剖析 · 设置:每张表的两项预算、被拒时报错落 await sec.locator('[data-profile-enabled]').check() const labels = await sec.locator('label.label').allInnerTexts() check('开启后五项预算展开:每张表最多查询数、每张表时长上限在前,没有整次的查询次数上限、总时长上限', - labels.join('、') === '每张表最多查询数、每张表时长上限、单条查询时限、抽样键值数、整表统计行数上限', labels.join('、')) + labels.join('、') === '每张表最多查询数、每张表时长上限、单条查询时限、核对键值数、整表统计行数上限', labels.join('、')) const units = await sec.locator('[data-profile-field] span').allInnerTexts() check('……每项带单位', units.join(',') === '条,秒,秒,个,行', units.join(',')) const hints = await sec.locator('[id$="-hint"]').allInnerTexts() @@ -783,8 +783,8 @@ await section('剖析 · 开始前:范围、个数与预算', async () => { check('……写明需要剖析的口径(含上次是空表的不算)、全部不含视图', (await inBox(page, '[data-profile-scope="needed"]')).includes('从没剖析过、上次没做完,或剖析之后又出现了待核实关联关系、要取码值或日期范围的列的表。上次是空表的不算') && (await inBox(page, '[data-profile-scope="all"]')).includes('不含视图')) const budget = await inBox(page, '[data-profile-budget]') - check('预算按表写:每张表最多 40 条、60 秒,单条 10 秒,抽样、整表统计门槛,按哪个源的设置', - ['每张表最多 40 条查询、60 秒,用完只停这张表', '单条查询不超过 10 秒', '抽样 2,000 个键值', '行数超过 100,000 的表不做整表统计', `按「${NAME1}」的数据剖析设置`] + check('预算按表写:每张表最多 40 条、60 秒,单条 10 秒,核对键值数、整表统计门槛,按哪个源的设置', + ['每张表最多 40 条查询、60 秒,用完只停这张表', '单条查询不超过 10 秒', '取 2,000 个不同键值核对', '行数超过 100,000 的表不做整表统计', `按「${NAME1}」的数据剖析设置`] .every((x) => budget.includes(x)), budget) check('……按这次的表数估上限:「这次 5 张表,最多约 200 条查询、5 分钟」', (await inBox(page, '[data-profile-budget-total]')) === '这次 5 张表,最多约 200 条查询、5 分钟', await inBox(page, '[data-profile-budget-total]')) diff --git a/scripts/dialect-it.sh b/scripts/dialect-it.sh new file mode 100755 index 00000000..0b7319ec --- /dev/null +++ b/scripts/dialect-it.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# 方言的容器集成测试(下一轮 03 细案 C2):本机 docker 起 PostgreSQL 16、MySQL 8.4、MariaDB 11、Oracle Free 23 四个容器, +# 建虚构的夹具(保留字列、Oracle 小写带引号的表、CHAR 码值、MySQL ENUM、零日期 / 无穷大 / 占位日期、做旧的统计信息、 +# 超过整表统计上限的大表、带索引的日期列),用 AgentLab 自己的探查和剖析代码跑全部剖析查询和取值观察的新方法, +# 断言零报错、结论和手算的一致。 +# +# 为什么不进 CI:要拉几 GB 的镜像、Oracle 起一次要一两分钟。要用户同意才在本机跑。跑完删容器、留镜像。 +# +# 用法:scripts/dialect-it.sh 四种库都跑 +# DIALECT_IT_ONLY=pg,mysql scripts/dialect-it.sh 只跑其中几种(pg、mysql、mariadb、oracle) +# PYTHON=/path/to/python scripts/dialect-it.sh 指定装着后端依赖的 Python(缺省 python3) +# +# 夹具全是虚构的通用名字(stores、orders、events、visits),不连任何业务库。 +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +PY="${PYTHON:-python3}" +ONLY="${DIALECT_IT_ONLY:-pg,mysql,mariadb,oracle}" +PREFIX="agentlab-dialect-it" +PASS="itpass" + +started=() +cleanup() { + for name in "${started[@]}"; do + docker rm -f "$name" >/dev/null 2>&1 || true + done +} +trap cleanup EXIT + +want() { [[ ",$ONLY," == *",$1,"* ]]; } + +run() { + local name="$PREFIX-$1"; shift + docker rm -f "$name" >/dev/null 2>&1 || true + docker run -d --name "$name" "$@" >/dev/null + started+=("$name") +} + +host_port() { docker port "$PREFIX-$1" "$2" | head -1 | sed 's/.*://'; } + +if want pg; then + run pg -e POSTGRES_PASSWORD="$PASS" -e POSTGRES_DB=it -p 127.0.0.1::5432 postgres:16 +fi +if want mysql; then + run mysql -e MYSQL_ROOT_PASSWORD="$PASS" -e MYSQL_DATABASE=it -p 127.0.0.1::3306 mysql:8.4 +fi +if want mariadb; then + run mariadb -e MARIADB_ROOT_PASSWORD="$PASS" -e MARIADB_DATABASE=it -p 127.0.0.1::3306 mariadb:11 +fi +if want oracle; then + run oracle -e ORACLE_PASSWORD="$PASS" -e APP_USER=it -e APP_USER_PASSWORD="$PASS" -p 127.0.0.1::1521 \ + gvenzl/oracle-free:23-slim +fi + +export DIALECT_IT_PASSWORD="$PASS" DIALECT_IT_ONLY="$ONLY" +want pg && export DIALECT_IT_PG_PORT="$(host_port pg 5432/tcp)" +want mysql && export DIALECT_IT_MYSQL_PORT="$(host_port mysql 3306/tcp)" +want mariadb && export DIALECT_IT_MARIADB_PORT="$(host_port mariadb 3306/tcp)" +want oracle && export DIALECT_IT_ORACLE_PORT="$(host_port oracle 1521/tcp)" + +# 等库起来由 Python 那边按连接重试(Oracle 最慢,最多等 6 分钟) +PYTHONPATH="$ROOT/backend" "$PY" "$ROOT/scripts/dialect_it.py" diff --git a/scripts/dialect_it.py b/scripts/dialect_it.py new file mode 100644 index 00000000..ff153dd1 --- /dev/null +++ b/scripts/dialect_it.py @@ -0,0 +1,388 @@ +"""方言容器集成测试的执行端(由 scripts/dialect-it.sh 起好容器后调用,不进 CI)。 + +对每种库: +1. 用管理账号建虚构夹具(见 _ddl / _rows); +2. 用 AgentLab 自己的探查(app.data.introspect)读结构,核对类型写法、ENUM 取值、Oracle 小写带引号的名字、索引首列; +3. 用 AgentLab 自己的剖析(catalog_profile.profile_table,目录读写换成内存里的)逐表剖析,经只读连接、守卫、单条时限, + 和线上走的是同一条路; +4. 直接跑取值观察要用的新查询(按日行数、时间窗口里的取值分布、块采样); +5. 断言:所有查询零报错(留痕里没有超时、失败、守卫拒),每个结论和下面按夹具定义手算的一致。 + +夹具的数(手算的依据):orders 40 行,id 1–40。 +- store_code = id % 5 + 1(整数子列,指向文字主键 stores.code,'1'–'5' 都在); +- status CHAR(2):id 是 3 的倍数为 'A',否则 'B'(13 / 27);type:奇数 'web'、偶数 'shop'(20 / 20); + level = id % 3 + 1(1: 13、2: 14、3: 13);pay_way(只 MySQL / MariaDB,ENUM('cash','card','coupon')):奇数 cash、偶数 card; +- paid_at = 2026-09-01 10:00 加 (id % 10) 天,id 39、40 是无效日期(MySQL 零日期、PG ±infinity、Oracle 0001-01-01 和 + 9999-12-31 23:59:59);date(日期列)同 paid_at 的日子,id 39、40 为空; +- events 先插 10 行、收集统计信息,再插 1500 行(统计信息做旧:说 10 行,其实 1510 行,超过整表统计上限 1000); +- Oracle 另有小写带引号的表 "visits"("id"、"level"),3 行,level 1、1、2。 +""" +from __future__ import annotations + +import asyncio +import datetime as _dt +import os +import sys +import tempfile +import time +from collections import Counter +from types import SimpleNamespace +from typing import Any + +# AgentLab 自己的库放到临时目录:不碰开发数据目录(必须在导入 app 之前设) +os.environ["AGENTLAB_DATA_DIR"] = tempfile.mkdtemp(prefix="agentlab-dialect-it-") + +from sqlalchemy import text # noqa: E402 +from sqlalchemy.ext.asyncio import create_async_engine # noqa: E402 + +from app.core.crypto import encrypt # noqa: E402 +from app.data import catalog, catalog_profile, introspect # noqa: E402 +from app.data import engine as data_engine # noqa: E402 +from app.data.catalog_profile import ProfileSettings, SqlDialect # noqa: E402 + +PASSWORD = os.environ.get("DIALECT_IT_PASSWORD", "itpass") +SETTINGS = ProfileSettings(enabled=True, table_max_queries=40, table_max_s=60, query_timeout_s=10, sample_size=100, + max_scan_rows=1000) +N_ORDERS, N_EVENTS_OLD, N_EVENTS_NEW = 40, 10, 1500 +BASE = _dt.datetime(2026, 9, 1, 10, 0, 0) +WINDOW_FROM = "2026-09-05" + +failures: list[str] = [] + + +def check(ok: bool, what: str) -> None: + print((" ✓ " if ok else " ✗ ") + what) + if not ok: + failures.append(what) + + +# ---------------------------------------------------------------- 手算 + + +def _order(i: int) -> dict[str, Any]: + return {"id": i, "store_code": i % 5 + 1, "status": "A" if i % 3 == 0 else "B", + "type": "web" if i % 2 else "shop", "level": i % 3 + 1, "pay_way": "cash" if i % 2 else "card", + "paid_at": None if i >= 39 else BASE + _dt.timedelta(days=i % 10), + "date": None if i >= 39 else (BASE + _dt.timedelta(days=i % 10)).date()} + + +ORDERS = [_order(i) for i in range(1, N_ORDERS + 1)] +VALID = [o for o in ORDERS if o["paid_at"] is not None] +IN_WINDOW = [o for o in VALID if o["paid_at"].date().isoformat() >= WINDOW_FROM] +EXPECT = { + "status": Counter(o["status"] for o in ORDERS), + "type": Counter(o["type"] for o in ORDERS), + "level": Counter(str(o["level"]) for o in ORDERS), + "pay_way": Counter(o["pay_way"] for o in ORDERS), + "paid_at": (min(o["paid_at"] for o in VALID), max(o["paid_at"] for o in VALID)), + "day_volume": Counter(o["paid_at"].date().isoformat() for o in IN_WINDOW), + "window_status": Counter(o["status"] for o in IN_WINDOW), +} + + +# ---------------------------------------------------------------- 各库的夹具 + + +class Target: + def __init__(self, key: str, kind: str, port: int) -> None: + self.key, self.kind, self.port = key, kind, port + self.oracle = kind == "oracle" + self.mysql = kind in ("mysql", "mariadb") + self.pg = kind == "postgresql" + + def admin_url(self) -> str: + if self.pg: + return f"postgresql+asyncpg://postgres:{PASSWORD}@127.0.0.1:{self.port}/it" + if self.mysql: + return f"mysql+aiomysql://root:{PASSWORD}@127.0.0.1:{self.port}/it" + return f"oracle+oracledb://it:{PASSWORD}@127.0.0.1:{self.port}/?service_name=FREEPDB1" + + def source(self) -> SimpleNamespace: + user = {"postgresql": "postgres", "mysql": "root", "mariadb": "root", "oracle": "it"}[self.kind] + return SimpleNamespace(id=f"it-{self.key}", name=f"it_{self.key}", kind=self.kind, host="127.0.0.1", + port=self.port, database="FREEPDB1" if self.oracle else "it", username=user, + password=encrypt(PASSWORD), options={}, readonly=True, description="", + schema_cache={}, origin="manual", enabled=True) + + +def _q(t: Target, name: str) -> str: + """建表时的列名:Oracle 上 TYPE、LEVEL、DATE 是保留字,按大写加引号建(库里就是大写的普通名字)。""" + if t.oracle and name in ("type", "level", "date"): + return f'"{name.upper()}"' + if t.mysql: + return f"`{name}`" + return f'"{name}"' if name in ("type", "level", "date") else name + + +def _ddl(t: Target) -> list[str]: + ts = "TIMESTAMP" if not t.mysql else "DATETIME" + intg = "NUMBER(10)" if t.oracle else "INTEGER" + vc = "VARCHAR2" if t.oracle else "VARCHAR" + cols = [f"id {intg} PRIMARY KEY", f"store_code {intg}", "status CHAR(2)", f"{_q(t, 'type')} {vc}(10)", + f"{_q(t, 'level')} {intg}", f"paid_at {ts}", f"{_q(t, 'date')} DATE"] + if t.mysql: + cols.append("pay_way ENUM('cash','card','coupon')") + out = [f"CREATE TABLE stores (code {vc}(10) PRIMARY KEY, name {vc}(20))", + f"CREATE TABLE orders ({', '.join(cols)})", + "CREATE INDEX ix_orders_paid_at ON orders (paid_at)"] + events = f"CREATE TABLE events (id {intg} PRIMARY KEY, kind {vc}(10))" + if t.pg: + events += " WITH (autovacuum_enabled = false)" + if t.mysql: + events += " STATS_PERSISTENT=1 STATS_AUTO_RECALC=0" + out.append(events) + if t.oracle: + out.append('CREATE TABLE "visits" ("id" NUMBER(10) PRIMARY KEY, "level" NUMBER(10))') + return out + + +def _paid(t: Target, o: dict[str, Any]) -> str: + if o["paid_at"] is not None: + stamp = o["paid_at"].strftime("%Y-%m-%d %H:%M:%S") + return f"TIMESTAMP '{stamp}'" if t.oracle else f"'{stamp}'" + # 无效日期:各家能存进去的那种 + if t.mysql: + return "'0000-00-00 00:00:00'" + if t.pg: + return "'infinity'" if o["id"] == 39 else "'-infinity'" + return "TIMESTAMP '9999-12-31 23:59:59'" if o["id"] == 39 else "TIMESTAMP '0001-01-01 00:00:00'" + + +def _rows(t: Target) -> list[str]: + out = [f"INSERT INTO stores (code, name) VALUES ('{c}', 'store {c}')" for c in ("1", "2", "3", "4", "5", "A1")] + for o in ORDERS: + day = f"DATE '{o['date'].isoformat()}'" if o["date"] is not None else "NULL" + cols = ["id", "store_code", "status", _q(t, "type"), _q(t, "level"), "paid_at", _q(t, "date")] + vals = [str(o["id"]), str(o["store_code"]), f"'{o['status']}'", f"'{o['type']}'", str(o["level"]), + _paid(t, o), day] + if t.mysql: + cols.append("pay_way") + vals.append(f"'{o['pay_way']}'") + out.append(f"INSERT INTO orders ({', '.join(cols)}) VALUES ({', '.join(vals)})") + if t.oracle: + out += ['INSERT INTO "visits" ("id", "level") VALUES (1, 1)', 'INSERT INTO "visits" ("id", "level") VALUES (2, 1)', + 'INSERT INTO "visits" ("id", "level") VALUES (3, 2)'] + return out + + +def _events(start: int, n: int) -> list[str]: + return [f"INSERT INTO events (id, kind) VALUES ({i}, '{'open' if i % 4 else 'close'}')" + for i in range(start, start + n)] + + +def _analyze(t: Target, table: str) -> list[str]: + if t.pg: + return [f"ANALYZE {table}"] + if t.mysql: + return [f"ANALYZE TABLE {table}"] + return [f"BEGIN DBMS_STATS.GATHER_TABLE_STATS(USER, '{table.upper()}'); END;"] + + +async def _wait(t: Target, deadline_s: float = 360) -> Any: + engine = create_async_engine(t.admin_url()) + started = time.monotonic() + while True: + try: + async with engine.connect() as conn: + await conn.execute(text("SELECT 1 FROM DUAL" if t.oracle else "SELECT 1")) + return engine + except Exception as e: # noqa: BLE001 - 容器还没起来:等 + if time.monotonic() - started > deadline_s: + raise RuntimeError(f"{t.key} 在 {deadline_s:.0f} 秒内没起来:{e}") from e + await asyncio.sleep(3) + + +async def _setup(t: Target) -> None: + engine = await _wait(t) + try: + async with engine.begin() as conn: + if t.mysql: + # 关掉严格模式才写得进零日期(老库里常见的情形) + await conn.execute(text("SET SESSION sql_mode = ''")) + for sql in _ddl(t) + _rows(t) + _events(1, N_EVENTS_OLD): + await conn.exec_driver_sql(sql) + async with engine.begin() as conn: + for sql in _analyze(t, "events") + _analyze(t, "orders") + _analyze(t, "stores"): + await conn.exec_driver_sql(sql) + if t.oracle: + await conn.exec_driver_sql("""BEGIN DBMS_STATS.GATHER_TABLE_STATS(USER, '"visits"'); END;""") + async with engine.begin() as conn: + if t.mysql: + await conn.execute(text("SET SESSION sql_mode = ''")) + for sql in _events(N_EVENTS_OLD + 1, N_EVENTS_NEW): + await conn.exec_driver_sql(sql) + finally: + await engine.dispose() + + +# ---------------------------------------------------------------- 剖析:目录换成内存里的 + + +async def _profile(source: Any, tables: list[str], notes: dict[str, dict[str, Any]]) -> dict[str, Any]: + cx = catalog_profile.make_context(source, SETTINGS, options={}) + captured: dict[str, Any] = {} + + async def db(fn: Any, *args: Any) -> Any: + if fn is catalog_profile._read_notes: + return notes.get(args[1], {}) + if fn is catalog_profile._record: + _sid, table, out, result, _actor, _rec = args + catalog_profile._settle_status(result) + captured[table] = (out, result) + return None + raise AssertionError(f"剖析要了没预料的库操作:{fn}") + + for table in tables: + await catalog_profile.profile_table(cx, table, db=db) + return captured + + +def _key(meta: dict[str, Any], name: str) -> str: + """表结构里这个名字的写法(Oracle 上普通名字是小写,TYPE 这类保留字是大写)。""" + return next(str(c["name"]) for c in meta["columns"] if str(c["name"]).lower() == name) + + +def _day(v: Any) -> str: + return str(v)[:10] + + +async def run_target(t: Target) -> None: + print(f"\n== {t.key}({t.kind},端口 {t.port})") + await _setup(t) + source = t.source() + try: + cache = await introspect.introspect(source) + source.schema_cache = cache + tables = cache["tables"] + orders, events, stores = tables["orders"], tables["events"], tables["stores"] + types = {c["name"].lower(): c for c in orders["columns"]} + check(not cache.get("failed") and {"orders", "events", "stores"} <= set(tables), "探查读到三张夹具表") + check("paid_at" in [c.lower() for c in orders.get("index_first_columns") or []], + f"索引首列记进了表结构:{orders.get('index_first_columns')}") + if t.oracle: + check(stores["columns"][0]["type"] == "VARCHAR2(10 CHAR)", f"Oracle 文字类型写全了长度和单位:" + f"{stores['columns'][0]['type']}") + check(types["id"]["type"] == "NUMBER(10, 0)", f"Oracle 数字类型写全了精度:{types['id']['type']}") + check("visits" in tables and tables["visits"].get("quote_name") is True, + "Oracle 小写带引号的表记了 quote_name") + check(all(c.get("quote_name") is True for c in tables["visits"]["columns"]), + "Oracle 小写带引号的列记了 quote_name") + check(_key(orders, "level") == "LEVEL" and "quote_name" not in types["level"], + "Oracle 的保留字列 LEVEL 是大写的普通名字,不记 quote_name") + if t.mysql: + check(types["pay_way"]["type"].startswith("ENUM(") and + types["pay_way"].get("enum_values") == ["cash", "card", "coupon"], + f"MySQL ENUM 记下了取值列表:{types['pay_way']}") + if t.pg: + check(types["paid_at"]["type"] == "TIMESTAMP WITHOUT TIME ZONE", + f"PostgreSQL 时间类型写全了时区:{types['paid_at']['type']}") + + store_col, code_col = _key(orders, "store_code"), _key(stores, "code") + rel = catalog.relation_id("orders", [store_col], "stores", [code_col]) + notes = {"orders": {"relations": [{"id": rel, "columns": [store_col], "to_table": "stores", + "to_columns": [code_col], "cardinality": "many_to_one", "source": "name", + "status": "proposed"}]}} + names = ["orders", "events", "stores"] + (["visits"] if t.oracle else []) + got = await _profile(source, names, notes) + + logs = [q for _, r in got.values() for q in r.query_log] + bad = [f"{q['kind']} {q['target']}:{q['outcome']} {q['detail']}" for q in logs if q["outcome"] != "ok"] + check(not bad, f"剖析的 {len(logs)} 条查询零报错" + (f":{bad[:3]}" if bad else "")) + + out, result = got["orders"] + check(result.status == "ok", f"orders 剖析完成:{result.status} {result.error or ''}") + [r] = out.relations or [None] + check(r is not None and r.coverage == 1.0 and r.sample == 5 and r.matched == 5 and r.holds, + f"整数子列对文字主键:取 5 个键值全对上、升为有确证({r and r.note})") + codes = {c.column.lower(): c for c in out.codes} + for col in ("status", "type", "level") + (("pay_way",) if t.mysql else ()): + c = codes.get(col) + check(c is not None and Counter(dict(c.values)) == EXPECT[col], + f"{col} 的取值分布和手算一致:{c and c.values}") + if t.oracle or t.pg: + check(codes["status"].values and all(not k.endswith(" ") for k, _ in codes["status"].values), + "CHAR(2) 码值去掉了补齐的空格") + if t.mysql: + check(codes["pay_way"].declared == ["coupon"] and "coupon" in codes["pay_way"].note, + "ENUM 列定义里没出现的取值也进了码值") + check(catalog_profile.MYSQL_CODES_COLLATION_NOTE in codes["status"].note, "MySQL 码值说明写了排序规则") + check(catalog_profile.MYSQL_COLLATION_NOTE in r.note, "MySQL 关系说明写了排序规则") + ranges = {d["column"].lower(): d for d in result.date_ranges} + paid = ranges.get("paid_at") + lo, hi = EXPECT["paid_at"] + check(paid is not None and paid["min"][:19].replace("T", " ") == str(lo) and + paid["max"][:19].replace("T", " ") == str(hi) and paid.get("invalid") == 2, + f"paid_at 的范围排除了 2 个无效日期:{paid}") + day = ranges.get("date") + check(day is not None and _day(day["min"]) == lo.date().isoformat() and _day(day["max"]) == + hi.date().isoformat() and not day.get("invalid"), f"date 列(保留字)的范围:{day}") + + out_e, result_e = got["events"] + size = result_e.row_estimate + d = SqlDialect(t.kind) + stats = await data_engine.run_query(source, d.stats_sql(events, "events")) + estimated = int(float(stats.rows[0][0] or 0)) if stats.rows else 0 + if estimated <= SETTINGS.max_scan_rows: + check(size is not None and size.rows is None and size.at_least == SETTINGS.max_scan_rows + 1, + f"统计信息做旧(说 {estimated} 行,其实 {N_EVENTS_OLD + N_EVENTS_NEW} 行)也认出了大表:{size}") + else: + # 有的库插入时自己就更新了行数估算,夹具做不旧:照实说,只核对按统计信息认出了大表 + print(f" · 统计信息没能做旧(库自己更新成了 {estimated} 行),只核对按统计信息认出大表") + check(size is not None and size.method == "stats" and (size.rows or 0) > SETTINGS.max_scan_rows, + f"按统计信息认出了大表:{size}") + kind_codes = next((c for c in out_e.codes if c.column.lower() == "kind"), None) + check(kind_codes is not None and not kind_codes.complete and "按存储顺序取前" in kind_codes.note, + f"大表的码值按存储顺序的前若干行,不标已列全:{kind_codes and kind_codes.note}") + + if t.oracle: + out_v, result_v = got["visits"] + lv = next((c for c in out_v.codes if c.column == "level"), None) + check(result_v.status == "ok" and lv is not None and Counter(dict(lv.values)) == Counter({"1": 2, "2": 1}), + f"小写带引号的表和列照样剖析:{result_v.status} {lv and lv.values}") + vstats = await data_engine.run_query(source, d.stats_sql(tables["visits"], "visits")) + check(bool(vstats.rows) and vstats.rows[0][0] is not None, f"小写表的统计信息按原名查得到:{vstats.rows}") + + # ---- 取值观察的新方法 + tbl = d.table(orders, "orders") + paid_col = d.column(orders, _key(orders, "paid_at")) + res = await data_engine.run_query(source, d.day_volume_sql(tbl, paid_col, WINDOW_FROM)) + got_days = Counter({_day(row[0]): int(row[1]) for row in res.rows}) + check(got_days == EXPECT["day_volume"], f"按日行数和手算一致(无效日期不算进窗口):{dict(got_days)}") + status_col = d.column(orders, _key(orders, "status")) + res = await data_engine.run_query(source, d.window_value_counts_sql(tbl, status_col, paid_col, WINDOW_FROM, 21)) + got_window = Counter({str(row[0]).rstrip(" "): int(row[1]) for row in res.rows}) + check(got_window == EXPECT["window_status"], f"时间窗口里的取值分布和手算一致:{dict(got_window)}") + sampled = d.sample_value_counts_sql(d.table(events, "events"), d.column(events, _key(events, "kind")), 50, 21) + if t.pg or t.oracle: + res = await data_engine.run_query(source, sampled) + total = sum(int(row[1]) for row in res.rows) + check(total <= N_EVENTS_OLD + N_EVENTS_NEW, f"块采样跑得通(取到 {total} 行)") + else: + check(sampled is None, "MySQL / MariaDB 没有块采样,照实返回 None") + finally: + await data_engine.engines.invalidate(source.id) + + +async def main() -> int: + only = set((os.environ.get("DIALECT_IT_ONLY") or "pg,mysql,mariadb,oracle").split(",")) + plan = [("pg", "postgresql", "DIALECT_IT_PG_PORT"), ("mysql", "mysql", "DIALECT_IT_MYSQL_PORT"), + ("mariadb", "mariadb", "DIALECT_IT_MARIADB_PORT"), ("oracle", "oracle", "DIALECT_IT_ORACLE_PORT")] + for key, kind, env in plan: + if key not in only: + continue + port = os.environ.get(env) + if not port: + failures.append(f"{key}:没有拿到容器端口({env})") + continue + try: + await run_target(Target(key, kind, int(port))) + except Exception as e: # noqa: BLE001 - 一种库出错不挡别的库,照实记下 + print(f" ✗ {key} 中途出错:{type(e).__name__}: {e}") + failures.append(f"{key}:{type(e).__name__}: {e}") + await data_engine.engines.close() + print("\n" + ("全部通过" if not failures else f"{len(failures)} 项不通过:\n- " + "\n- ".join(failures))) + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(asyncio.run(main())) From 4c0e83bc80355a27dc1a97d49eee16a05656027a Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Fri, 16 Oct 2026 10:28:01 +0800 Subject: [PATCH 13/39] =?UTF-8?q?=E7=9B=AE=E5=BD=95=E8=A7=86=E5=9B=BE?= =?UTF-8?q?=E6=8E=A5=E7=BC=9D=20data/catalog=5Fview.py=EF=BC=88=E7=BB=86?= =?UTF-8?q?=E6=A1=88=2001=20=E5=81=9A=E6=B3=95=205=EF=BC=8C=E6=80=BB?= =?UTF-8?q?=E6=96=B9=E6=A1=88=20X7=EF=BC=89=EF=BC=9A=E8=AF=BB=E7=9B=AE?= =?UTF-8?q?=E5=BD=95=E7=9A=84=E5=9B=9B=E5=A4=84=E7=BB=9F=E4=B8=80=E7=BB=8F?= =?UTF-8?q?=E5=AE=83=EF=BC=8C=E3=80=8C=E5=BC=80=E7=9B=AE=E5=BD=95=20/=20?= =?UTF-8?q?=E5=85=B3=E7=9B=AE=E5=BD=95=20/=20=E5=8F=AA=E7=94=A8=E5=B7=B2?= =?UTF-8?q?=E6=A0=B8=E5=AE=9E=E7=9A=84=E9=A1=B9=20/=20=E7=9B=AE=E5=BD=95?= =?UTF-8?q?=E5=BF=AB=E7=85=A7=E3=80=8D=E7=9A=84=E5=AF=B9=E7=85=A7=E6=89=8D?= =?UTF-8?q?=E5=81=9A=E5=BE=97=E4=BA=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - CatalogView{mode: live | snapshot | off, statuses: all | reviewed, snapshot}:缺省 live + all,行为和以前一样;reviewed 只留已核实、已确认的项(驳回的项原样留着交给各处,照旧经 visible_notes 去掉,冻结快照和 SQL 检查靠驳回的关系编号认出不该现推的关系——为此在 catalog.py 加 restrict_notes,没给 visible_notes 加参数) - 视图从哪来:运行从 Run.options['catalog_view'] 读(catalog_view.of_run:RunContext.extra 里的发起快照,审批恢复、续跑沿用),ToolContext 多一个 catalog_view,节点(调用工具、llm/agent、多 agent)构造时带上;生成过程从参数读(下一个提交接 answer_ops) - 四处消费一律经它:运行时 db_schema / db_query(tools/datasource._catalog_of,冻结进表结构快照的也是这一份)、look 表详情(TableKind.load、copilot_context.look_service 收 catalog_view)、自查的 SQL 检查器(sqlcheck.load_checkers 收 catalog_view,改用 catalog.read_catalogs 一次读出)、双盲复算(_Sources 带视图,交给盲算的目录和比较 SQL 用的中文名都按它;核查 agent、反方审稿的 db_schema 也按这次运行的视图) - 冻结快照:不是缺省视图的在引用 meta 里记下 catalog_view(不进内容哈希,缺省视图不记,快照和以前一字不差),证据面板据此写得出「本次运行目录:关」 - snapshot 模式只定读法(kind=eval_catalog_snapshot 的工件,{sources: {源 id: {tables: {表: {version, notes}}}}}),内容由第 2 波的题集包写;快照不在了照实报错(CatalogSnapshotMissing),不悄悄退回当前目录 - 新增 tests/test_catalog_view.py:四种视图在四处给的是同一份、缺省等于当前目录、运行从 options 读且冻结 meta 记下、lint(消费模块不许直接 read_catalog / read_entry / 查 CatalogNote);去掉 _catalog_of 里的视图回跑两项失败 --- backend/app/api/copilot_context.py | 7 +- backend/app/data/catalog.py | 25 ++- backend/app/data/catalog_view.py | 181 +++++++++++++++++ backend/app/data/sqlcheck.py | 26 +-- backend/app/engine/nodes/challenge.py | 4 +- backend/app/engine/nodes/dual_check.py | 25 ++- backend/app/engine/nodes/llm.py | 3 + backend/app/engine/nodes/multi.py | 3 + backend/app/engine/nodes/tools.py | 3 + backend/app/engine/verify_agent.py | 8 +- backend/app/look/tables.py | 12 +- backend/app/tools/datasource.py | 25 ++- backend/app/tools/registry.py | 3 + backend/tests/test_catalog_view.py | 270 +++++++++++++++++++++++++ 14 files changed, 548 insertions(+), 47 deletions(-) create mode 100644 backend/app/data/catalog_view.py create mode 100644 backend/tests/test_catalog_view.py diff --git a/backend/app/api/copilot_context.py b/backend/app/api/copilot_context.py index 6c8e807e..7375d5f0 100644 --- a/backend/app/api/copilot_context.py +++ b/backend/app/api/copilot_context.py @@ -65,16 +65,17 @@ SYSTEM_LOOK_UPGRADE_NEED = "要升级的工作流里,查库节点的 SQL 用 async def look_service(sources: list[Any], *, spec: ModelSpec | None, graph: Mapping[str, Any] | None = None, - kinds: tuple[str, ...] = ("table", "capability"), answer: bool = False) -> LookService: + kinds: tuple[str, ...] = ("table", "capability"), answer: bool = False, + catalog_view: Any = None) -> LookService: """按入口组装这一轮的查看服务。读目录、读能力登记都用各自的短会话,开流前后都能调。 spec 是助手的模型规格:挑选模型第一次要用时才按它拿(关思考、额度按 look/pick.PICK_MAX_TOKENS)。graph 是现有工作流,概览里列出它查过的表。 - answer:问数据(意图为回答),能力只列只读的。 + answer:问数据(意图为回答),能力只列只读的。catalog_view:表详情里给哪一份数据目录(data/catalog_view.py),不给是当前目录。 """ picker = Picker(spec) made: list[Any] = [] if "table" in kinds: - made.append(await TableKind.load(sources, graph=graph, picker=picker)) + made.append(await TableKind.load(sources, graph=graph, picker=picker, catalog_view=catalog_view)) if "capability" in kinds: made.append(await CapabilityKind.load(picker=picker, read_only=answer)) return LookService(made) diff --git a/backend/app/data/catalog.py b/backend/app/data/catalog.py index 3247aea4..60617db2 100644 --- a/backend/app/data/catalog.py +++ b/backend/app/data/catalog.py @@ -420,6 +420,18 @@ async def read_catalog(session: AsyncSession, source_id: str) -> dict[str, Catal return {row.table_name: _entry(row) for row in rows} +async def read_catalogs(session: AsyncSession, source_ids: Iterable[str]) -> dict[str, dict[str, CatalogEntry]]: + """几个数据源的目录一次查询读出:{源 id: {表名: CatalogEntry}},没有目录的源不出现。给模型、给自查读目录的地方经 + data/catalog_view.py 调它,不直接查 catalog_notes。""" + ids = sorted({i for i in source_ids if i}) + if not ids: + return {} + out: dict[str, dict[str, CatalogEntry]] = {} + for row in (await session.execute(select(CatalogNote).where(CatalogNote.source_id.in_(ids)))).scalars(): + out.setdefault(row.source_id, {})[row.table_name] = _entry(row) + return out + + async def write_entry(session: AsyncSession, source_id: str, table: str, notes: dict[str, Any], *, if_version: int, actor: str | None, commit: bool = True) -> CatalogEntry: """整份写入一张表的目录(乐观锁),提交事务,返回写入后的目录。 @@ -592,6 +604,14 @@ def visible_notes(notes: Mapping[str, Any] | None) -> dict[str, Any]: return _assemble({k: v for k, v in _slots(notes).items() if v.get("status") != "rejected"}) +def restrict_notes(notes: Mapping[str, Any] | None, statuses: Iterable[str]) -> dict[str, Any]: + """只留状态在 statuses 里的项(表级项、列级项、关系都按项算);驳回的项原样留着——给人、给模型看的照旧经 + visible_notes 去掉,冻结快照和 SQL 检查要靠驳回的关系编号认出不该按外键、命名现推回来的关系(frozen_catalog)。 + 目录视图「只用已核实、已确认的项」用它(data/catalog_view.py)。""" + keep = set(statuses) | {"rejected"} + return _assemble({k: v for k, v in _slots(notes).items() if v.get("status") in keep}) + + def status_counts(notes: Mapping[str, Any] | None) -> dict[str, int]: """各状态的项数(表级项、列级项、关系都算),四种状态都有键。""" counts = dict.fromkeys(ITEM_STATUSES, 0) @@ -2508,8 +2528,9 @@ __all__ = [ "fk_relations", "frozen_catalog", "infer_name_relations", "initial_status", "is_date_column", "is_profile_note", "join_paths", "make_item", "merge_notes", "name_words", "now_iso", "parse_path", "plan_patch", "profile_relation_holds", - "query_snapshot_meta", "read_catalog", "read_entry", "relation_graph", "relation_id", "render_table_index", - "render_table_notes", "resolve_draft_model", "resolve_table_name", "review_entry", "review_item", "same_notes", + "query_snapshot_meta", "read_catalog", "read_catalogs", "read_entry", "relation_graph", "relation_id", "render_table_index", + "render_table_notes", "resolve_draft_model", "resolve_table_name", "restrict_notes", "review_entry", "review_item", + "same_notes", "same_profile_note", "save_human_edit", "save_patch", "status_counts", "structure_relation_ids", "system_notes_source", "table_usage", "unreviewed_status", "validate_notes", "visible_notes", "write_entry", ] diff --git a/backend/app/data/catalog_view.py b/backend/app/data/catalog_view.py new file mode 100644 index 00000000..5af6e2f9 --- /dev/null +++ b/backend/app/data/catalog_view.py @@ -0,0 +1,181 @@ +"""目录视图:运行和助手读数据目录统一经这一个接缝(docs/design/next-round/01-效果度量与回归评测.md 做法 5,总方案 X7)。 + +**为什么要接缝。** 目录在四处被读:运行时的 db_schema / db_query 工具(目录同时冻结进表结构快照)、助手查看(look 表详情)、 +自查用的 SQL 检查器、双盲复算(交给盲算的说明、比较两条 SQL 用的中文名)。以前各读各的、没有统一的开关,「开目录和关 +目录」「只用已核实的项」「起草模型 A 的目录和 B 的」这些对照都做不了,评测期间目录被起草或剖析改了也不知道。现在四处都 +经这里取,一处定读哪一份;lint 测试(tests/test_catalog_view.py)不许这几个模块绕过它直接 read_catalog、直接查 CatalogNote。 + +**视图**(CatalogView): +- mode:live 当前目录(缺省,行为和以前一样);snapshot 一份冻结的目录快照(工件 id 在 snapshot,内容见 SNAPSHOT_KIND); + off 不给目录。 +- statuses:all 全部(驳回的照旧不给模型看);reviewed 只要已核实(verified)和已确认(confirmed)的项,推断的 + (proposed)去掉——量的是模型起草、还没审过的那些项是帮忙还是添乱。 + +**视图从哪来。** 运行从 Run.options['catalog_view'] 读(launch_run 收 options,发起时定下、随 run.started 封存,审批恢复、 +重启续跑都沿用——用上下文变量传的话恢复之后就丢了);生成过程(问数据的 answer_ops、评测的作答)从参数读。都不给就是 live。 +冻结进表结构快照时,不是缺省视图的在工件引用的 meta 里记下视图(describe),证据面板据此写得出「本次运行目录:关」。 + +**快照的内容**(第 2 波的评测题集包写,这里只定读法):kind=eval_catalog_snapshot 的内容寻址工件, +{"sources": {源 id: {"tables": {表名: {"version": 版本, "notes": 目录}}}}}。快照里没有这个源就是这个源没有目录; +快照工件不在了照实报错(CatalogSnapshotMissing),不悄悄退回当前目录——那样对照就白做了。 +""" +from __future__ import annotations + +import logging +from collections.abc import Iterable, Mapping +from dataclasses import dataclass +from typing import Any + +from app.data import catalog +from app.data.catalog import CatalogEntry + +logger = logging.getLogger(__name__) + +#: Run.options 里放视图的键 +OPTION_KEY = "catalog_view" +MODES = ("live", "snapshot", "off") +#: 各状态集合留下哪些项的状态(all 不筛:驳回的由各处照旧经 visible_notes 去掉) +STATUS_SETS: dict[str, tuple[str, ...] | None] = {"all": None, "reviewed": ("verified", "confirmed")} +#: 目录快照工件的 kind(细案 01 做法 6) +SNAPSHOT_KIND = "eval_catalog_snapshot" + + +class CatalogViewInvalid(ValueError): + """视图写得不对(服务端代码写错了才会有:客户端写不了 Run.options)。""" + + +class CatalogSnapshotMissing(RuntimeError): + """视图指定的目录快照工件不在了、或者内容认不出。""" + + +@dataclass(frozen=True) +class CatalogView: + mode: str = "live" + statuses: str = "all" + #: mode=snapshot 时快照工件的 id + snapshot: str | None = None + + def __post_init__(self) -> None: + if self.mode not in MODES: + raise CatalogViewInvalid(f"目录视图的 mode 只能是 {'、'.join(MODES)},当前为「{self.mode}」") + if self.statuses not in STATUS_SETS: + raise CatalogViewInvalid(f"目录视图的 statuses 只能是 {'、'.join(STATUS_SETS)},当前为「{self.statuses}」") + if (self.mode == "snapshot") != bool(self.snapshot): + raise CatalogViewInvalid("目录视图只有 mode=snapshot 时带快照工件 id,而且必须带") + + @property + def is_default(self) -> bool: + return self.mode == "live" and self.statuses == "all" + + def as_option(self) -> dict[str, Any]: + """写进 Run.options['catalog_view'] 的形状。""" + return {"mode": self.mode, "statuses": self.statuses, **({"snapshot": self.snapshot} if self.snapshot else {})} + + @classmethod + def parse(cls, raw: Any) -> CatalogView: + """Run.options['catalog_view']、生成参数 → 视图。None 是缺省(live);写得不对抛 CatalogViewInvalid。""" + if raw is None: + return LIVE + if isinstance(raw, CatalogView): + return raw + if not isinstance(raw, Mapping): + raise CatalogViewInvalid(f"目录视图需要是一个对象,当前为「{raw}」") + unknown = set(raw) - {"mode", "statuses", "snapshot"} + if unknown: + raise CatalogViewInvalid(f"目录视图里不认识的键:{'、'.join(sorted(unknown))}") + return cls(mode=str(raw.get("mode") or "live"), statuses=str(raw.get("statuses") or "all"), + snapshot=str(raw["snapshot"]) if raw.get("snapshot") else None) + + +LIVE = CatalogView() + + +def of_options(options: Mapping[str, Any] | None) -> CatalogView: + """一次运行的视图(Run.options 里的 catalog_view)。没有就是 live。""" + return CatalogView.parse((options or {}).get(OPTION_KEY)) + + +def of_run(run_ctx: Any) -> CatalogView: + """运行里的节点构造 ToolContext 时用:RunContext.extra['run_options'] 里的视图(runner 每段执行都放这份)。""" + from app.engine.runner import RUN_OPTIONS_KEY + + return of_options((getattr(run_ctx, "extra", None) or {}).get(RUN_OPTIONS_KEY)) + + +def describe(view: CatalogView | None) -> dict[str, Any] | None: + """记进冻结快照 meta 的说法:缺省视图(live、全部)不记,别的照 as_option 记下。""" + view = view or LIVE + return None if view.is_default else view.as_option() + + +def restrict(view: CatalogView | None, notes: Mapping[str, Any] | None) -> dict[str, Any]: + """一张表的目录在这个视图下留下什么(状态集合;驳回的项原样留着,见 catalog.restrict_notes)。""" + allowed = STATUS_SETS[(view or LIVE).statuses] + if allowed is None: + return dict(notes or {}) + return catalog.restrict_notes(notes, allowed) + + +def _restricted(view: CatalogView, entries: Mapping[str, CatalogEntry]) -> dict[str, CatalogEntry]: + if STATUS_SETS[view.statuses] is None: + return dict(entries) + out: dict[str, CatalogEntry] = {} + for table, entry in entries.items(): + if notes := restrict(view, entry.notes): + out[table] = CatalogEntry(table=entry.table, notes=notes, version=entry.version, + updated_by=entry.updated_by, updated_at=entry.updated_at) + return out + + +def _snapshot_tables(view: CatalogView, source_id: str) -> dict[str, CatalogEntry]: + from app.core.artifact_store import load + + try: + content = load(view.snapshot or "") + except ValueError as e: # 内容和哈希对不上 + raise CatalogSnapshotMissing(f"目录快照 {str(view.snapshot)[:12]}… 读不出:{e}") from e + if not isinstance(content, dict) or not isinstance(content.get("sources"), dict): + raise CatalogSnapshotMissing(f"目录快照 {str(view.snapshot)[:12]}… 不在了或内容认不出,这次不能按它读目录") + tables = ((content["sources"].get(source_id) or {}).get("tables")) or {} + out: dict[str, CatalogEntry] = {} + for table, body in tables.items(): + if isinstance(body, Mapping) and isinstance(body.get("notes"), Mapping): + out[str(table)] = CatalogEntry(table=str(table), notes=dict(body["notes"]), + version=int(body.get("version") or 0)) + return out + + +async def read_many(session: Any, view: CatalogView | None, source_ids: Iterable[str]) -> dict[str, dict[str, CatalogEntry]]: + """几个数据源在这个视图下的目录:{源 id: {表名: CatalogEntry}},没有目录的源不出现。live 一次查询读出。""" + view = view or LIVE + ids = [i for i in dict.fromkeys(source_ids) if i] + if view.mode == "off" or not ids: + return {} + if view.mode == "snapshot": + raw = {sid: found for sid in ids if (found := _snapshot_tables(view, sid))} + else: + raw = await catalog.read_catalogs(session, ids) + return {sid: kept for sid, entries in raw.items() if (kept := _restricted(view, entries))} + + +async def read(view: CatalogView | None, source: Any) -> dict[str, CatalogEntry]: + """一个数据源在这个视图下的目录:{表名: CatalogEntry}。用自己的短会话读(调用方的会话里已经加载的对象不会因为回滚 + 过期)。live 读不到(库出错)当没有目录、记日志:目录是附加说明,不能挡住查询;快照不在了照实抛 CatalogSnapshotMissing。""" + from app.db.base import SessionLocal + + view = view or LIVE + source_id = getattr(source, "id", None) + if view.mode == "off" or not source_id: + return {} + if view.mode == "snapshot": + return _restricted(view, _snapshot_tables(view, source_id)) + try: + async with SessionLocal() as own: + return (await read_many(own, view, [source_id])).get(source_id, {}) + except Exception: # noqa: BLE001 + logger.exception("读取数据源 %s 的数据目录失败,本次不附目录", getattr(source, "name", source_id)) + return {} + + +__all__ = ["CatalogSnapshotMissing", "CatalogView", "CatalogViewInvalid", "LIVE", "MODES", "OPTION_KEY", "SNAPSHOT_KIND", + "STATUS_SETS", "describe", "of_options", "of_run", "read", "read_many", "restrict"] diff --git a/backend/app/data/sqlcheck.py b/backend/app/data/sqlcheck.py index 2a300acd..eface254 100644 --- a/backend/app/data/sqlcheck.py +++ b/backend/app/data/sqlcheck.py @@ -1084,36 +1084,36 @@ def graph_checks(nodes: Iterable[Any], checkers: Mapping[str, SqlChecker], *, return out -async def load_checkers(session: Any, sources: Iterable[Any]) -> dict[str, SqlChecker]: +async def load_checkers(session: Any, sources: Iterable[Any], *, catalog_view: Any = None) -> dict[str, SqlChecker]: """给一批数据源(DataSource 行)各建一个检查器:{源名: SqlChecker}。没有目录的源不建。 - 一次查询读出这些源的全部目录。读不到就当没有目录(返回空 dict):检查是附加的,不能挡住自查和发布本身。 + 一次查询读出这些源的全部目录,读哪一份由 catalog_view 定(data/catalog_view.py:当前目录、目录快照、关;只用已核实 + 的项),不给是当前目录。读不到当前目录就当没有目录(返回空 dict):检查是附加的,不能挡住自查和发布本身;视图指定的 + 目录快照不在了照实报错。 """ - from sqlalchemy import select - - from app.db.models import CatalogNote + from app.data.catalog_view import CatalogSnapshotMissing, CatalogView, read_many rows = [s for s in sources if getattr(s, "id", None) and dialect_of(getattr(s, "kind", None))] if not rows: return {} try: - notes = (await session.execute( - select(CatalogNote.source_id, CatalogNote.table_name, CatalogNote.notes) - .where(CatalogNote.source_id.in_([r.id for r in rows])) - )).all() + entries = await read_many(session, CatalogView.parse(catalog_view), [r.id for r in rows]) + except CatalogSnapshotMissing: + raise except Exception: # noqa: BLE001 logger.exception("读取数据目录失败,本次不做基于目录的 SQL 检查") return {} by_source: dict[str, dict[str, Any]] = {} - for source_id, table, value in notes: - if isinstance(value, dict) and value: - by_source.setdefault(source_id, {})[table] = value + for source_id, tables in entries.items(): + for table, entry in tables.items(): + if isinstance(entry.notes, dict) and entry.notes: + by_source.setdefault(source_id, {})[table] = entry.notes return {r.name: SqlChecker(kind=r.kind, schema_cache=r.schema_cache, notes=by_source[r.id], source_name=r.name) for r in rows if r.id in by_source} async def load_checkers_by_name(session: Any, names: Iterable[str]) -> dict[str, SqlChecker]: - """按数据源名建检查器(只看启用着的源)。发布门禁用:图里写的是源名。""" + """按数据源名建检查器(只看启用着的源)。发布门禁用:图里写的是源名,读的是当前目录。""" from sqlalchemy import select from app.db.models import DataSource diff --git a/backend/app/engine/nodes/challenge.py b/backend/app/engine/nodes/challenge.py index 88753e2b..4b8f21ce 100644 --- a/backend/app/engine/nodes/challenge.py +++ b/backend/app/engine/nodes/challenge.py @@ -67,6 +67,7 @@ from app.core.errors import describe_exception from app.core.events import EventType from app.db.base import SessionLocal from app.db.models import DataSource +from app.data import catalog_view from app.engine import deadline, verify_agent from app.engine import judge as judging from app.engine.context import NodeContext, NodeError @@ -767,7 +768,8 @@ class _Review: spec=self.spec, system=CHALLENGE_SYSTEM, task=task_text, sources=live, output_schema=CHALLENGE_SCHEMA, max_queries=self.max_queries, timeout_s=allot.timeout_s, daily_max_usd=self.daily_max_usd, emit=deadline.relabel(ctx.emit, allot), run_id=ctx.run.run_id, - node_id=ctx.node.id, data_versions=run_versions(ctx.run), masked_values=known) + node_id=ctx.node.id, data_versions=run_versions(ctx.run), masked_values=known, + catalog_view=catalog_view.of_run(ctx.run)) outcome = await asyncio.to_thread(_masked_rationale, outcome, known, given=[cand.text], kinds=self.kinds, masks=self.masks) if outcome.error and not outcome.model: diff --git a/backend/app/engine/nodes/dual_check.py b/backend/app/engine/nodes/dual_check.py index 0152179c..6790cb60 100644 --- a/backend/app/engine/nodes/dual_check.py +++ b/backend/app/engine/nodes/dual_check.py @@ -68,6 +68,7 @@ from app.core.errors import describe_exception from app.core.events import EventType from app.db.base import SessionLocal from app.db.models import DataSource +from app.data import catalog_view from app.engine import deadline, recompute from app.engine.approval import read_decision from app.engine.context import NodeContext, NodeError @@ -346,9 +347,11 @@ def _skipped(ctx: NodeContext, reason: str) -> dict[str, Any]: class _Sources: - """一次复算里要读的查询快照、数据源和快照要遮的列:同一份只读一次、只算一次。""" + """一次复算里要读的查询快照、数据源和快照要遮的列:同一份只读一次、只算一次。view 是这次运行的目录视图 + (data/catalog_view.py):交给盲算的目录、比较两条 SQL 用的中文名都按它读,和运行时工具给模型看的是同一份。""" - def __init__(self) -> None: + def __init__(self, view: Any = None) -> None: + self.view = catalog_view.CatalogView.parse(view) self.snapshots: dict[str, tuple[Any, str | None]] = {} self.rows: dict[str, DataSource | None] = {} self.masks: dict[tuple[str, str | None], frozenset[str]] = {} @@ -782,14 +785,14 @@ async def _constant_columns(snapshot: Mapping[str, Any], sources: _Sources) -> f async def _catalog_text(sources: _Sources, name: str) -> str: """交给盲算 agent 的数据目录:表清单(表名、中文名、粒度)和有目录的表的业务说明(业务日期、有效记录条件、去重、 - 字段含义、码值、关联关系)。太长的截断,其余请它用 db_schema 查。""" + 字段含义、码值、关联关系)。太长的截断,其余请它用 db_schema 查。读哪一份由这次运行的目录视图定(sources.view), + 和运行时工具给模型看的是同一份。""" from app.data import catalog row = await sources.source(name) if row is None: return BLIND_NO_CATALOG_PROMPT - async with SessionLocal() as session: - entries = await catalog.read_catalog(session, row.id) + entries = await catalog_view.read(sources.view, row) cache = row.schema_cache if isinstance(row.schema_cache, dict) else {} tables = cache.get("tables") or {} notes = {table: entry.notes for table, entry in entries.items()} @@ -843,7 +846,7 @@ async def _blind(ctx: NodeContext, jobs: dict[str, dict[str, Any]], options: _Op return {}, None, False, usage async with SessionLocal() as session: spec = await verify_model_spec(session, ctx.config, settings=dict(settings)) - sources = _Sources() + sources = _Sources(catalog_view.of_run(ctx.run)) catalogs: dict[str, str] = {} for job in jobs.values(): if job["source"] not in catalogs: @@ -870,7 +873,7 @@ async def _blind(ctx: NodeContext, jobs: dict[str, dict[str, Any]], options: _Op spec=spec, system=BLIND_SYSTEM, task=_task_text(job, catalogs[job["source"]]), sources=[job["source"]], output_schema=BLIND_SCHEMA, max_queries=options.max_queries, timeout_s=allot.timeout_s, daily_max_usd=options.daily_max_usd, emit=deadline.relabel(ctx.emit, allot), run_id=ctx.run.run_id, - node_id=ctx.node.id, data_versions=run_versions(ctx.run)) + node_id=ctx.node.id, data_versions=run_versions(ctx.run), catalog_view=sources.view) keys = list(jobs) await one(keys[0], jobs[keys[0]]) @@ -1078,7 +1081,7 @@ async def _review(state: GraphState, ctx: NodeContext, checked: dict[str, Any], """ rows = [r for r in checked["rows"] if r["status"] == DIFFERENT] results = checked["results"] - sources = _Sources() + sources = _Sources(catalog_view.of_run(ctx.run)) infos: dict[str, dict[str, Any]] = {} metrics: list[dict[str, Any]] = [] summary: list[dict[str, Any]] = [] @@ -1179,15 +1182,15 @@ async def _source_info(sources: _Sources, name: str) -> dict[str, Any]: (uploaded:导入时日期一律存成文字)和它导入时认作日期的列(upload_dates,按当前版本的配方读,见 dual_check_facts.upload_dates)——业务日期据此认上传表格的日期列,认不出的原因照实说(_dated_problem)。两项都进 候选:审批卡的预告按发起审批时读的这一份;裁决后的真写按写入时的版本重读 upload_dates,和现读的表结构一起以写入时 - 为准(dual_check_facts.record_choices)。数据源没了就都不给,照样比原文。""" - from app.data import catalog + 为准(dual_check_facts.record_choices)。数据源没了就都不给,照样比原文。目录按这次运行的目录视图读(sources.view), + 比较两条 SQL 用到的中文名和运行时给模型看的是同一份。""" from app.data.sql_semantics import catalog_labels row = await sources.source(name) if row is None: return {"kind": None, "schema": None, "names": {}, "notes": {}, "uploaded": False, "upload_dates": {}} + entries = await catalog_view.read(sources.view, row) async with SessionLocal() as session: - entries = await catalog.read_catalog(session, row.id) dates = await upload_dates(session, row) notes = {table: entry.notes for table, entry in entries.items()} return {"kind": row.kind, "schema": row.schema_cache if isinstance(row.schema_cache, dict) else None, diff --git a/backend/app/engine/nodes/llm.py b/backend/app/engine/nodes/llm.py index 3c93e39f..bf737196 100644 --- a/backend/app/engine/nodes/llm.py +++ b/backend/app/engine/nodes/llm.py @@ -48,6 +48,7 @@ from app.providers.factory import ( get_chat_model, ) from app.engine.sql_problems import event_checks +from app.data import catalog_view from app.tools.datasource import QUERY_PREFIX, run_masks_results, run_reads_only, run_scope_fixed, run_versions from app.tools.registry import ( ToolArgsError, @@ -341,6 +342,8 @@ async def _resolve_tools(ctx: NodeContext, state: GraphState) -> list[BaseTool]: mask_results=run_masks_results(ctx.run), force_readonly=run_reads_only(ctx.run), fixed_scope=fixed, + # 读哪一份数据目录:发起时定下的(data/catalog_view.py),审批恢复、续跑都沿用 + catalog_view=catalog_view.of_run(ctx.run), # 数据源工具运行中途补固定版本时经它留一条事件(tools/datasource.py 的 _pin_late) emit=ctx.emit, ) diff --git a/backend/app/engine/nodes/multi.py b/backend/app/engine/nodes/multi.py index 9815407c..c83efada 100644 --- a/backend/app/engine/nodes/multi.py +++ b/backend/app/engine/nodes/multi.py @@ -28,6 +28,7 @@ from app.engine.toolcalls import ( from app.providers import catalog from app.providers.factory import ModelSpec, bind_tools_safely, get_chat_model from app.engine.sql_problems import event_checks +from app.data import catalog_view from app.tools.datasource import QUERY_PREFIX, run_masks_results, run_reads_only, run_scope_fixed, run_versions from app.tools.registry import ( ToolArgsError, @@ -317,6 +318,8 @@ async def run_supervisor(state: GraphState, ctx: NodeContext) -> dict[str, Any]: mask_results=run_masks_results(ctx.run), force_readonly=run_reads_only(ctx.run), fixed_scope=run_scope_fixed(ctx.run), + # 读哪一份数据目录:发起时定下的(data/catalog_view.py),审批恢复、续跑都沿用 + catalog_view=catalog_view.of_run(ctx.run), # 数据源工具运行中途补固定版本时经它留一条事件(tools/datasource.py 的 _pin_late) emit=ctx.emit, ) diff --git a/backend/app/engine/nodes/tools.py b/backend/app/engine/nodes/tools.py index 1b26352d..ec256f30 100644 --- a/backend/app/engine/nodes/tools.py +++ b/backend/app/engine/nodes/tools.py @@ -20,6 +20,7 @@ from app.sandbox.base import SandboxLimits from app.sandbox.manager import sandbox_manager from app.engine.sql_problems import event_checks from app.capabilities.registry import is_platform_query, snapshot_owned +from app.data import catalog_view from app.tools.datasource import (DATA_UNAVAILABLE, SCHEMA_PREFIX, run_masks_results, run_reads_only, run_scope_fixed, run_versions) from app.tools.registry import ( @@ -54,6 +55,8 @@ def _tool_ctx(ctx: NodeContext) -> ToolContext: mask_results=run_masks_results(ctx.run), force_readonly=run_reads_only(ctx.run), fixed_scope=fixed, + # 读哪一份数据目录:发起时定下的(data/catalog_view.py),审批恢复、续跑都沿用 + catalog_view=catalog_view.of_run(ctx.run), # 数据源工具运行中途补固定版本时经它留一条事件(tools/datasource.py 的 _pin_late) emit=ctx.emit, ) diff --git a/backend/app/engine/verify_agent.py b/backend/app/engine/verify_agent.py index 2127610c..f44d5c03 100644 --- a/backend/app/engine/verify_agent.py +++ b/backend/app/engine/verify_agent.py @@ -758,6 +758,7 @@ async def run_verify_agent( node_id: str = "", data_versions: dict[str, Any] | Versions | None = Versions.UNSET, masked_values: Any = None, + catalog_view: Any = None, ) -> VerifyOutcome: """让一个只读 agent 查库,按 output_schema 交回有出处的结果。永远不抛。 @@ -775,6 +776,8 @@ async def run_verify_agent( - masked_values:调用方已经认出的遮罩值(engine/masking 的 _Secrets,审批质询、反方审稿收的报告快照里的值)。 查询工具从第一条查询起就按它们按值遮:agent 查到的备注、别的表里写着同一个号码时,模型看不到。随查询时遮掉的值 一起交回(VerifyOutcome.masked_values)。不改它。 + - catalog_view:db_schema 给 agent 看哪一份数据目录(data/catalog_view.py)。运行里的节点传这次运行的视图 + (catalog_view.of_run);不给是当前目录(质询接口)。 """ state = _State(started=time.monotonic(), seen=SeenValues( given=masked_values if masked_values is not None else _Secrets())) @@ -782,7 +785,7 @@ async def run_verify_agent( out = await _verify(state, spec=spec, system=system, task=task, sources=sources, output_schema=output_schema, max_queries=max_queries, timeout_s=timeout_s, max_cost_usd=max_cost_usd, daily_max_usd=daily_max_usd, max_steps=max_steps, emit=emit, - run_id=run_id, node_id=node_id, data_versions=data_versions) + run_id=run_id, node_id=node_id, data_versions=data_versions, catalog_view=catalog_view) except Exception as e: # noqa: BLE001 - 永远不抛:库写不进去、建工具出错这类意外也落在返回值里 logger.warning("核查出错", exc_info=True) out = state.outcome(error=f"核查出错({describe_exception(e)})") @@ -798,6 +801,7 @@ async def run_verify_agent( async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sources: Iterable[str], output_schema: dict[str, Any], max_queries: int | None, timeout_s: float | None, max_cost_usd: float | None, daily_max_usd: float | None | Default, max_steps: int | None, + catalog_view: Any = None, emit: Emit | None, run_id: str, node_id: str, data_versions: dict[str, Any] | Versions | None) -> VerifyOutcome: deadline = state.started + timeout_s if timeout_s is not None else None @@ -834,7 +838,7 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou # ---- 工具:只读的 db_schema / db_query,查询结果先遮再交给模型(遮掉的值并进 state.seen) wanted = list(dict.fromkeys(s for s in sources if isinstance(s, str) and s)) - ctx = ToolContext(run_id=run_id, node_id=node_id, data_versions=data_versions, emit=emit, + ctx = ToolContext(run_id=run_id, node_id=node_id, data_versions=data_versions, emit=emit, catalog_view=catalog_view, extra={MASKED_VALUES_KEY: state.seen}) async with SessionLocal() as session: tools = await build_datasource_tools([f"{p}{s}" for s in wanted for p in (SCHEMA_PREFIX, QUERY_PREFIX)], ctx, diff --git a/backend/app/look/tables.py b/backend/app/look/tables.py index 25fe1e5c..0d4e61d4 100644 --- a/backend/app/look/tables.py +++ b/backend/app/look/tables.py @@ -159,12 +159,16 @@ class TableKind: @classmethod async def load(cls, sources: list[Any], *, graph: Mapping[str, Any] | None = None, - picker: Picker | None = None) -> TableKind: - """读各源的数据目录。graph 是现有工作流(改图时的原图、追问时上一轮的图),概览里列出它查过的表。""" - from app.tools.datasource import _catalog_of + picker: Picker | None = None, catalog_view: Any = None) -> TableKind: + """读各源的数据目录。graph 是现有工作流(改图时的原图、追问时上一轮的图),概览里列出它查过的表。 + 读哪一份目录由 catalog_view 定(data/catalog_view.py:当前目录、目录快照、关;只用已核实的项),不给是当前目录。 + 查看结果里的目录和运行时 db_schema 给模型看的、自查用的是同一份——评测的「目录对照」量的才是整份贡献。""" + from app.data.catalog_view import CatalogView, read + + view = CatalogView.parse(catalog_view) seeds = graph_tables(graph, sources) - plans = [SourcePlan(source=s, entries=await _catalog_of(s) if introspect.table_names(s) else {}, + plans = [SourcePlan(source=s, entries=await read(view, s) if introspect.table_names(s) else {}, seeds=seeds.get(s.name, [])) for s in sources] return cls(plans, picker=picker) diff --git a/backend/app/tools/datasource.py b/backend/app/tools/datasource.py index 53e11ecf..d38ae483 100644 --- a/backend/app/tools/datasource.py +++ b/backend/app/tools/datasource.py @@ -277,7 +277,7 @@ async def build_datasource_tools( except table_versions.SnapshotMissing as e: tools.extend(_unavailable_tool(row, prefix, str(e)) for prefix in prefixes) continue - entries = await _catalog_of(row) + entries = await _catalog_of(row, ctx.catalog_view) for prefix in prefixes: if prefix == QUERY_PREFIX: tools.append(_make_query_tool(source, ctx, fixed=fixed, catalog=entries, force_readonly=force_readonly, @@ -287,20 +287,17 @@ async def build_datasource_tools( return tools -async def _catalog_of(row: DataSource) -> dict[str, data_catalog.CatalogEntry]: +async def _catalog_of(row: Any, view: Any = None) -> dict[str, data_catalog.CatalogEntry]: """这个源的数据目录,建工具时读一次:db_schema 给模型看的和查询时冻结进表结构快照的是同一版。 - 读不到就当没有目录,工具照常可用——目录是附加说明,不能挡住查询。用单独的会话读:出错要回滚, - 而回滚会让调用方会话里已经加载的对象(row)全部过期,之后再读属性就是异步环境里的懒加载。 + 读哪一份由目录视图定(view,data/catalog_view.py:当前目录、目录快照、关;只用已核实的项),不给就是当前目录。 + 读不到就当没有目录,工具照常可用——目录是附加说明,不能挡住查询;视图指定的目录快照不在了照实报错(不悄悄换成当前 + 目录)。用单独的会话读:出错要回滚,而回滚会让调用方会话里已经加载的对象(row)全部过期,之后再读属性就是异步环境里 + 的懒加载。 """ - from app.db.base import SessionLocal + from app.data import catalog_view - try: - async with SessionLocal() as own: - return await data_catalog.read_catalog(own, row.id) - except Exception: # noqa: BLE001 - logger.exception("读取数据源 %s 的数据目录失败,本次不附目录", row.name) - return {} + return await catalog_view.read(catalog_view.CatalogView.parse(view), row) # -------------------------------------------------------------------------- @@ -753,6 +750,12 @@ async def _store_schema(source: Any, ctx: ToolContext, *, if frozen := data_catalog.frozen_catalog(catalog or {}, tables): content[CATALOG_SNAPSHOT_KEY] = frozen meta["catalog_tables"] = len(frozen) + # 这次运行读的不是缺省的当前目录(评测的对照:关、目录快照、只用已核实的项):记在引用的 meta 里,证据面板写得出 + # 「本次运行目录:关」。不进内容哈希,缺省视图不记,快照和以前一字不差 + from app.data.catalog_view import CatalogView, describe + + if (view := describe(CatalogView.parse(ctx.catalog_view))) is not None: + meta["catalog_view"] = view try: return await put_json(content, kind=SCHEMA_SNAPSHOT, run_id=ctx.run_id or "", node_id=ctx.node_id or "", meta=meta) diff --git a/backend/app/tools/registry.py b/backend/app/tools/registry.py index c1215267..c80f0ed4 100644 --- a/backend/app/tools/registry.py +++ b/backend/app/tools/registry.py @@ -49,6 +49,9 @@ class ToolContext: #: 个人应用的刷新、个人蓝图的试运行)。记忆、知识库的内容不经遮罩:kb_search 的 collection 参数(调用工具节点写的、 #: 参数名纠错改出来的、图级 defaults 补上的、模型挑的)一律拒,只能检索 collection 这一个 fixed_scope: bool = False + #: 读哪一份数据目录(data/catalog_view.CatalogView):运行里由节点从 Run.options 取来(catalog_view.of_run),db_schema + #: 给模型看的、查询冻结进表结构快照的、查询时的 SQL 检查都按它。None 是当前目录(缺省,行为同以前) + catalog_view: Any = None extra: dict[str, Any] = field(default_factory=dict) diff --git a/backend/tests/test_catalog_view.py b/backend/tests/test_catalog_view.py new file mode 100644 index 00000000..3edd71d2 --- /dev/null +++ b/backend/tests/test_catalog_view.py @@ -0,0 +1,270 @@ +"""目录视图接缝(data/catalog_view.py;细案 01 做法 5,总方案 X7)。 + +目录在四处被读:运行时的 db_schema / db_query(目录同时冻结进表结构快照)、助手查看的表详情、自查的 SQL 检查器、双盲复算。 +这里守:四种视图(当前目录、只用已核实和已确认的项、关、目录快照)在四处给出的是同一份;缺省是当前目录、行为不变;运行 +从 Run.options 读视图(随发起定下,审批恢复、续跑沿用),冻结快照的 meta 记下不是缺省的视图;lint:这几个消费模块不许 +绕过视图直接 read_catalog、直接查 CatalogNote。 + +库、表、列、目录内容都是虚构的。 +""" +from __future__ import annotations + +import asyncio +import re +import sqlite3 +import uuid +from pathlib import Path + +import pytest +from sqlalchemy import select + +from app.core.artifact_store import put_json +from app.data import catalog, catalog_view +from app.data.catalog import make_item +from app.data.catalog_view import CatalogSnapshotMissing, CatalogView, CatalogViewInvalid +from app.data.introspect import introspect +from app.db.base import SessionLocal +from app.db.models import Artifact, DataSource, Run +from app.engine.runner import run_manager +from tests.fixtures.sources import drop_source + +APP = Path(__file__).resolve().parents[1] / "app" + +#: 已确认的一项、推断的一项、驳回的一项:三种视图各留哪些一眼看得出 +NOTES = { + "label": make_item("门店订单", "human"), # confirmed + "grain": make_item("每笔订单一行", "llm"), # proposed + "description": make_item("旧说明", "llm", "rejected"), # rejected + "columns": {"amount": {"label": make_item("实收金额", "human")}, + "status": {"label": make_item("订单状态", "llm")}}, +} +SNAPSHOT_NOTES = {"label": make_item("快照里的订单", "human")} + +VIEWS = { + "live": CatalogView(), + "reviewed": CatalogView(statuses="reviewed"), + "off": CatalogView(mode="off"), +} + + +@pytest.fixture +async def shop(tmp_path): + path = tmp_path / "view_shop.db" + db = sqlite3.connect(path) + db.executescript("CREATE TABLE orders (id INTEGER PRIMARY KEY, amount REAL NOT NULL, status INTEGER);" + "INSERT INTO orders VALUES (1, 30.0, 1), (2, 20.0, 9);") + db.commit() + db.close() + async with SessionLocal() as session: + row = DataSource(name=f"viewshop_{uuid.uuid4().hex[:6]}", kind="sqlite", database=str(path)) + session.add(row) + await session.commit() + row.schema_cache = await introspect(row) + await session.commit() + await session.refresh(row) + await catalog.write_entry(session, row.id, "orders", NOTES, if_version=0, actor=None) + session.expunge(row) + yield row + await drop_source(row.id) + + +async def _snapshot_view(source) -> CatalogView: + sid = await put_json({"sources": {source.id: {"tables": {"orders": {"version": 7, "notes": SNAPSHOT_NOTES}}}}}, + kind=catalog_view.SNAPSHOT_KIND) + return CatalogView(mode="snapshot", snapshot=sid) + + +def _says(text: str) -> dict[str, bool]: + return {word: word in text for word in ("门店订单", "每笔订单一行", "旧说明", "实收金额", "订单状态", "快照里的订单")} + + +EXPECT = { + "live": {"门店订单": True, "每笔订单一行": True, "旧说明": False, "实收金额": True, "订单状态": True, "快照里的订单": False}, + "reviewed": {"门店订单": True, "每笔订单一行": False, "旧说明": False, "实收金额": True, "订单状态": False, + "快照里的订单": False}, + "off": dict.fromkeys(("门店订单", "每笔订单一行", "旧说明", "实收金额", "订单状态", "快照里的订单"), False), + "snapshot": {"门店订单": False, "每笔订单一行": False, "旧说明": False, "实收金额": False, "订单状态": False, + "快照里的订单": True}, +} + + +async def _all_views(source) -> dict[str, CatalogView]: + return {**VIEWS, "snapshot": await _snapshot_view(source)} + + +# -------------------------------------------------------------------------- +# 视图本身 +# -------------------------------------------------------------------------- + + +def test_parse_and_defaults(): + assert CatalogView.parse(None) is catalog_view.LIVE and catalog_view.LIVE.is_default + assert CatalogView.parse({"mode": "off"}) == CatalogView(mode="off") + assert catalog_view.describe(None) is None and catalog_view.describe(CatalogView()) is None + assert catalog_view.describe(CatalogView(mode="off")) == {"mode": "off", "statuses": "all"} + assert catalog_view.of_options({"catalog_view": {"statuses": "reviewed"}}) == CatalogView(statuses="reviewed") + for bad in ({"mode": "nope"}, {"statuses": "some"}, {"mode": "snapshot"}, {"mode": "live", "snapshot": "x"}, + {"extra": 1}, "off"): + with pytest.raises(CatalogViewInvalid): + CatalogView.parse(bad) + + +async def test_a_missing_snapshot_is_an_error_not_the_live_catalog(shop): + with pytest.raises(CatalogSnapshotMissing): + await catalog_view.read(CatalogView(mode="snapshot", snapshot="0" * 64), shop) + + +# -------------------------------------------------------------------------- +# 四处一致 +# -------------------------------------------------------------------------- + + +async def test_runtime_db_schema_follows_the_view(shop): + from app.tools.datasource import SCHEMA_PREFIX, build_datasource_tools + from app.tools.registry import ToolContext, Versions + + for name, view in (await _all_views(shop)).items(): + ctx = ToolContext(run_id="", node_id="", data_versions=Versions.CURRENT, catalog_view=view) + async with SessionLocal() as session: + [tool] = await build_datasource_tools([f"{SCHEMA_PREFIX}{shop.name}"], ctx, session) + assert _says(await tool.coroutine(table="orders")) == EXPECT[name], name + + +async def test_look_table_detail_follows_the_view(shop): + from app.look.service import LookService + from app.look.tables import TableKind + + for name, view in (await _all_views(shop)).items(): + looks = LookService([await TableKind.load([shop], catalog_view=view)]) + got = await looks.look({"op": "look", "need": "订单", "targets": [{"kind": "table", "name": "orders"}]}) + assert _says(got.text) == EXPECT[name], name + + +async def test_sql_checkers_follow_the_view(shop): + from app.data.sqlcheck import load_checkers + + for name, view in (await _all_views(shop)).items(): + async with SessionLocal() as session: + checkers = await load_checkers(session, [shop], catalog_view=view) + if name == "off": + assert checkers == {} + continue + notes = checkers[shop.name]._raw["orders"] + assert _says(str(notes)) | {"旧说明": False} == EXPECT[name] | {"旧说明": False}, name + if name == "reviewed": + # 驳回的项原样留着交给检查器(它自己按驳回处理),推断的去掉 + assert notes["description"]["status"] == "rejected" and "grain" not in notes + + +async def test_dual_check_blind_catalog_follows_the_view(shop): + from app.engine.nodes.dual_check import BLIND_NO_CATALOG_PROMPT, _catalog_text, _source_info, _Sources + + for name, view in (await _all_views(shop)).items(): + sources = _Sources(view) + text = await _catalog_text(sources, shop.name) + assert _says(text) == EXPECT[name], name + info = await _source_info(sources, shop.name) + assert ("orders" in info["notes"]) == (name != "off"), name + if name == "off": + # 关目录:表清单(表结构里的事实)照给,目录的说明一句都没有 + assert "orders" in text and "数据目录(" not in text and BLIND_NO_CATALOG_PROMPT not in text + + +async def test_the_default_is_the_live_catalog_everywhere(shop): + """不给视图:和以前一样读当前目录。""" + from app.engine.nodes.dual_check import _catalog_text, _Sources + from app.look.service import LookService + from app.look.tables import TableKind + from app.tools.datasource import _catalog_of + + assert _says(str({t: e.notes for t, e in (await _catalog_of(shop)).items()}))["门店订单"] + looks = LookService([await TableKind.load([shop])]) + got = await looks.look({"op": "look", "need": "订单", "targets": [{"kind": "table", "name": "orders"}]}) + assert _says(got.text) == EXPECT["live"] + assert _says(await _catalog_text(_Sources(), shop.name)) == EXPECT["live"] + + +# -------------------------------------------------------------------------- +# 运行:从 Run.options 读视图,冻结快照记下 +# -------------------------------------------------------------------------- + + +def _graph(source: str) -> dict: + def node(nid, ntype, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}} + + return {"nodes": [node("start", "input"), + node("q", "tool", tool=f"db_query__{source}", args={"sql": "SELECT SUM(amount) AS 金额 FROM orders"}, + assign_to="rows"), + node("out", "output", fields=[{"name": "rows", "value": "{{ vars.rows }}"}])], + "edges": [{"source": "start", "target": "q"}, {"source": "q", "target": "out"}]} + + +async def _frozen(options: dict | None, source) -> tuple[dict, dict]: + from app.core.artifact_store import load + from app.engine.evidence import SCHEMA_SNAPSHOT + from app.engine.launch import launch_run + + async with SessionLocal() as session: + run = await launch_run(session, graph=_graph(source.name), input_payload={}, actor="测试", + origin_kind="canvas", options=options) + run_id = run.id + for _ in range(200): + async with SessionLocal() as session: + status = (await session.get(Run, run_id)).status + if status in ("succeeded", "failed", "cancelled"): + break + await asyncio.sleep(0.05) + assert status == "succeeded", status + async with SessionLocal() as session: + ref = (await session.execute(select(Artifact).where( + Artifact.run_id == run_id, Artifact.kind == SCHEMA_SNAPSHOT))).scalars().first() + assert ref is not None + return load(ref.id), ref.meta or {} + + +async def test_a_run_reads_its_view_from_options_and_the_freeze_says_so(shop): + await run_manager.setup() + try: + content, meta = await _frozen(None, shop) + assert "orders" in content.get("catalog", {}), "缺省:当前目录照旧冻结" + assert "catalog_view" not in meta, "缺省视图不记,meta 和以前一样" + + content, meta = await _frozen({"catalog_view": {"mode": "off"}}, shop) + assert "catalog" not in content, "关目录:快照里没有目录" + assert meta["catalog_view"] == {"mode": "off", "statuses": "all"} + + content, meta = await _frozen({"catalog_view": {"statuses": "reviewed"}}, shop) + frozen = content["catalog"]["orders"]["notes"] + assert "grain" not in frozen and frozen["label"]["value"] == "门店订单" + assert meta["catalog_view"] == {"mode": "live", "statuses": "reviewed"} + finally: + await run_manager.shutdown() + + +def test_of_run_reads_the_run_options(): + from types import SimpleNamespace + + from app.engine.runner import RUN_OPTIONS_KEY + + run = SimpleNamespace(extra={RUN_OPTIONS_KEY: {"catalog_view": {"mode": "off"}}}) + assert catalog_view.of_run(run) == CatalogView(mode="off") + assert catalog_view.of_run(SimpleNamespace(extra={})) == catalog_view.LIVE + + +# -------------------------------------------------------------------------- +# lint:消费模块只经视图读目录 +# -------------------------------------------------------------------------- + +#: 给模型、给自查读目录的模块(四处消费 + 它们的帮手)。要读目录一律经 data/catalog_view.py +CONSUMERS = ("tools/datasource.py", "look/tables.py", "data/sqlcheck.py", "engine/nodes/dual_check.py", + "engine/verify_agent.py", "api/copilot_context.py") +FORBIDDEN = re.compile(r"\bread_catalogs?\s*\(|\bread_entry\s*\(|\bCatalogNote\b") + + +@pytest.mark.parametrize("module", CONSUMERS) +def test_consumers_never_read_the_catalog_directly(module): + text = (APP / module).read_text(encoding="utf-8") + code = "\n".join(line for line in text.splitlines() if not line.lstrip().startswith("#")) + hits = [m.group(0) for m in FORBIDDEN.finditer(code)] + assert not hits, f"{module} 绕过了目录视图:{hits}" From 2fb16c271943d6759917d54bf7385eddbddcf7e8 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Fri, 16 Oct 2026 10:31:26 +0800 Subject: [PATCH 14/39] =?UTF-8?q?=E8=AE=BE=E7=BD=AE=E9=A1=B5=E5=8A=A0?= =?UTF-8?q?=E3=80=8C=E9=80=9A=E7=94=A8=E3=80=8D=E6=A0=87=E7=AD=BE=E6=94=BE?= =?UTF-8?q?=E5=AF=B9=E5=A4=96=E8=AE=BF=E9=97=AE=E5=9C=B0=E5=9D=80=EF=BC=9B?= =?UTF-8?q?=E7=8E=AF=E5=A2=83=E5=8F=98=E9=87=8F=E6=8C=87=E5=AE=9A=E6=97=B6?= =?UTF-8?q?=E6=98=BE=E7=A4=BA=E3=80=8C=E7=94=B1=E7=8E=AF=E5=A2=83=E5=8F=98?= =?UTF-8?q?=E9=87=8F=E6=8C=87=E5=AE=9A=E3=80=8D=E4=B8=8D=E5=8F=AF=E6=94=B9?= =?UTF-8?q?=EF=BC=8C=E5=AE=9A=E6=97=B6=E4=BB=BB=E5=8A=A1=E9=87=8C=E7=9A=84?= =?UTF-8?q?=E5=85=A5=E5=8F=A3=E8=B7=9F=E7=9D=80=E8=AE=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 服务端已经只认一个对外访问地址(core/public_url.py),界面上这一项以前只在定时任务的 Webhook 一节里改得到,叫「服务 访问地址」,和设置接口、通知里的「对外访问地址」两种说法;环境变量设了时那里照样能改,改了也不生效。 - 新增「设置 → 通用」(pages/settings/GeneralTab.tsx):写明它管哪几件事(通知和 Webhook 的链接、单点登录的回调、来源 核对);管理员能改,保存前先重读 notify 一组、只换这一项,末尾的 / 去掉,带 ? 或 # 的当场标错;改了没保存离开先问; 环境变量指定了的只显示它、写明「由环境变量 AGENTLAB_PUBLIC_URL 指定」,不给输入框;搭建者只看。使用者看不到这一页 - 定时任务 Webhook 一节的快捷入口改叫「对外访问地址」,按 GET /api/settings 的 public_url 一组认:环境变量指定的只显示、 不给修改按钮,设置里存着的旧值不再冒充生效的那个;地址校验和「通用」同一个(lib/publicUrl.ts) - 文案进 lib/terms.ts(GENERAL_SETTING_TEXT,SCHEDULE_TEXT 的 linkBase 一组改说法) 检查:check-manage 加「设置 · 通用:对外访问地址」一节(标签、说明、校验、离开先问、只写 notify 一组、环境变量只读、 搭建者只读);check-schedules 加「对外访问地址由环境变量指定」一节。check-manage、check-schedules 在隔离的一套后端 + vite 上全部通过,tsc、check-copy 通过。 --- frontend/src/lib/publicUrl.ts | 31 +++++++ frontend/src/lib/terms.ts | 42 +++++++-- frontend/src/pages/SettingsPage.tsx | 7 +- .../src/pages/schedules/WebhookFields.tsx | 26 ++++-- frontend/src/pages/schedules/model.ts | 2 +- frontend/src/pages/settings/GeneralTab.tsx | 93 +++++++++++++++++++ scripts/check-manage.mjs | 71 ++++++++++++++ scripts/check-schedules.mjs | 23 ++++- 8 files changed, 275 insertions(+), 20 deletions(-) create mode 100644 frontend/src/lib/publicUrl.ts create mode 100644 frontend/src/pages/settings/GeneralTab.tsx diff --git a/frontend/src/lib/publicUrl.ts b/frontend/src/lib/publicUrl.ts new file mode 100644 index 00000000..781500a9 --- /dev/null +++ b/frontend/src/lib/publicUrl.ts @@ -0,0 +1,31 @@ +/** + * 对外访问地址:别人打开 AgentLab 用的那个地址。服务端全站只认一个(backend/app/core/public_url.py):环境变量 + * AGENTLAB_PUBLIC_URL 优先,其次设置里 notify 一组的 public_base_url(界面上在「设置 → 通用」改,定时任务的 Webhook 一节 + * 有快捷入口)。通知和 Webhook 里的链接、单点登录的回调、来源核对都按它。 + * + * GET /api/settings 另给只读的 public_url 一组 {value, env_locked}:环境变量指定了的话显示它、写明「由环境变量指定」、 + * 不让改(改了服务端回 409)。老服务端没有这一组时按设置里存的那一项、可以改。 + */ + +export interface PublicAddress { + /** 生效的地址;null 是没设置(通知里不附链接) */ + value: string | null + /** 环境变量指定的:设置里的那一项不生效、不能改 */ + env: boolean +} + +export function publicAddressOf(settings: Record | null | undefined): PublicAddress { + const env = settings?.public_url?.env_locked === true + const raw = env ? settings?.public_url?.value : settings?.notify?.public_base_url + return { value: typeof raw === 'string' && raw ? raw : null, env } +} + +/** 填的地址去掉首尾空白和末尾的 /(服务端存的也是这样) */ +export function cleanPublicUrl(text: string): string { + return text.trim().replace(/\/+$/, '') +} + +/** http:// 或 https:// 开头、有主机名、中间没有空白、不带 ? 或 # 之后的部分(同服务端 public_url.url_problem) */ +export function isPublicUrl(text: string): boolean { + return /^https?:\/\/[^\s/?#]+(?:\/[^\s?#]*)?$/i.test(text.trim()) +} diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index 51fb6dba..0f0a6dad 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -3889,19 +3889,20 @@ export const SCHEDULE_TEXT = { testResult: (head: string, status: number | null, detail: string) => `${head}${status != null ? `(HTTP ${status})` : ''}${detail ? `:${detail}` : ''}`, - // ---- 通知里的链接:运行、对比页都按这个地址拼(设置 notify 组,所有定时任务共用) + // ---- 通知里的链接:运行、对比页都按全站的对外访问地址拼(GENERAL_SETTING_TEXT,这里是它的快捷入口) linkBase: '通知中的链接', linkBaseValue: (url: string) => `按 ${url} 拼接`, - linkBaseUnset: '未设置服务访问地址,通知中不附带运行和对比页链接', + linkBaseUnset: '未设置对外访问地址,通知中不附带运行和对比页链接', linkBaseEdit: '修改', linkBaseSet: '设置', - linkBaseLabel: '服务访问地址', - linkBaseHint: '收件人打开运行、对比页链接用的地址,例如 https://agentlab.example.com。所有定时任务共用', + linkBaseLabel: '对外访问地址', + linkBaseHint: '收件人打开运行、对比页链接用的地址,例如 https://agentlab.example.com。全站共用一个,也可以在「设置 → 通用」中修改', linkBasePlaceholder: 'https://agentlab.example.com', - linkBaseInvalid: '请输入以 http:// 或 https:// 开头的地址', - linkBaseSaved: '已保存服务访问地址', - linkBaseCleared: '已清除服务访问地址', - linkBaseFailed: '无法读取服务访问地址', + linkBaseInvalid: '请输入以 http:// 或 https:// 开头的完整地址,不带 ? 或 # 之后的部分', + linkBaseSaved: '已保存对外访问地址', + linkBaseCleared: '已清除对外访问地址', + linkBaseFailed: '无法读取对外访问地址', + linkBaseEnv: '由环境变量 AGENTLAB_PUBLIC_URL 指定,不能在这里修改', // ---- 表单:保存 save: '保存', @@ -5825,6 +5826,31 @@ export const APP_SETTING_TEXT = { personalAppsHint: '开启后,使用者可以从说需求、场景模板或问数据的回答建一个只有自己看得见的应用:只读、结果遮罩,每人每天的刷新次数按上面的每日上限算;想给别人用要提交给搭建者审核发布。搭建者及以上不受这一项限制', } +/** + * 设置 → 通用:全站共用的部署设置。对外访问地址全站只有一个(服务端 core/public_url.py):环境变量 AGENTLAB_PUBLIC_URL + * 优先,其次这里填的;定时任务 Webhook 一节里是它的快捷入口(SCHEDULE_TEXT.linkBase*) + */ +export const GENERAL_SETTING_TEXT = { + tab: '通用', + publicUrl: '对外访问地址', + publicUrlHint: '别人打开 AgentLab 用的地址。通知和定时任务 Webhook 里的链接、单点登录的回调地址都按它拼,提交操作时也按它核对请求来源。留空时通知里不附链接,回调和来源核对按浏览器访问的地址认', + address: '地址', + addressHint: '以 http:// 或 https:// 开头的完整地址,例如 https://agentlab.example.com;放在反向代理的子路径下时带上路径', + placeholder: 'https://agentlab.example.com', + invalid: '请输入以 http:// 或 https:// 开头的完整地址,不带 ? 或 # 之后的部分', + unset: '未设置', + envLocked: '由环境变量 AGENTLAB_PUBLIC_URL 指定', + envLockedHint: '部署时用环境变量定了这个地址,这里不能修改。要改请修改部署配置后重启服务', + readOnly: '全站共用,只有管理员可以修改', + save: '保存', + saved: '已保存对外访问地址', + cleared: '已清除对外访问地址', + leaveTitle: '对外访问地址还没保存', + leaveBody: '离开此页面将丢失这次修改。', + leaveConfirm: '放弃修改', + leaveCancel: '返回保存', +} + /** 自定义工具绑定登记的端点(§3.9):只填路径,地址和认证头由端点提供 */ export const TOOL_ENDPOINT_TEXT = { bind: '端点', diff --git a/frontend/src/pages/SettingsPage.tsx b/frontend/src/pages/SettingsPage.tsx index 68314672..4eb66a44 100644 --- a/frontend/src/pages/SettingsPage.tsx +++ b/frontend/src/pages/SettingsPage.tsx @@ -25,6 +25,7 @@ import { localActor, setLocalActor } from '../lib/actor' import { AGENT_GUARD_TEXT, APP_SETTING_TEXT, DRAFT_MODEL_SETTING_TEXT, IDENTITY_TEXT, JUDGE_SETTING_TEXT, NOTIFY_TEXT, PROVIDER_REF_TEXT, SETTINGS_ROLE_TEXT, TOOL_GATE_TEXT, VERIFY_SETTING_TEXT, judgeByModelText, judgeUnlimitedText, + GENERAL_SETTING_TEXT, } from '../lib/terms' import { atLeast, loginOn, prefsLocalOnly, usePrincipal } from '../store/auth' import type { ThemePref } from '../lib/theme' @@ -32,6 +33,7 @@ import { ProviderOptions, VerifySection, verifyChanged, verifyDraftOf, verifyErrors, verifyLabels, verifyPayload, type VerifyDraft, } from './VerifySettings' import { DraftModelSection } from './DraftModelSettings' +import { GeneralTab } from './settings/GeneralTab' import { IdentityTab } from './settings/IdentityTab' import { NotifyTab } from './settings/NotifyTab' import { AuditTab } from './settings/AuditTab' @@ -41,6 +43,8 @@ import { AuditTab } from './settings/AuditTab' const TABS = [ { key: 'providers', label: '模型接入' }, { key: 'prefs', label: '偏好设置' }, + // 全站共用的部署设置:对外访问地址(环境变量指定时只读) + { key: 'general', label: GENERAL_SETTING_TEXT.tab }, { key: 'system', label: '运行环境' }, // 应用平台(A7):登录方式和人、通知渠道、管理类操作的留痕 { key: 'identity', label: IDENTITY_TEXT.tab }, @@ -74,12 +78,13 @@ export function SettingsPage() { return (
    } title="设置" - subtitle={builder ? '模型接入、偏好、运行环境、身份和通知' : SETTINGS_ROLE_TEXT.subtitleUser} /> + subtitle={builder ? '模型接入、偏好、对外访问地址、运行环境、身份和通知' : SETTINGS_ROLE_TEXT.subtitleUser} />
    {tab === 'providers' && } {tab === 'prefs' && (builder ? : )} + {tab === 'general' && } {tab === 'system' && } {tab === 'identity' && } {tab === 'notify' && } diff --git a/frontend/src/pages/schedules/WebhookFields.tsx b/frontend/src/pages/schedules/WebhookFields.tsx index 2041d5c9..ab27bcb4 100644 --- a/frontend/src/pages/schedules/WebhookFields.tsx +++ b/frontend/src/pages/schedules/WebhookFields.tsx @@ -4,12 +4,13 @@ import clsx from 'clsx' import { api } from '../../api/client' import { Field, Notice, Spinner, isComposing, toast, useRadioGroup } from '../../components/ui' import { errorMessage } from '../../lib/errors' +import { cleanPublicUrl, isPublicUrl, publicAddressOf } from '../../lib/publicUrl' import { SCHEDULE_TEXT as T } from '../../lib/terms' import type { ScheduleEvent, ScheduleWebhook, ScheduleWebhookTest } from '../../types' -import { SCHEDULE_EVENTS, WEBHOOK_FORMATS, isHttpUrl, webhookTestText, type FormField, type WebhookForm } from './model' +import { SCHEDULE_EVENTS, WEBHOOK_FORMATS, webhookTestText, type FormField, type WebhookForm } from './model' /** - * Webhook 通知:格式、地址、签名密钥、订阅事件、发送测试消息,外加通知里运行、对比页链接用的服务访问地址。 + * Webhook 通知:格式、地址、签名密钥、订阅事件、发送测试消息,外加通知里运行、对比页链接用的对外访问地址(快捷入口)。 * * 地址和密钥保存之后服务端不再返回明文:地址只显示主机和末 4 位(url_hint),可以替换或删除;密钥只说「已设置」, * 可以替换或清除。没点替换时请求里不带它们,改别的字段不会把地址清掉。测试消息用的是已保存的设置, @@ -195,11 +196,12 @@ function SecretField({ value: w, saved, onChange, error }: { } /** - * 通知里运行、对比页链接用的服务访问地址:设置里 notify 一组的 public_base_url,所有定时任务共用。这里给出当前值和修改入口, - * 写的时候先重读一遍 notify 组、只换这一项,不把同组别的设置冲掉 + * 通知里运行、对比页链接用的对外访问地址:全站只有一个(lib/publicUrl.ts,「设置 → 通用」),这里给出当前值和修改的快捷入口。 + * 环境变量指定了的只显示、写明由环境变量指定,不给修改。写的时候先重读一遍 notify 组、只换这一项,不把同组别的设置冲掉 */ function LinkBase() { const [value, setValue] = useState(undefined) + const [env, setEnv] = useState(false) const [failed, setFailed] = useState(false) const [editing, setEditing] = useState(false) const [draft, setDraft] = useState('') @@ -208,13 +210,18 @@ function LinkBase() { useEffect(() => { let alive = true api.settings.get() - .then((s) => { if (alive) setValue(typeof s?.notify?.public_base_url === 'string' ? s.notify.public_base_url : null) }) + .then((s) => { + if (!alive) return + const address = publicAddressOf(s) + setValue(address.value) + setEnv(address.env) + }) .catch(() => { if (alive) setFailed(true) }) return () => { alive = false } }, []) - const cleaned = draft.trim().replace(/\/+$/, '') - const invalid = !!cleaned && !isHttpUrl(cleaned) + const cleaned = cleanPublicUrl(draft) + const invalid = !!cleaned && !isPublicUrl(cleaned) const save = async () => { if (invalid || busy) return setBusy(true) @@ -222,7 +229,7 @@ function LinkBase() { const current = await api.settings.get() const notify = { ...(current?.notify ?? {}), public_base_url: cleaned || null } const out = await api.settings.put({ notify }) - setValue(typeof out?.notify?.public_base_url === 'string' ? out.notify.public_base_url : notify.public_base_url) + setValue(out?.notify ? publicAddressOf(out).value : notify.public_base_url) setFailed(false) setEditing(false) toast.ok(cleaned ? T.linkBaseSaved : T.linkBaseCleared) @@ -240,12 +247,13 @@ function LinkBase() { {failed ? T.linkBaseFailed : value === undefined ? : value ? T.linkBaseValue(value) : T.linkBaseUnset} - {!editing && value !== undefined && ( + {!editing && value !== undefined && !env && ( )}
    + {env &&

    {T.linkBaseEnv}

    } {editing && (
    diff --git a/frontend/src/pages/schedules/model.ts b/frontend/src/pages/schedules/model.ts index 30b3ab7d..b6a68b15 100644 --- a/frontend/src/pages/schedules/model.ts +++ b/frontend/src/pages/schedules/model.ts @@ -559,7 +559,7 @@ export function freqProblem(f: Freq): string | null { return null } -/** 以 http:// 或 https:// 开头、中间没有空白的地址(Webhook、服务访问地址都用它) */ +/** 以 http:// 或 https:// 开头、中间没有空白的地址(Webhook 地址用它;对外访问地址另有 lib/publicUrl 的 isPublicUrl) */ export function isHttpUrl(text: string): boolean { return /^https?:\/\/\S+$/i.test(text.trim()) } diff --git a/frontend/src/pages/settings/GeneralTab.tsx b/frontend/src/pages/settings/GeneralTab.tsx new file mode 100644 index 00000000..a29fd693 --- /dev/null +++ b/frontend/src/pages/settings/GeneralTab.tsx @@ -0,0 +1,93 @@ +import { useEffect, useState } from 'react' +import { Check, Lock } from 'lucide-react' +import { api } from '../../api/client' +import { ErrorState, Field, SectionBar, Skeleton, Spinner, confirmDialog, isComposing, toast } from '../../components/ui' +import { useLeaveGuard } from '../../lib/leave' +import { cleanPublicUrl, isPublicUrl, publicAddressOf } from '../../lib/publicUrl' +import { GENERAL_SETTING_TEXT as T } from '../../lib/terms' +import { useAtLeast } from '../../store/auth' + +/* + * 设置 → 通用:全站共用的部署设置。现在只有「对外访问地址」一项:服务端全站只认一个对外地址(core/public_url.py), + * 环境变量 AGENTLAB_PUBLIC_URL 优先,其次这里填的(存在 notify 一组)。以前两处各管各的,设置里这一项只在定时任务的 + * Webhook 一节里改得到,那里现在是它的快捷入口。 + * + * 环境变量设了:只显示它,写明「由环境变量指定」,不给输入框(GET /api/settings 的 public_url 一组)。只有管理员能改 + * (整个设置接口的改动都要管理员),搭建者只看。保存时先重读一遍 notify 组、只换这一项,不把同组别的设置冲掉 + */ +export function GeneralTab() { + const admin = useAtLeast('admin') + const [settings, setSettings] = useState | null>(null) + const [loadError, setLoadError] = useState(null) + // null:没在改,输入框显示存着的值 + const [draft, setDraft] = useState(null) + const [busy, setBusy] = useState(false) + + const load = () => api.settings.get().then((s) => { setSettings(s); setLoadError(null) }, setLoadError) + useEffect(() => { void load() }, []) + + const address = publicAddressOf(settings) + const editable = admin && !address.env + const saved = address.value ?? '' + const cleaned = cleanPublicUrl(draft ?? saved) + const invalid = !!cleaned && !isPublicUrl(cleaned) + const dirty = settings != null && editable && cleaned !== saved + useLeaveGuard(dirty, () => confirmDialog({ + title: T.leaveTitle, body: T.leaveBody, confirmLabel: T.leaveConfirm, cancelLabel: T.leaveCancel, danger: true, + })) + + const save = async () => { + if (!dirty || invalid || busy) return + setBusy(true) + try { + const current = await api.settings.get() + const out = await api.settings.put({ notify: { ...(current?.notify ?? {}), public_base_url: cleaned || null } }) + setSettings(out) + setDraft(null) + toast.ok(cleaned ? T.saved : T.cleared) + } catch (e) { + toast.error(e) + // 环境变量这期间被设上了(服务端回 409)这类:按服务端现在的样子重读 + void load() + } finally { + setBusy(false) + } + } + + return ( +
    +
    + + {!settings ? ( + loadError ? void load()} /> : + ) : editable ? ( + +
    + setDraft(e.target.value)} + onKeyDown={(e) => { if (e.key === 'Enter' && !isComposing(e)) { e.preventDefault(); void save() } }} + data-public-url-input="" /> + +
    +
    + ) : ( +
    +
    + {address.value ?? T.unset} +
    +

    + + {address.env ? `${T.envLocked}。${T.envLockedHint}` : T.readOnly} +

    +
    + )} +
    +
    + ) +} diff --git a/scripts/check-manage.mjs b/scripts/check-manage.mjs index d5546dbd..eb02c474 100644 --- a/scripts/check-manage.mjs +++ b/scripts/check-manage.mjs @@ -731,6 +731,77 @@ await section('设置 · 应用:使用者可以建个人应用(只有管理 await builder.close() }) +await section('设置 · 通用:对外访问地址', async () => { + // 全站只认一个对外访问地址(服务端 core/public_url.py):环境变量优先,其次这里填的(存在 notify 一组)。通知和 Webhook 的 + // 链接、单点登录的回调、来源核对都按它。管理员能改;环境变量指定了的只显示、写明来源;搭建者只看 + let stored = await get('/settings') + stored = { ...stored, notify: { ...(stored.notify ?? {}), public_base_url: null }, public_url: { value: null, env_locked: false } } + const { page, sent, errors, natives, close } = await open('/settings/general', { + handlers: [ + [/^GET \/settings$/, (route) => json(stored)(route)], + [/^PUT \/settings$/, (route, { body }) => { + stored = { ...stored, ...body.values, public_url: { value: body.values.notify?.public_base_url ?? null, env_locked: false } } + return json(stored)(route) + }], + ], + }) + const box = page.locator('[data-general-public-url]') + await box.waitFor() + check('设置页有「通用」标签,里面是对外访问地址', (await page.getByRole('tab', { name: '通用' }).getAttribute('aria-selected')) === 'true' + && (await box.innerText()).includes('对外访问地址')) + const said = await box.innerText() + check('……写明它管哪几件事:通知和 Webhook 的链接、单点登录的回调、来源核对', said.includes('Webhook') && said.includes('单点登录') + && said.includes('来源'), said.replace(/\s+/g, ' ')) + const input = box.locator('[data-public-url-input]') + const save = box.locator('[data-public-url-save]') + check('没改动时保存不可点', await save.isDisabled()) + await input.fill('https://agentlab.example.com/?from=x') + check('带 ? 的地址:当场标错、不让存', await save.isDisabled() && (await box.innerText()).includes('不带 ? 或 #')) + await input.fill('agentlab.example.com') + check('没写协议:同样不让存', await save.isDisabled()) + await input.fill('https://agentlab.example.com/') + await page.getByRole('tab', { name: '运行环境' }).click() + await page.waitForTimeout(300) + check('改了没保存就切标签:先问', (await dialog(page).innerText().catch(() => '')).includes('还没保存')) + await dialog(page).getByRole('button', { name: '返回保存' }).click() + await page.waitForTimeout(200) + check('……选「返回保存」:还在通用页,改动还在', page.url().endsWith('/settings/general') + && (await input.inputValue()) === 'https://agentlab.example.com/') + await save.click() + await until(() => sent.some((x) => x.key === 'PUT /settings')) + const put = sent.filter((x) => x.key === 'PUT /settings').at(-1)?.body?.values + check('保存只 PUT notify 一组、去掉末尾的斜杠,同组别的项原样带回', Object.keys(put ?? {}).join() === 'notify' + && put.notify.public_base_url === 'https://agentlab.example.com', JSON.stringify(put)) + check('……存完按钮回到不可点', await until(() => save.isDisabled())) + check('没有运行时报错、原生对话框', errors.length === 0 && natives.length === 0, [...errors, ...natives][0] ?? '') + await close() + + // 环境变量指定了:只显示它、写明来源,不给输入框 + const env = await open('/settings/general', { + handlers: [[/^GET \/settings$/, json({ ...stored, public_url: { value: 'https://agentlab.example.internal', env_locked: true } })]], + }) + const ro = env.page.locator('[data-public-url-readonly="env"]') + await ro.waitFor() + const roText = await ro.innerText() + check('环境变量指定的:显示它、写明由 AGENTLAB_PUBLIC_URL 指定,不给输入框', roText.includes('https://agentlab.example.internal') + && roText.includes('AGENTLAB_PUBLIC_URL') && await env.page.locator('[data-public-url-input]').count() === 0, roText) + await env.close() + + // 搭建者(开了登录):只看,写明只有管理员可以修改 + const builder = await open('/settings/general', { + handlers: [ + [/^GET \/auth\/me$/, json({ mode: 'local', username: 'lisi', display_name: '李四', role: 'builder', groups: [], verified: true })], + [/^GET \/settings$/, json(stored)], + ], + }) + const ro2 = builder.page.locator('[data-public-url-readonly="role"]') + await ro2.waitFor() + const ro2Text = await ro2.innerText() + check('搭建者:只显示当前值、写明只有管理员可以修改', ro2Text.includes('https://agentlab.example.com') && ro2Text.includes('只有管理员') + && await builder.page.locator('[data-public-url-input]').count() === 0, ro2Text) + await builder.close() +}) + await section('设置 · 点名的接入已停用或不存在', async () => { // 点名停用的接入以前照常调用,现在运行时照实报错(K1):存着的值后来被停用了,那一格下面要说;别的停用接入列着但选不了; // 服务端按项拒收(422 带 field)时整句落到那一格下面、焦点移过去,保存条上原因不截断。接入表、设置读写都是假的 diff --git a/scripts/check-schedules.mjs b/scripts/check-schedules.mjs index 32723b47..b43fd823 100644 --- a/scripts/check-schedules.mjs +++ b/scripts/check-schedules.mjs @@ -535,7 +535,9 @@ function applyPatch(s, body) { let realSettings = null async function settingsOf(state) { realSettings ??= await fetch(`${API}/settings`).then((r) => r.json()).catch(() => ({})) - return { ...realSettings, ui: { ...(realSettings.ui ?? {}), theme: state.theme ?? 'dark' }, notify: state.notify } + // 对外访问地址只读的那一组:state.publicUrl 设了就按它(环境变量指定的情形),否则照真服务端 + return { ...realSettings, ui: { ...(realSettings.ui ?? {}), theme: state.theme ?? 'dark' }, notify: state.notify, + public_url: state.publicUrl ?? realSettings.public_url } } /** 一个请求怎么答。返回 {status, json};返回 null 表示放行(GET)或拦下(写请求) */ @@ -1245,6 +1247,25 @@ await section('详情:Webhook 与运行链接地址', async () => { check('没有页面报错、原生对话框', !errors.length, errors.join(' | ')) }) +await section('详情:对外访问地址由环境变量指定', async () => { + // 全站只认一个对外访问地址(服务端 core/public_url.py):环境变量设了就以它为准,这里只显示、写明来源,不给修改; + // notify 里存着的旧值不生效,也不显示 + const env = 'https://agentlab.example.internal' + const { page, sent, errors } = await open('/schedules/s-weekly', { + state: freshState({ notify: { public_base_url: 'https://stale.example.com' }, publicUrl: { value: env, env_locked: true } }), + }) + await page.locator('[data-link-base]').waitFor() + const shown = await until(async () => { + const t = await text(page, '[data-link-base]') + return t.includes(env) ? t : null + }) + check('显示环境变量给的地址,不显示设置里存着的旧值', !!shown && !shown.includes('stale.example.com'), oneLine([shown ?? ''])) + check('……写明由环境变量 AGENTLAB_PUBLIC_URL 指定', (await text(page, '[data-link-base-env]').catch(() => '')).includes('AGENTLAB_PUBLIC_URL')) + check('……没有修改按钮,也没发 PUT', await page.locator('[data-link-base-edit]').count() === 0 + && writes(sent, /^PUT \/settings$/).length === 0) + check('没有页面报错、原生对话框', !errors.length, errors.join(' | ')) +}) + await section('详情:保存冲突与页头操作', async () => { const { page, sent, errors, state } = await open('/schedules/s-weekly') await page.locator('[data-schedule-form]').waitFor() From d14fc97da7f19c2c3bc6d321278295599805c410 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Fri, 16 Oct 2026 10:33:41 +0800 Subject: [PATCH 15/39] =?UTF-8?q?=E5=BA=94=E7=94=A8=E5=85=A5=E5=8F=A3?= =?UTF-8?q?=E3=80=81=E6=8C=89=E9=9C=80=E8=A7=86=E5=9B=BE=E7=9A=84=E8=A1=A8?= =?UTF-8?q?=E5=8D=95=E6=B2=A1=E9=80=9A=E8=BF=87=E6=A0=A1=E9=AA=8C=E4=B9=9F?= =?UTF-8?q?=E5=B8=A6=20code=EF=BC=88form=5Finvalid=EF=BC=89=EF=BC=8C?= =?UTF-8?q?=E5=92=8C=E5=BE=85=E5=8A=9E=E5=9B=9E=E5=A4=8D=E3=80=81=E5=85=A5?= =?UTF-8?q?=E7=AB=99=E9=92=A9=E5=AD=90=E5=90=8C=E4=B8=80=E4=B8=AA=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit api/apps.py 里这两处直接回 JSONResponse(detail + errors),上一次补错误码时只查了 HTTPException,漏了它们:外部系统发起 实例时表单不对,仍然只能按中文句子认。detail 和 errors 不变。 tests/test_integration_errors.py 的静态检查跟着扩:这几个模块里直接回的 JSONResponse 错误,内容里也要有 code(改之前 这条检查报出了这两处)。 --- backend/app/api/apps.py | 6 +++--- backend/app/api/coded.py | 2 ++ backend/tests/test_integration_errors.py | 18 ++++++++++++++---- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/backend/app/api/apps.py b/backend/app/api/apps.py index 6d0b281f..8d97e99d 100644 --- a/backend/app/api/apps.py +++ b/backend/app/api/apps.py @@ -21,7 +21,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.api.coded import ( APP_BINDINGS_INVALID, APP_DRAFT_INVALID, APP_GRANTS_INVALID, APP_HOOK_NOT_DECLARED, APP_HOOK_NOT_FOUND, - APP_INSTANCE_NOT_FOUND, APP_NOT_FOUND, APP_PARAMS_INVALID, APP_VERSION_NOT_FOUND, RUN_LAUNCH_REFUSED, + APP_INSTANCE_NOT_FOUND, APP_NOT_FOUND, APP_PARAMS_INVALID, APP_VERSION_NOT_FOUND, FORM_INVALID, RUN_LAUNCH_REFUSED, CodedHTTPException, ) from app.api.deps import Principal, actor_of, current_principal @@ -841,7 +841,7 @@ async def start(app_id: str, entry_id: str, payload: StartIn, session: AsyncSess try: run = await start_entry(app.id, entry_id, payload.values, principal) except FormRejected as e: - return JSONResponse(status_code=422, content={"detail": "表单没有通过校验", "errors": e.errors}) + return JSONResponse(status_code=422, content={"detail": "表单没有通过校验", "code": FORM_INVALID, "errors": e.errors}) except (EntryRefused, LaunchRefused) as e: raise _view_refused(e) from e return {"run_id": run.id, "title": instance_title(run)} @@ -912,7 +912,7 @@ async def refresh_view(app_id: str, view_id: str, payload: RefreshIn, session: A run, cached = await run_view(session, app=app, version=version, view_id=view_id, params=payload.params, principal=principal, actor=actor_of(x_actor)) except FormRejected as e: - return JSONResponse(status_code=422, content={"detail": "参数没有通过校验", "errors": e.errors}) + return JSONResponse(status_code=422, content={"detail": "参数没有通过校验", "code": FORM_INVALID, "errors": e.errors}) except (ViewRefused, LaunchRefused) as e: raise _view_refused(e) from e return {"run_id": run.id, "cached": cached} diff --git a/backend/app/api/coded.py b/backend/app/api/coded.py index 27e7ea66..4227b648 100644 --- a/backend/app/api/coded.py +++ b/backend/app/api/coded.py @@ -54,6 +54,8 @@ APP_DRAFT_INVALID = "app_draft_invalid" APP_GRANTS_INVALID = "app_grants_invalid" #: 页面参数不是 JSON 对象(422) APP_PARAMS_INVALID = "app_params_invalid" +#: 入口表单、按需视图的参数没通过校验(422,另带 errors:[{path, code, message}])。同待办回复、入站钩子的表单错误 +FORM_INVALID = "form_invalid" #: 发起实例、刷新视图、补算时引擎不发起运行,又没有更细的码(engine/launch.LaunchRefused 没带 code 的那几种) RUN_LAUNCH_REFUSED = "run_launch_refused" # 派生应用草稿时工作流、版本不存在(404)的码在 apps/derive.py(WORKFLOW_GONE、VERSION_GONE);入口、视图、补算的码 diff --git a/backend/tests/test_integration_errors.py b/backend/tests/test_integration_errors.py index cd8c0262..f3e39b6f 100644 --- a/backend/tests/test_integration_errors.py +++ b/backend/tests/test_integration_errors.py @@ -46,13 +46,23 @@ async def new_app() -> str: return row.id -def test_no_bare_http_exception_left_in_the_integration_modules(): - """这几个模块里不许再直接抛 HTTPException(没有 code):一律 CodedHTTPException。""" +def _called(node: ast.AST, name: str) -> bool: + return isinstance(node, ast.Call) and (getattr(node.func, "id", None) or getattr(node.func, "attr", None)) == name + + +def test_no_error_without_a_code_left_in_the_integration_modules(): + """这几个模块里不许再直接抛 HTTPException(没有 code),一律 CodedHTTPException;直接回 JSONResponse 的错误(表单 + 没通过校验那几处)内容里也要有 code。""" for rel in ("api/apps.py", "api/tasks.py", "api/artifacts.py", "api/deps.py", "auth/__init__.py"): tree = ast.parse((APP_DIR / rel).read_text()) - bare = [node.lineno for node in ast.walk(tree) if isinstance(node, ast.Call) - and (getattr(node.func, "id", None) or getattr(node.func, "attr", None)) == "HTTPException"] + bare = [node.lineno for node in ast.walk(tree) if _called(node, "HTTPException")] assert not bare, f"{rel} 第 {bare} 行抛的 HTTPException 没有 code" + for node in ast.walk(tree): + if not _called(node, "JSONResponse"): + continue + content = next((k.value for k in node.keywords if k.arg == "content"), node.args[0] if node.args else None) + keys = {k.value for k in getattr(content, "keys", []) if isinstance(k, ast.Constant)} + assert "code" in keys, f"{rel} 第 {node.lineno} 行回的 JSONResponse 没有 code" def test_codes_agree_with_the_ones_already_in_use(): From 3293ac0b5811d12f264652a1a24a42a1e493d2b6 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Fri, 16 Oct 2026 10:34:18 +0800 Subject: [PATCH 16/39] =?UTF-8?q?=E8=87=AA=E7=94=B1=E5=BD=A2=E8=93=9D?= =?UTF-8?q?=E5=9B=BE=EF=BC=88=E5=90=8E=E7=AB=AF=EF=BC=89=EF=BC=9A=E6=8C=89?= =?UTF-8?q?=E5=AF=BC=E5=85=A5=E7=9A=84=E5=9B=BE=E5=88=97=E5=87=BA=E3=80=8C?= =?UTF-8?q?=E8=B0=81=E6=9D=A5=E5=81=9A=E3=80=8D=EF=BC=8C=E5=8F=AA=E5=AE=9A?= =?UTF-8?q?=E7=94=B1=E8=B0=81=E5=81=9A=EF=BC=9B=E6=B2=A1=E7=BB=91=E6=97=B6?= =?UTF-8?q?=20bp.people=5Funbound=20=E6=8C=87=E5=90=91=E9=82=A3=E5=BC=A0?= =?UTF-8?q?=E5=8D=A1=E7=89=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 为什么:自由形的骨架就是导入的图,图里派给岗位的人工节点、没登记查询的写入在应用里都要绑人,可蓝图上没有「谁来做」卡片。 第一次发布挡在 app.slot_unbound / app.reconciler_unbound 上,白话叫人「在「谁来做」里给这一步选人」,卡片却不存在;这时 应用还没建,也没法去应用编辑器绑,只能绕开蓝图(05-已知遗留 (d))。 做了什么: - freeform 的 cards 加 people(「可选:只定每一步由谁做;会签、时限、超时在画布上改」)。freeform.people_rows 按 tree.slot_refs 列岗位行(同一个岗位用在几处合成一行,label 用节点标题拼、detail 写全每一处用途,path 是 /people//who);写入能力没登记 查询时加一行「写入结果不明时谁去核实」(path /writeback/reconciler);发起人、表达式、外部系统回调的步骤列成只读行。 blueprint_out 给出 derived_people,界面照这一份画。 - 存储复用现有路径:岗位绑定存在 spec.people[](id 就是岗位名、只写 who),_common.slot_of / bindings_of 原样生效;核实人存在 /writeback/reconciler。needs_reconciler 对自由形按图里的写入算(with_graph 把蓝图存的图带给模板函数)。 - bp_set 对自由形的 people 只收 who(只收人和组,只由用户定),改会签、时限、超时、加减步骤报 bp.template_mismatch「在画布上 改」。apply_bp_op / blueprint_issues 新增 graph 参数,卡片接口、蓝图检查、试运行、实例化、蓝图助手都把蓝图存的图传下去。 - 检查:派生岗位、核实人没绑报 bp.people_unbound(试运行 warning、发布 error),路径就是那一行,带 slot;图里已经没有的岗位、 卡片上还留着的绑定报 warning,不静默删除。发布单照核实人的办法,去掉同一件事的 app.slot_unbound(slots.)和 app.reconciler_unbound。 偏离:岗位行不让助手代填(同「结果不明时谁去核实」,派活给谁是人的决定)。 测试:test_publish_plan 新增「财务审批(岗位)+ 写回(没登记查询)」建自由形蓝图一条(两行、不选人时只有 bp.people_unbound、 改会签被拒、选好人后绑到 finance 和核实岗位、发布单不再挡);老的两条按新说法改;test_blueprint_templates 新增 people_rows 的合并、升级岗位、只读行、残留绑定提醒。 --- backend/app/api/copilot_blueprint.py | 2 +- backend/app/blueprints/instantiate.py | 2 +- backend/app/blueprints/ops.py | 94 ++++++++++++++++++-- backend/app/blueprints/publish_plan.py | 14 ++- backend/app/blueprints/store.py | 20 +++-- backend/app/blueprints/templates/_common.py | 19 ++++ backend/app/blueprints/templates/freeform.py | 68 +++++++++++++- backend/app/blueprints/trial.py | 2 +- backend/tests/test_blueprint_templates.py | 34 +++++++ backend/tests/test_publish_plan.py | 65 +++++++++++++- 10 files changed, 297 insertions(+), 23 deletions(-) diff --git a/backend/app/api/copilot_blueprint.py b/backend/app/api/copilot_blueprint.py index 8068edc0..32cc2064 100644 --- a/backend/app/api/copilot_blueprint.py +++ b/backend/app/api/copilot_blueprint.py @@ -293,7 +293,7 @@ async def blueprint_stream(payload: BlueprintStreamIn, session: AsyncSession = D system = "\n\n".join(part for part in ( BLUEPRINT_SYSTEM_PROMPT, _templates_text(personal=personal), await looks.overview(), BLUEPRINT_PROTOCOL_PROMPT, looks.guide(), BLUEPRINT_LOOK_WHEN_PROMPT) if part) - missing = [i for i in blueprint_issues(spec, for_="trial") if i["level"] == "error"] + missing = [i for i in blueprint_issues(spec, for_="trial", graph=row.graph) if i["level"] == "error"] human = copilot._with_history(BLUEPRINT_REQUEST_PROMPT.format( spec=json.dumps(spec, ensure_ascii=False), instruction=instruction, issues=BLUEPRINT_ISSUES_PROMPT.format(lines="\n".join(f"- {i['path']} {i['message']}" for i in missing)) diff --git a/backend/app/blueprints/instantiate.py b/backend/app/blueprints/instantiate.py index d7e966ab..1cb96967 100644 --- a/backend/app/blueprints/instantiate.py +++ b/backend/app/blueprints/instantiate.py @@ -198,7 +198,7 @@ def instantiate(blueprint: dict[str, Any], *, filled: dict[str, Any] | None = No spec = _spec_of(blueprint) ref = spec.get("template") or {} template = get_template(str(ref.get("id") or "")) - problems = [i for i in blueprint_issues(spec, for_="trial") if i["level"] == "error"] + problems = [i for i in blueprint_issues(spec, for_="trial", graph=graph) if i["level"] == "error"] if template is None: from app.apps.spec import empty_spec diff --git a/backend/app/blueprints/ops.py b/backend/app/blueprints/ops.py index a626fed5..59d1aaa4 100644 --- a/backend/app/blueprints/ops.py +++ b/backend/app/blueprints/ops.py @@ -520,8 +520,47 @@ def _container(spec: dict[str, Any], tokens: list[str]) -> tuple[dict[str, Any], raise BpRefused(f"不能改这个位置:/{'/'.join(tokens)}") -def _set(spec: dict[str, Any], tokens: list[str], value: Any, source: str) -> None: +def is_freeform(spec: dict[str, Any]) -> bool: + from app.blueprints.templates._common import is_freeform as imports_graph + + return imports_graph(spec) + + +#: 自由形的「谁来做」只定由谁做(05-已知遗留 (d)、遗留-Q5) +FREEFORM_PEOPLE_ONLY_WHO = "自由形蓝图的「谁来做」只定每一步由谁做;会签、时限、超时、加减步骤在画布上改" +FREEFORM_PEOPLE_USER = "「谁来做」要由你来定:派活给谁是人的决定,助手不替你选" + + +def _freeform_who(spec: dict[str, Any], tokens: list[str], value: Any, source: str, graph: Any) -> None: + """自由形的 /people//who:i 是按导入的图列出的第几个岗位(freeform.people_rows 的岗位行,界面照那一份的 path 写), + 绑定存进 spec.people[](id 就是岗位名、只有 who)。只收人和组:岗位、发起人在图上已经写定了。""" + from app.blueprints.templates import freeform + from app.blueprints.templates._common import with_graph + + if len(tokens) != 3 or tokens[2] != "who": + raise BpRefused(FREEFORM_PEOPLE_ONLY_WHO, code="bp.template_mismatch") + if source != "user": + raise BpRefused(FREEFORM_PEOPLE_USER) + if graph is None: + raise BpRefused("这张蓝图还没有导入的图,没法按图定谁来做") + rows = [r for r in freeform.people_rows(with_graph(spec, graph)) if r["kind"] == "slot"] + slot = rows[_index(tokens[1], len(rows))]["slot"] + checked = _checked("people", "who", value, spec) + if checked and any(p["kind"] not in ("user", "group") for p in checked): + raise BpRefused("这里只选人或组:岗位、发起人在图上已经定了") + entries = spec.setdefault("people", []) + entry = next((e for e in entries if e.get("id") == slot), None) + if entry is None: + entry = {"id": slot} + entries.append(entry) + entry["who"] = _item(checked, source) + + +def _set(spec: dict[str, Any], tokens: list[str], value: Any, source: str, graph: Any = None) -> None: card = tokens[0] + if card == "people" and is_freeform(spec): + _freeform_who(spec, tokens, value, source, graph) + return if card == "access": if len(tokens) != 2 or tokens[1] not in ACCESS_KINDS: raise BpRefused("「谁能看」只有使用、查看全部、管理三档") @@ -577,6 +616,8 @@ def _set(spec: dict[str, Any], tokens: list[str], value: Any, source: str) -> No def _add(spec: dict[str, Any], tokens: list[str], value: Any, source: str) -> None: card = tokens[0] + if card == "people" and is_freeform(spec): + raise BpRefused(FREEFORM_PEOPLE_ONLY_WHO, code="bp.template_mismatch") if card == "access": if len(tokens) not in (2, 3) or tokens[1] not in ACCESS_KINDS: raise BpRefused("「谁能看」只有使用、查看全部、管理三档") @@ -598,8 +639,13 @@ def _add(spec: dict[str, Any], tokens: list[str], value: Any, source: str) -> No entries.insert(at, _new_entry(card, value, source, spec)) -def _remove(spec: dict[str, Any], tokens: list[str], source: str) -> None: +def _remove(spec: dict[str, Any], tokens: list[str], source: str, graph: Any = None) -> None: card = tokens[0] + if card == "people" and is_freeform(spec): + if len(tokens) != 3: + raise BpRefused(FREEFORM_PEOPLE_ONLY_WHO, code="bp.template_mismatch") + _freeform_who(spec, tokens, None, source, graph) + return if card == "access": if len(tokens) != 3 or tokens[1] not in ACCESS_KINDS: raise BpRefused("「谁能看」要写明删哪一档的第几个") @@ -655,8 +701,9 @@ def _template(spec: dict[str, Any], op: BpOp, source: str) -> None: prefill(template.id, spec) -def apply_bp_op(spec: dict[str, Any], op: dict[str, Any], *, source: str) -> dict[str, Any]: - """把一个 bp_* 操作落到蓝图上,交回新的蓝图(不改入参)。source 是 user / assistant。不合规时抛 BpRefused。""" +def apply_bp_op(spec: dict[str, Any], op: dict[str, Any], *, source: str, graph: Any = None) -> dict[str, Any]: + """把一个 bp_* 操作落到蓝图上,交回新的蓝图(不改入参)。source 是 user / assistant。不合规时抛 BpRefused。 + graph 是蓝图存的图:自由形的「谁来做」按它列岗位(没给时那张卡片改不了)。""" if source not in BP_SOURCES: raise BpRefused(f"操作的来源只能是用户或助手,当前为「{source}」") if not isinstance(op, dict) or op.get("op") not in BP_OPS: @@ -677,11 +724,11 @@ def apply_bp_op(spec: dict[str, Any], op: dict[str, Any], *, source: str) -> dic if source != "user" and (only := next((k for k in USER_ONLY if tuple(tokens[:len(k)]) == k), None)): raise BpRefused(_USER_ONLY_TEXT[only[-1]]) if parsed.op == "bp_set": - _set(out, tokens, parsed.value, source) + _set(out, tokens, parsed.value, source, graph) elif parsed.op == "bp_add": _add(out, tokens, parsed.value, source) else: - _remove(out, tokens, source) + _remove(out, tokens, source, graph) try: return parse_blueprint(out).model_dump(mode="json", by_alias=True) except ValidationError as e: @@ -708,9 +755,9 @@ def _value(item: Any) -> Any: return item.get("value") if isinstance(item, dict) else None -def blueprint_issues(spec: dict[str, Any], *, for_: str) -> list[dict[str, Any]]: +def blueprint_issues(spec: dict[str, Any], *, for_: str, graph: Any = None) -> list[dict[str, Any]]: """蓝图检查(§3.13 bp.*):for_ = trial(缺项挡试运行)/ publish(待确认项挡发布)/ personal。 - 每条 {code, level, path, message}。""" + 每条 {code, level, path, message}。graph 是蓝图存的图:自由形的「谁来做」按它查(没给时不查这一张)。""" from app.blueprints.templates import get_template if for_ not in ISSUE_PURPOSES: @@ -743,7 +790,10 @@ def blueprint_issues(spec: dict[str, Any], *, for_: str) -> list[dict[str, Any]] _data_issues(spec, "data" in required, add) if "cadence" in required and _value(spec.get("cadence")) is None: add("bp.missing", "/cadence", "「多久更新」还没定:每天、每周还是每月,几点,统计哪一期") - if "people" in used: + if "people" in used and template.imports_graph: + if graph is not None and for_ != "personal": + out += _freeform_people_issues(spec, graph, publish) + elif "people" in used: _people_issues(spec, "people" in required, publish, add) if "fields" in used: _field_issues(spec, "fields" in required, add) @@ -836,6 +886,32 @@ def _people_issues(spec: dict[str, Any], required: bool, publish: bool, add: Cal f"「{label}」的条件拿「{threshold['field']}」比大小,它要是数字字段") +def _freeform_people_issues(spec: dict[str, Any], graph: Any, publish: bool) -> list[dict[str, Any]]: + """自由形的「谁来做」(freeform.people_rows):岗位、核实人没绑的报 bp.people_unbound(试运行 warning:待办派给试运行的 + 人自己;发布 error),路径就是那一行的 path,带上 slot(发布单据此去掉同一件事的 app.slot_unbound / app.reconciler_unbound)。 + 图里已经没有的岗位、卡片上还留着绑定的,照实提醒(warning),不静默删掉。""" + from app.blueprints.templates import freeform + from app.blueprints.templates._common import with_graph + + out: list[dict[str, Any]] = [] + level = "error" if publish else "warning" + rows = freeform.people_rows(with_graph(spec, graph)) + for row in rows: + if row["kind"] not in ("slot", "reconciler") or row["who"]: + continue + message = (f"{row['label']}还没定:{row['detail']},写入外部系统的结果不明时要有人去目标系统里核实" + if row["kind"] == "reconciler" else f"{row['label']}还没定谁来处理({row['detail']})") + out.append({"code": "bp.people_unbound", "level": level, "path": row["path"], "message": message, + "slot": row["slot"]}) + known = {r["slot"] for r in rows if r["kind"] == "slot"} + for step in spec.get("people") or []: + slot = str(step.get("id") or "") + if slot and slot not in known and _value(step.get("who")): + out.append({"code": "bp.template_mismatch", "level": "warning", "path": "/people", + "message": f"岗位「{slot}」在导入的图里已经没有了:卡片上留着的这条绑定不会生效"}) + return out + + def _field_issues(spec: dict[str, Any], required: bool, add: Callable[..., None]) -> None: fields = spec.get("fields") or [] if required and not fields: diff --git a/backend/app/blueprints/publish_plan.py b/backend/app/blueprints/publish_plan.py index cdc1749b..b3c7a289 100644 --- a/backend/app/blueprints/publish_plan.py +++ b/backend/app/blueprints/publish_plan.py @@ -75,6 +75,8 @@ TRIAL_REJECTED = "plan.trial_rejected" TRIAL_DIFFERS = "plan.trial_differs" #: 蓝图检查:写回的结果不明时没人核实(blueprints/ops.blueprint_issues) RECONCILER_UNBOUND = "bp.reconciler_unbound" +#: 蓝图检查:「谁来做」有一步还没定由谁处理;自由形的带着 slot(哪个岗位,核实人是 apps/derive.RECONCILER) +PEOPLE_UNBOUND = "bp.people_unbound" ADMIN_REQUIRED = "plan.admin_required" DOWNGRADE_ADMIN = "plan.downgrade_admin" PREFLIGHT = "plan.preflight" @@ -239,8 +241,10 @@ async def _simulated_pins(session: AsyncSession, spec: dict[str, Any], target: _ def bindings_of(bp: Blueprint) -> dict[str, Any]: """岗位绑定:「谁来做」卡片上写了具体的人或组的,按模板定的岗位名绑上(templates/_common.bindings_of)。""" from app.blueprints.templates._common import bindings_of as from_cards + from app.blueprints.templates._common import with_graph - return from_cards(bp.spec or {}) + # 带上蓝图存的图:自由形的核实人要不要绑按导入的图算(05-已知遗留 (d)) + return from_cards(with_graph(bp.spec or {}, bp.graph)) def grant_items(bp: Blueprint) -> list[tuple[str, str, str]]: @@ -354,6 +358,14 @@ async def _preflight(session: AsyncSession, bp: Blueprint, target: _Target, *, l issues = [i for i in issues if i.get("code") != "app.reconciler_unbound" and not (i.get("code") == "app.slot_unbound" and i.get("path") == f"slots.{RECONCILER}")] + if unbound := {str(i["slot"]) for i in out if i.get("code") == PEOPLE_UNBOUND and i.get("slot")}: + # 自由形的「谁来做」:岗位、核实人没绑,蓝图检查已经指到卡片上那一行(bp.people_unbound 带着 slot),应用检查说的是 + # 同一件事,不再重复列(同上面核实岗位的办法) + from app.apps.derive import RECONCILER + + issues = [i for i in issues if not (i.get("code") == "app.slot_unbound" + and str(i.get("path") or "").removeprefix("slots.") in unbound) + and not (i.get("code") == "app.reconciler_unbound" and RECONCILER in unbound)] out += [{**i, "source": "app"} for i in issues] # 授权:给的人和组要在名录里(同授权接口) _, problem = await grants_problem(session, grant_items(bp)) diff --git a/backend/app/blueprints/store.py b/backend/app/blueprints/store.py index a3ef2fc4..700711bc 100644 --- a/backend/app/blueprints/store.py +++ b/backend/app/blueprints/store.py @@ -183,7 +183,7 @@ async def apply_ops_and_save(session: AsyncSession, blueprint: Any, ops: list[di try: op = await _verified_reuse(session, raw) before = _at(spec, op.get("path")) - spec = apply_bp_op(spec, op, source=source) + spec = apply_bp_op(spec, op, source=source, graph=blueprint.graph) except BpRefused as e: if not partial: raise @@ -313,7 +313,7 @@ def blueprint_issues_of(blueprint: Any) -> list[dict[str, Any]]: from app.plain import with_plain spec = blueprint.spec or {} - issues = blueprint_issues(spec, for_="personal" if blueprint.scope == "personal" else "publish") + issues = blueprint_issues(spec, for_="personal" if blueprint.scope == "personal" else "publish", graph=blueprint.graph) for joint_id, state in (blueprint.joints or {}).items(): if isinstance(state, dict) and state.get("status") in ("pending", "refused"): issues.append({"code": "tpl.joint_unfilled" if state["status"] == "pending" else "tpl.joint_refused", @@ -375,11 +375,19 @@ async def blueprint_view(session: AsyncSession, blueprint: Any) -> dict[str, Any return blueprint_out(blueprint, extra=await data_source_issues(session, blueprint)) +def _derived_people(blueprint: Any) -> list[dict[str, Any]]: + from app.blueprints.templates import freeform + from app.blueprints.templates._common import with_graph + + return freeform.people_rows(with_graph(blueprint.spec or {}, blueprint.graph)) + + def blueprint_out(blueprint: Any, *, issues: bool = True, extra: list[dict[str, Any]] | None = None) -> dict[str, Any]: """一张蓝图交给前端的样子(frontend/src/typesBlueprint.ts 的 Blueprint)。关节只给状态和模板声明的说法、类型、分组 (JointState:status、by、note、label、kind、group),不给填的值(值在图上看)。reconciler_needed:写回卡片上要不要 - 摆「结果不明时谁去核实」(目标能力没登记查询,templates/_common.needs_reconciler),界面不自己判。""" - from app.blueprints.templates._common import needs_reconciler + 摆「结果不明时谁去核实」(目标能力没登记查询,templates/_common.needs_reconciler),界面不自己判。derived_people:自由形 + 按导入的图列出的「谁来做」(templates/freeform.people_rows,05-已知遗留 (d)),别的模板为空。""" + from app.blueprints.templates._common import is_freeform, needs_reconciler, with_graph joints = {jid: {k: v for k, v in state.items() if k in ("status", "by", "note", "label", "kind", "group")} for jid, state in (blueprint.joints or {}).items() if isinstance(state, dict)} @@ -389,7 +397,9 @@ def blueprint_out(blueprint: Any, *, issues: bool = True, extra: list[dict[str, "template_version": blueprint.template_version, "source": blueprint.source or {}, "conversation_id": blueprint.conversation_id, "workflow_id": blueprint.workflow_id, "app_id": blueprint.app_id, "last_trial_run_id": blueprint.last_trial_run_id, "graph_hash": blueprint.graph_hash, "joints": joints, - "copied_from": blueprint.copied_from, "reconciler_needed": needs_reconciler(blueprint.spec or {}), + "copied_from": blueprint.copied_from, + "reconciler_needed": needs_reconciler(with_graph(blueprint.spec or {}, blueprint.graph)), + "derived_people": _derived_people(blueprint) if is_freeform(blueprint.spec or {}) else None, "created_at": blueprint.created_at.isoformat() if blueprint.created_at else None, "updated_at": blueprint.updated_at.isoformat() if blueprint.updated_at else None, } diff --git a/backend/app/blueprints/templates/_common.py b/backend/app/blueprints/templates/_common.py index 589c221a..20ee6e0c 100644 --- a/backend/app/blueprints/templates/_common.py +++ b/backend/app/blueprints/templates/_common.py @@ -78,6 +78,19 @@ def context(bp: dict[str, Any]) -> dict[str, Any]: return ctx if isinstance(ctx, dict) else {} +def with_graph(spec: dict[str, Any], graph: Any) -> dict[str, Any]: + """只有卡片(Blueprint.spec)、没有实例化上下文的地方,带上蓝图存的图再交给模板的函数:自由形的「谁来做」、要不要 + 核实人都按导入的图算(freeform.people_rows / needs_reconciler)。""" + return {**spec, CONTEXT_KEY: {**context(spec), "graph": graph}} + + +def is_freeform(spec: dict[str, Any]) -> bool: + """卡片上选的是导入图的模板(自由形)。""" + ref = spec.get("template") + template = get_template(str(ref.get("id") or "")) if isinstance(ref, dict) else None + return template is not None and template.imports_graph + + def app_name(bp: dict[str, Any], fallback: str) -> str: """应用和工作流的名字:实例化时给的名字,没给就用目的的前一段。""" name = str(context(bp).get("name") or "").strip() @@ -434,6 +447,12 @@ def needs_reconciler(bp: dict[str, Any]) -> bool: 蓝图(reconciler_needed)共用这一份。""" ref = bp.get("template") template = get_template(str(ref.get("id") or "")) if isinstance(ref, dict) else None + if template is not None and template.imports_graph: + # 自由形没有「写回哪里」:按导入的图里的写入算(bp 要带着图,with_graph),核实人存在 /writeback/reconciler + # (05-已知遗留 (d)) + from app.blueprints.templates import freeform + + return freeform.needs_reconciler(bp) if template is None or "writeback" not in template.cards: return False capability = value((bp.get("writeback") or {}).get("capability")) diff --git a/backend/app/blueprints/templates/freeform.py b/backend/app/blueprints/templates/freeform.py index 42cc6c3a..db886bb4 100644 --- a/backend/app/blueprints/templates/freeform.py +++ b/backend/app/blueprints/templates/freeform.py @@ -1,5 +1,11 @@ """自由形 freeform(§3.3):从已有的运行(问数据的一轮回答,经 from-run 提取)或已发布的工作流版本导入,图原样用, -没有关节。蓝图只有目的、谁能看、多久更新三张卡片;要改图只能回画布(搭建者)。 +没有关节。蓝图有目的、谁能看、多久更新、谁来做四张卡片;要改图只能回画布(搭建者)。 + +「谁来做」(docs/design/next-round/05-已知遗留.md (d)):骨架就是导入的图,卡片只填「谁」这个关节——按图列出用到的岗位 +(人工节点派给谁、超时升级给谁、写入的发送确认派给谁,people_rows),写入能力没登记查询时再加一行「写入结果不明时谁去核实」。 +只定由谁做:会签、时限、超时在画布上改(改了会改图,图哈希就和发布的那一版对不上)。绑定存在 spec.people[](Step.id 就是 +岗位名、只写 who,_common.slot_of / bindings_of 原样生效),核实人存在 /writeback/reconciler。以前这张卡片不存在,第一次 +发布挡在 app.slot_unbound / app.reconciler_unbound 上,白话却叫人去「谁来做」里选人。 图由实例化的上下文给(graph,导入时存在蓝图上)。应用骨架按 apps/derive.py 的派生规则:有人工节点或写入的按流程, 否则按看板;写入节点按能力登记的进程内快照认(和运行时同一个判断,capabilities.registry.writes_externally)。 @@ -38,6 +44,65 @@ def _writes(graph: dict[str, Any]) -> dict[str, str]: return out +#: 「谁来做」一行里每处用途怎么说(apps/tree.SLOT_ROLES,加上核实) +USE_TEXT = {"assignee": "处理", "escalation": "超时升级", "confirm": "发送前确认", "reconcile": "写入结果不明时去核实"} +#: 不按岗位派的步骤(只读行):发起人自己、按表达式算出来的人、等外部系统回调 +FIXED_TEXT = {"initiator": "由发起人自己处理", "expression": "按图里的表达式算出由谁处理(在画布上改)", + "hook": "等外部系统回调,不派给人"} +#: 核实人那一行的名字 +RECONCILER_LABEL = "写入结果不明时谁去核实" + + +def people_rows(bp: dict[str, Any]) -> list[dict[str, Any]]: + """按导入的图列出「谁来做」:[{kind, slot, label, detail, path, who, uses}]。 + + kind=slot:图里引用的一个岗位(同一个岗位用在几处合成一行,按节点出现的顺序),path 是 /people//who(i 是岗位行的 + 序号),who 是卡片上绑的人或组;kind=reconciler:写入能力没登记查询时的核实人,path 是 /writeback/reconciler; + kind=initiator / expression / hook:不按岗位派的步骤,只读(path 为空)。label 用节点在图上的名字拼(岗位名是画布上 + 随手写的 id),detail 写全每一处用途。界面照这一份画,不自己从图里推。""" + from app.apps.derive import RECONCILER + from app.apps.tree import node_map, slot_refs + + graph = imported(bp) + writes = _writes(graph) + nodes = node_map(graph) + bound = {str(step.get("id")): c.value(step.get("who")) for step in bp.get("people") or [] if isinstance(step, dict)} + rows: list[dict[str, Any]] = [] + by_slot: dict[str, dict[str, Any]] = {} + for ref in slot_refs(graph, writes): + row = by_slot.get(ref["slot"]) + if row is None: + row = by_slot[ref["slot"]] = {"kind": "slot", "slot": ref["slot"], "path": f"/people/{len(by_slot)}/who", + "who": bound.get(ref["slot"]), "uses": []} + rows.append(row) + row["uses"].append({"node_id": ref["node_id"], "title": ref["title"], "role": ref["role"]}) + unchecked = [(nid, cap) for nid, cap in writes.items() if not c.write_lookup(cap)] + if unchecked: + rows.append({"kind": "reconciler", "slot": RECONCILER, "path": "/writeback/reconciler", + "who": c.value((bp.get("writeback") or {}).get("reconciler")), + "uses": [{"node_id": nid, "title": nodes[nid]["title"], "role": "reconcile"} for nid, _ in unchecked]}) + for nid, n in nodes.items(): + if n["type"] != "human" and nid not in writes: + continue + assignee = n["config"].get("assignee") + kind = assignee.get("kind") if isinstance(assignee, dict) else None + if kind in FIXED_TEXT: + rows.append({"kind": kind, "slot": None, "path": None, "who": None, + "uses": [{"node_id": nid, "title": n["title"], + "role": "assignee" if n["type"] == "human" else "confirm"}]}) + for row in rows: + titles = list(dict.fromkeys(u["title"] for u in row["uses"])) + row["label"] = RECONCILER_LABEL if row["kind"] == "reconciler" else "、".join(f"「{t}」" for t in titles) + row["detail"] = FIXED_TEXT[row["kind"]] if row["kind"] in FIXED_TEXT else ";".join( + f"「{u['title']}」{USE_TEXT[u['role']]}" for u in row["uses"]) + return rows + + +def needs_reconciler(bp: dict[str, Any]) -> bool: + """导入的图里有写入能力没登记查询(同 app_skeleton 加核实岗位的判断)。""" + return any(not c.write_lookup(cap) for cap in _writes(imported(bp)).values()) + + def app_skeleton(bp: dict[str, Any], graph: dict[str, Any]) -> dict[str, Any]: from app.apps.derive import ENTRY, INSTANCE_VIEW, PIN, VIEW, _entries, _results from app.apps.forms import input_schema @@ -103,6 +168,7 @@ TEMPLATE = register(TemplateDef( applies_when="用户已经有一次满意的问数据回答,或者一个已发布的工作流,想直接做成应用;需求在五个场景之外时也用它。", cards={"purpose": "必填:一句话说清这个应用做什么", "cadence": "可选:要不要定期自动出数", + "people": "可选:只定每一步由谁做;会签、时限、超时在画布上改", "access": "可选:谁能用、谁能看全部、谁能管理"}, personal="导入的图满足问数据的规则(只读、没有人工节点和写入)时可以", skeleton=skeleton, app_skeleton=app_skeleton, schedule=schedule, tags=["导入"], diff --git a/backend/app/blueprints/trial.py b/backend/app/blueprints/trial.py index 1142ece8..96578058 100644 --- a/backend/app/blueprints/trial.py +++ b/backend/app/blueprints/trial.py @@ -147,7 +147,7 @@ async def materialize(session: AsyncSession, bp: Blueprint, *, principal: Princi digest = graph_hash(graph or {}) app_spec = _draft_pins(result.app_spec, bp, digest) # 实例化交回的是挡路的 error;蓝图检查里的提醒(模板版本旧了之类)另外带上 - warnings = [i for i in blueprint_issues(bp.spec or {}, for_="trial") if i.get("level") != "error"] + warnings = [i for i in blueprint_issues(bp.spec or {}, for_="trial", graph=bp.graph) if i.get("level") != "error"] problems = [with_plain(p) for p in [*result.problems, *warnings]] bp.graph, bp.graph_hash, bp.joints = graph, digest, result.joints bp.updated_by = principal.username if principal else bp.updated_by diff --git a/backend/tests/test_blueprint_templates.py b/backend/tests/test_blueprint_templates.py index 2f054fdd..888d890f 100644 --- a/backend/tests/test_blueprint_templates.py +++ b/backend/tests/test_blueprint_templates.py @@ -265,3 +265,37 @@ def test_trial_inputs_follow_the_cadence(): inputs = trial_inputs(get_template("periodic_board").example(), at=datetime(2026, 10, 15, 3, tzinfo=timezone.utc)) assert inputs == {"period": "2026-W41", "period_start": "2026-10-05", "period_end": "2026-10-11"} assert trial_inputs(get_template("approval").example()) == {} + + +def test_freeform_people_rows_follow_the_imported_graph(): + """自由形的「谁来做」按导入的图列(05-已知遗留 (d)):同一个岗位用在几处合成一行、说明写全;发起人、表达式派单的步骤是 + 只读行;没有人工节点和写入的图一行都没有。卡片上留着、图里已经没有的岗位照实提醒,不静默删掉。""" + from app.blueprints.ops import blueprint_issues + from app.blueprints.templates import freeform + + graph = {"nodes": [ + c.node("start", "input", "发起", fields=[{"name": "q"}]), + c.node("a", "human", "初审", mode="approve", assignee={"kind": "slot", "slot": "finance"}, + on_timeout={"action": "escalate", "to_slot": "director"}, due="PT1H"), + c.node("b", "human", "复审", mode="approve", assignee={"kind": "slot", "slot": "finance"}), + c.node("me", "human", "本人确认", mode="approve", assignee={"kind": "initiator"}), + c.node("calc", "human", "按金额派", mode="approve", assignee={"kind": "expression", "expression": "input.q"}), + c.node("done", "output", "完成", fields=[])], + "edges": [c.edge("start", "a"), c.edge("a", "b"), c.edge("b", "me"), c.edge("me", "calc"), c.edge("calc", "done")]} + spec = {**get_template("freeform").example(), "people": [ + {"id": "finance", "who": c.user_item([{"kind": "user", "name": "zhang"}])}, + {"id": "gone", "who": c.user_item([{"kind": "user", "name": "li"}])}]} + rows = freeform.people_rows(c.with_graph(spec, graph)) + assert [(r["kind"], r["slot"], r["path"]) for r in rows] == [ + ("slot", "finance", "/people/0/who"), ("slot", "director", "/people/1/who"), + ("initiator", None, None), ("expression", None, None)] + assert rows[0]["label"] == "「初审」、「复审」" and rows[0]["detail"] == "「初审」处理;「复审」处理" + assert rows[1]["detail"] == "「初审」超时升级" and rows[0]["who"] == [{"kind": "user", "name": "zhang"}] + issues = blueprint_issues(spec, for_="publish", graph=graph) + unbound = [i for i in issues if i["code"] == "bp.people_unbound"] + assert [(i["path"], i["slot"], i["level"]) for i in unbound] == [("/people/1/who", "director", "error")] + assert [i["level"] for i in blueprint_issues(spec, for_="trial", graph=graph) if i["code"] == "bp.people_unbound"] \ + == ["warning"], "试运行时待办派给试运行的人自己,不挡" + stale = [i for i in issues if i["code"] == "bp.template_mismatch" and "gone" in i["message"]] + assert stale and stale[0]["level"] == "warning" + assert freeform.people_rows(c.with_graph(get_template("freeform").example(), freeform.example_graph())) == [] diff --git a/backend/tests/test_publish_plan.py b/backend/tests/test_publish_plan.py index 00afa3ea..4fcdf785 100644 --- a/backend/tests/test_publish_plan.py +++ b/backend/tests/test_publish_plan.py @@ -218,8 +218,12 @@ async def test_a_process_with_an_unbound_step_stops_at_preflight(client): assert (await client.post(f"/api/blueprints/{bp['id']}/instantiate")).status_code == 200 plan = (await client.post(f"/api/blueprints/{bp['id']}/publish-plan", json={"level": "published"})).json() assert plan["ok"] is False - unbound = [p for p in plan["problems"] if p["code"] == "app.slot_unbound"] - assert unbound and unbound[0]["plain"]["who"] == "you" and "谁来做" in unbound[0]["plain"]["action"] + # 自由形按导入的图列出「谁来做」(05-已知遗留 (d)):会签的 finance、超时升级的 director 各一行,没绑人报在那一行上; + # 应用检查说的是同一件事(app.slot_unbound),不再重复列——它指的「谁来做」以前在自由形蓝图上不存在 + unbound = {p["path"]: p for p in plan["problems"] if p["code"] == "bp.people_unbound"} + assert set(unbound) == {"/people/0/who", "/people/1/who"} + assert unbound["/people/0/who"]["plain"]["who"] == "you" and "谁来做" in unbound["/people/0/who"]["plain"]["action"] + assert "app.slot_unbound" not in {p["code"] for p in plan["problems"]} # 流程类不跑第一次;已发布的那一版按哈希复用,不另存 assert plan["options"]["first_run"] is False summary = {x["step"]: x["text"] for x in plan["summary"]} @@ -255,6 +259,59 @@ async def erp_write(): await registry.refresh_cache(session) +async def test_a_freeform_blueprint_binds_who_does_each_step_of_the_imported_graph(client, erp_write): + """从已发布的「财务审批(岗位 finance)+ 写回(没登记查询)」工作流建自由形蓝图(05-已知遗留 (d)):卡片上按图列出两行 + ——finance 和写入结果不明时的核实人;不选人时发布单在这两行上报 bp.people_unbound,不再有指不到卡片的 app.slot_unbound / + app.reconciler_unbound;选好人之后绑到应用的岗位,发布单不再挡。卡片上只定由谁做,改会签被拒(在画布上改)。""" + from app.blueprints.publish_plan import bindings_of + + graph = {"nodes": [ + s.node("start", "input", "发起", fields=[{"name": "amount", "type": "number"}]), + s.node("review", "human", "财务审批", mode="approve", title="审一下", assignee={"kind": "slot", "slot": "finance"}), + s.node("pay", "tool", "写回报销单", tool=erp_write, args={}), + s.node("done", "output", "完成", fields=[{"name": "结论", "value": "已写回"}])], + "edges": [{"source": "start", "target": "review"}, + {"source": "review", "target": "pay", "sourceHandle": "approved"}, + {"source": "pay", "target": "done"}]} + wf_id = await s.published_workflow(graph, f"写回-{uuid.uuid4().hex[:4]}", description="报销写回") + bp = (await client.post("/api/blueprints", json={"source": {"kind": "workflow_version", "id": wf_id}, + "scope": "shared"})).json() + await s.edit_spec(bp["id"], purpose=s.item("报销审批后写回")) + assert (await client.post(f"/api/blueprints/{bp['id']}/instantiate")).status_code == 200 + out = (await client.get(f"/api/blueprints/{bp['id']}")).json() + rows = out["derived_people"] + assert [(r["kind"], r["slot"], r["path"], r["who"]) for r in rows] == [ + ("slot", "finance", "/people/0/who", None), ("reconciler", "reconciler", "/writeback/reconciler", None)] + assert rows[0]["label"] == "「财务审批」" and rows[0]["detail"] == "「财务审批」处理" + assert rows[1]["detail"] == "「写回报销单」写入结果不明时去核实" + + plan = (await client.post(f"/api/blueprints/{bp['id']}/publish-plan", json={"level": "published"})).json() + assert plan["ok"] is False + codes = [p["code"] for p in plan["problems"]] + assert sorted(p["path"] for p in plan["problems"] if p["code"] == "bp.people_unbound") == [ + "/people/0/who", "/writeback/reconciler"] + assert "app.slot_unbound" not in codes and "app.reconciler_unbound" not in codes + summary_you = [p for p in plan["problems"] if p["code"] == "bp.people_unbound"] + assert all(p["who"] == "you" for p in summary_you) + + refused = await client.patch(f"/api/blueprints/{bp['id']}", json={"ops": [ + {"op": "bp_set", "path": "/people/0/policy", "value": {"kind": "all"}}]}) + assert refused.status_code == 422 and refused.json()["code"] == "bp.template_mismatch" + assert "在画布上改" in refused.json()["detail"] + r = await client.patch(f"/api/blueprints/{bp['id']}", json={"ops": [ + {"op": "bp_set", "path": "/people/0/who", "value": [{"kind": "user", "name": "zhang"}]}, + {"op": "bp_set", "path": "/writeback/reconciler", "value": [{"kind": "group", "name": "财务组"}]}]}) + assert r.status_code == 200, r.text + rows = r.json()["derived_people"] + assert rows[0]["who"] == [{"kind": "user", "name": "zhang"}] and rows[1]["who"] == [{"kind": "group", "name": "财务组"}] + assert not [i for i in r.json()["issues"] if i["code"] == "bp.people_unbound"] + assert bindings_of(await s.blueprint(bp["id"]))["slots"] == { + "finance": [{"kind": "user", "name": "zhang"}], "reconciler": [{"kind": "group", "name": "财务组"}]} + plan = (await client.post(f"/api/blueprints/{bp['id']}/publish-plan", json={"level": "published"})).json() + codes = [p["code"] for p in plan["problems"]] + assert "bp.people_unbound" not in codes and "app.slot_unbound" not in codes and "app.reconciler_unbound" not in codes + + async def test_the_reconciler_comes_from_the_writeback_card(client, erp_write): """审批通过后写回一项没登记查询的能力:结果不明时要人核实。发布单只列一条、指到「写回哪里」卡片上那一项(应用检查说的 核实岗位没绑、写入没人核实是同一件事,不再重复列);在卡片上选了人,绑到应用的核实岗位,这几条都不再有。""" @@ -524,8 +581,8 @@ async def test_problems_are_split_into_three_kinds(client): plan = (await client.post(f"/api/blueprints/{bp['id']}/publish-plan", json={"level": "published"})).json() assert all(p["who"] in ("you", "assistant", "admin") for p in plan["problems"]) assert {p["source"] for p in plan["problems"]} <= {"blueprint", "plan", "workflow", "schedule", "app", "grants"} - unbound = next(p for p in plan["problems"] if p["code"] == "app.slot_unbound") - assert unbound["who"] == "you" and unbound["source"] == "app" + unbound = next(p for p in plan["problems"] if p["code"] == "bp.people_unbound") + assert unbound["who"] == "you" and unbound["source"] == "blueprint" # -------------------------------------------------------------------------- From ba28dee1f05da52d4c1aba01fda03365d2437acb Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Fri, 16 Oct 2026 10:45:23 +0800 Subject: [PATCH 17/39] =?UTF-8?q?=E7=94=9F=E6=88=90=E8=BF=87=E7=A8=8B?= =?UTF-8?q?=E6=8A=BD=E6=88=90=20answer=5Fops=E3=80=81=E5=A4=8D=E6=A0=B8?= =?UTF-8?q?=E6=8A=BD=E6=88=90=20review=5Fof=5Frun=EF=BC=8C=E5=8A=A9?= =?UTF-8?q?=E6=89=8B=E5=BB=BA=E5=9B=BE=E5=92=8C=E5=A4=8D=E6=A0=B8=E7=9A=84?= =?UTF-8?q?=E7=94=A8=E9=87=8F=E4=B9=9F=E8=AE=B0=E4=B8=8B=EF=BC=88=E7=BB=86?= =?UTF-8?q?=E6=A1=88=2001=20=E5=81=9A=E6=B3=95=202=E3=80=813=EF=BC=89?= =?UTF-8?q?=EF=BC=9A=E8=AF=84=E6=B5=8B=E5=92=8C=E9=97=AE=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E9=A1=B5=E8=B5=B0=E5=90=8C=E4=B8=80=E6=9D=A1=E7=94=9F=E6=88=90?= =?UTF-8?q?=E8=B7=AF=E5=BE=84=EF=BC=8C=E8=BF=99=E4=B8=80=E8=BD=AE=E7=9A=84?= =?UTF-8?q?=E8=8A=B1=E8=B4=B9=E4=B8=8D=E5=86=8D=E4=BD=8E=E6=8A=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - api/copilot.py:event_stream 原样搬成 async generator answer_ops(payload, *, catalog_view=None),交出的 op dict 不变;开流前的准备(数据源、目录检查器、助手模型、历史、查看服务)用自己的短会话,在第一条操作之前做完。SSE 路由只包一层:先取第一条,开流前的拒绝(模型没配好、限定的数据源都不在)照旧是 HTTP 错误 - catalog_view 一路传给查看服务和自查的 SQL 检查器(目录视图接缝的「生成过程从参数读」) - providers/usage.py:Meter 是一个 LangChain 回调,和 get_usage_metadata_callback 同一个机制(登记过的上下文变量),按模型累计 tokens、次数,金额按价格目录;估不出的记 unpriced_calls。没直接用 get_usage_metadata_callback:with 块跨 yield 会把变量留在调用方上下文里,调用方随后发起的运行会继承它、被重复计一遍;metered 只在每一步 __anext__ 期间挂上,嵌套时并进外层 - 生成过程读到 done / look / ask 就关模型的流(后面的闲话不要),流末尾才报的用量拿不到:这一次照样计次数和已报的 token,记成估不出,不悄悄少算(遗留:真实接入上建图那几次多半是下限) - providers/budget.py:BudgetFallback.astream 用 aclosing 包住里面那条流——以前调用方关流时里面那条要等垃圾回收才关,模型那次调用的收尾(用量回调)落在算完账之后;测试去掉它回跑会少计一次 - answer_ops 末尾多一条 {op: usage, by_model, input_tokens, output_tokens, calls, cost_usd, unpriced_calls};/copilot/review 拆出 review_of_run(问数据页、评测共用),结果带 usage(没调模型时全 0)和 retry_instruction(要不要带着原图重来、说什么,按 chat/turn.py 的决定算;ReviewIn 多一个 attempt) - 新增 app/chat/turn.py(问数据一轮的决定,复核这边先用上;final.next、对照测试、前端照做在下一个提交);新增 tests/test_copilot_answer_ops.py(八种结局的 SSE 输出和抽出来之前录下的 fixtures/copilot_answer_ops_baseline.json 逐条相同,末尾多一条 usage;直接迭代和 SSE 同一串;开流前的拒绝仍是 HTTP 错误)、tests/test_usage_meter.py;读 SSE 的十一个测试文件的收流帮手跳过末尾那条 usage --- backend/app/api/copilot.py | 211 ++-- backend/app/chat/turn.py | 129 +++ backend/app/providers/budget.py | 16 +- backend/app/providers/usage.py | 204 ++++ .../fixtures/copilot_answer_ops_baseline.json | 1030 +++++++++++++++++ backend/tests/test_catalog_patch.py | 3 +- backend/tests/test_conversations.py | 3 +- backend/tests/test_copilot_answer_ops.py | 165 +++ backend/tests/test_copilot_evidence_guide.py | 3 +- backend/tests/test_copilot_look.py | 3 +- backend/tests/test_copilot_merge.py | 3 +- backend/tests/test_copilot_patch_layout.py | 3 +- backend/tests/test_copilot_prompt_evidence.py | 3 +- backend/tests/test_copilot_scope.py | 3 +- backend/tests/test_copilot_selfcheck.py | 3 +- backend/tests/test_copilot_sqlcheck.py | 3 +- backend/tests/test_copilot_truncation.py | 9 +- backend/tests/test_usage_meter.py | 283 +++++ 18 files changed, 1983 insertions(+), 94 deletions(-) create mode 100644 backend/app/chat/turn.py create mode 100644 backend/app/providers/usage.py create mode 100644 backend/tests/fixtures/copilot_answer_ops_baseline.json create mode 100644 backend/tests/test_copilot_answer_ops.py create mode 100644 backend/tests/test_usage_meter.py diff --git a/backend/app/api/copilot.py b/backend/app/api/copilot.py index 2f8766a5..469e5864 100644 --- a/backend/app/api/copilot.py +++ b/backend/app/api/copilot.py @@ -110,11 +110,12 @@ def _tool_catalog(rows: list[Any], *, builtins: frozenset[str] | None = None) -> async def _look_service(session: AsyncSession, payload: GenerateIn, sources: list[Any], *, graph: dict[str, Any] | None, kinds: tuple[str, ...] = ("table", "capability"), - ) -> Any: - """这一轮的查看服务(copilot_context.look_service)。挑选模型用同一个助手模型,第一次要挑时才拿。""" + catalog_view: Any = None) -> Any: + """这一轮的查看服务(copilot_context.look_service)。挑选模型用同一个助手模型,第一次要挑时才拿。catalog_view:表详情 + 里给哪一份数据目录(data/catalog_view.py),不给是当前目录。""" spec = await copilot_model_spec(session, payload, max_tokens=copilot_context.PICK_MAX_TOKENS) return await copilot_context.look_service(sources, spec=spec, graph=graph, kinds=kinds, - answer=payload.intent == "answer") + answer=payload.intent == "answer", catalog_view=catalog_view) NODE_REFERENCE = """\ @@ -1756,45 +1757,79 @@ def _repair_request( @router.post("/generate-stream") -async def generate_stream(payload: GenerateIn, session: AsyncSession = Depends(get_session)): +async def generate_stream(payload: GenerateIn): """流式版生成:SSE 逐操作推送,前端边收边把节点摆上画布。 - 生成过程(copilot_gen)读到 look 就停下、查看、接着写,查看记录作为 look 事件发给前端(前端存进这一轮的 - meta.looks)。结束时(done 或流断)发一条 final:服务端把累积的图排版、校验后整体给出——过程可见性来自操作流, - 最终质量仍由和手工编排相同的 layout + validate 保证。模型问了用户(ask)的,这一轮到此为止,不自查修正、不自动运行。 + 生成过程整个在 answer_ops 里(评测直接迭代它,和问数据页走同一条路),这里只包一层 SSE。开流前的拒绝(模型没配好、 + 对话读不了、限定的数据源都不在)发生在 answer_ops 交出第一条操作之前:先取第一条,拒绝照旧是 HTTP 错误,不进流。 """ from fastapi.responses import StreamingResponse - from app.data.sqlcheck import load_checkers + ops = answer_ops(payload) + first = await ops.__anext__() + async def event_stream(): + yield _sse(first) + async for op in ops: + yield _sse(op) + + return StreamingResponse( + event_stream(), + media_type="text/event-stream", + headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, + ) + + +async def answer_ops(payload: GenerateIn, *, catalog_view: Any = None): + """生成过程:按需求起草、按需查看、自查修正、排版校验,逐条交出操作(op dict)。问数据页(经 SSE)和评测(直接迭代) + 走的是这同一条路(细案 01 做法 2)。 + + 读到 look 就停下、查看、接着写,查看记录作为 look 操作交出(前端存进这一轮的 meta.looks)。结束时(done 或流断)交一条 + final:累积的图排版、校验后整体给出——过程可见性来自操作流,最终质量仍由和手工编排相同的 layout + validate 保证。模型 + 问了用户(ask)的,这一轮到此为止,不自查修正、不自动运行。 + + 开流前的准备(取数据源、读目录检查器、取助手模型、拼历史、组查看服务)用自己的短会话,在交出第一条操作之前做完: + 拒绝在这里抛 HTTPException(调用方没迭代到第一条就知道)。catalog_view 是这一轮读哪一份数据目录(data/catalog_view.py: + 查看的表详情、自查的 SQL 检查器都按它),不给是当前目录。 + + 最后多交一条 usage(providers/usage.py):{op: usage, by_model, input_tokens, output_tokens, calls, cost_usd, + unpriced_calls}——这一轮助手建图(含挑表、接着写、自查修正)花了多少。前面每一条和抽出来之前逐条一样。运行不在这里, + 不计。 + """ from app.api.deps import restricted + from app.data.catalog_view import CatalogView + from app.data.sqlcheck import load_checkers + from app.db.base import SessionLocal + from app.providers import usage - # 带了对话的,先核对这个人能不能用它(D-d):历史、上一轮的图都从它来 - await _ensure_conversation_access(session, payload.conversation_id) - # 开登录后的 user 角色(D-c):图只能用 chat_policy 放行的节点和工具,提示词先说、自查按同一个判断挡 - chat_user = restricted() - sources = await _sources(session, payload.datasource_ids) - scope = {r.name for r in sources} if payload.datasource_ids else None - # 基于数据目录的 SQL 检查器:在开流之前读好目录(流里不再碰这个会话),自查和收尾共用 - checkers = await load_checkers(session, sources) - nodes, edges = _start_graph(payload) - # 用户摆好的坐标,收尾排版时原样留着 - pinned = _pinned_positions(payload.base_graph) + view = CatalogView.parse(catalog_view) + async with SessionLocal() as session: + # 带了对话的,先核对这个人能不能用它(D-d):历史、上一轮的图都从它来 + await _ensure_conversation_access(session, payload.conversation_id) + # 开登录后的 user 角色(D-c):图只能用 chat_policy 放行的节点和工具,提示词先说、自查按同一个判断挡 + chat_user = restricted() + sources = await _sources(session, payload.datasource_ids) + scope = {r.name for r in sources} if payload.datasource_ids else None + # 基于数据目录的 SQL 检查器:在开流之前读好目录(流里不再碰这个会话),自查和收尾共用 + checkers = await load_checkers(session, sources, catalog_view=view) + nodes, edges = _start_graph(payload) + # 用户摆好的坐标,收尾排版时原样留着 + pinned = _pinned_positions(payload.base_graph) - try: - model, model_id = await _op_model(session, payload) - except ProviderNotConfigured as e: - raise HTTPException(400, _unconfigured(e)) from e + try: + model, model_id = await _op_model(session, payload) + except ProviderNotConfigured as e: + raise HTTPException(400, _unconfigured(e)) from e - human = _with_history(_graph_request(payload), await _history_section(session, payload.conversation_id)) - # 兜底用:模型只发了改动操作时,把它们重放到这张图上 - prev_graph = await _previous_graph(session, payload.conversation_id) - # 比对工具绑定的基准:画布上是改之前的图;问数据页是上一轮的图(沿用它的节点 id) - baseline = copy.deepcopy( - (payload.base_graph or {}).get("nodes") or (prev_graph or {}).get("nodes") or []) - # 查看服务:读目录、读能力登记在开流之前做完。现有工作流(画布上的原图、问数据追问时上一轮的图)查过的表列在概览里 - looks = await _look_service(session, payload, sources, graph=payload.base_graph if ( - payload.base_graph or {}).get("nodes") else prev_graph) + human = _with_history(_graph_request(payload), await _history_section(session, payload.conversation_id)) + # 兜底用:模型只发了改动操作时,把它们重放到这张图上 + prev_graph = await _previous_graph(session, payload.conversation_id) + # 比对工具绑定的基准:画布上是改之前的图;问数据页是上一轮的图(沿用它的节点 id) + baseline = copy.deepcopy( + (payload.base_graph or {}).get("nodes") or (prev_graph or {}).get("nodes") or []) + # 查看服务:读目录、读能力登记在开流之前做完。现有工作流(画布上的原图、问数据追问时上一轮的图)查过的表列在概览里 + looks = await _look_service(session, payload, sources, graph=payload.base_graph if ( + payload.base_graph or {}).get("nodes") else prev_graph, catalog_view=view) # 这一份 system 一直用到最后:接着写、自查修正都沿用,不换 system = _graph_system(sources, scope, await copilot_context.section(looks), looks, chat_user=chat_user) applier = GraphApplier(nodes, edges) @@ -1803,8 +1838,8 @@ async def generate_stream(payload: GenerateIn, session: AsyncSession = Depends(g checker=GraphChecker(applier, scope=scope, sources=sources, baseline=baseline, checkers=checkers, chat_user=chat_user)) - async def event_stream(): - yield _sse({"op": "model", "model": model_id}) + async def steps(): + yield {"op": "model", "model": model_id} patches = 0 try: async for op in _with_heartbeat(gen.draft(human), started=started): @@ -1813,23 +1848,23 @@ async def generate_stream(payload: GenerateIn, session: AsyncSession = Depends(g # reply 先压着(applier.replied):前端收到 reply 就收尾,提案得排在它前面转出 if (checked := await catalog_patch_op(op, sources)) is not None: patches += 1 - yield _sse(checked) + yield checked continue - yield _sse(op) + yield op if applier.replied is not None: # 这句话不需要工作流:后面那套排版校验对着一张空图跑,只会得到"图是空的,先拖一个节点进来" - yield _sse(applier.replied) + yield applier.replied return if gen.asked is not None and not applier.effective: # 先问清楚才能动手,什么都还没改:这一轮到此为止。照样发一条 final 收尾,带上问题(asked)和没动过的图: # 前端据此把这一轮记成「助手反问」、问题进历史(以前流就这么断了,问数据页一直停在「进行中」), # 不运行、不重排。图给一份对象(画布上原来的图、问数据上一轮的图,都没有就是空图),老前端照样认得 - yield _sse(_asked_final(gen.asked, payload.base_graph, prev_graph)) + yield _asked_final(gen.asked, payload.base_graph, prev_graph) return # 只发了目录修改提案,既没回答也没改图(用户只是在说一条数据事实):补一句回答收尾, # 不能按下面的「模型没有给出任何修改」报错——提案已经摆在用户面前了 if patches and all(o.get("op") == "plan" for o in applier.seen): - yield _sse({"op": "reply", "text": PATCH_ONLY_REPLY}) + yield {"op": "reply", "text": PATCH_ONLY_REPLY} return # 流正常结束,却没有 done、也没有一个操作能认(格式跑偏,每一行都被跳过): # 往下走就是拿原图去自查、交付,界面上写「无需修改」——而模型什么都没说 @@ -1837,12 +1872,11 @@ async def generate_stream(payload: GenerateIn, session: AsyncSession = Depends(g raise CopilotIncomplete("模型没有给出任何修改,也没有说明原因", hint="画布未修改。可以重试,或把需求描述得更具体") except CopilotIncomplete as e: - yield _sse({"op": "error", "message": f"助手本轮未完成:{e}", "hint": e.hint, "detail": e.detail}) + yield {"op": "error", "message": f"助手本轮未完成:{e}", "hint": e.hint, "detail": e.detail} return except Exception as e: # noqa: BLE001 reason, hint = explain_error(e) - yield _sse({"op": "error", "message": f"助手本轮未完成:{reason}", - "hint": hint, "detail": raw_error(e)}) + yield {"op": "error", "message": f"助手本轮未完成:{reason}", "hint": hint, "detail": raw_error(e)} return # 兜底:一个操作都没落地,而上一轮有图。 @@ -1858,7 +1892,7 @@ async def generate_stream(payload: GenerateIn, session: AsyncSession = Depends(g # 以前这些问题照样交付、照样自动开跑——跑到那一步才炸(开发库里四次运行死在 # 循环 / 分支条件里套了 {{ }}),前面的步骤白跑,报错还停在半路 async for op in _with_heartbeat(gen.repair(), phase="repairing"): - yield _sse(op) + yield op remaining = _blocking_issues(nodes, edges, scope, sources=sources, baseline=baseline, checkers=checkers, chat_user=chat_user) # 工具绑定变化不挡运行,但要在自查这一步就说出来:工具被改没了的图照样 @@ -1871,11 +1905,9 @@ async def generate_stream(payload: GenerateIn, session: AsyncSession = Depends(g # 改不好也照实交付、列出问题,但不自动运行:明知跑不起来的图,开跑只会 # 在半路报一个用户看不懂的错 autorun = False - yield _sse({"op": "check", "status": "failed", "issues": remaining, - "warnings": warnings}) + yield {"op": "check", "status": "failed", "issues": remaining, "warnings": warnings} elif remaining is not None: - yield _sse({"op": "check", "status": "passed", "repaired": gen.repaired, - "warnings": warnings}) + yield {"op": "check", "status": "passed", "repaired": gen.repaired, "warnings": warnings} # 收尾:排版 + 校验,把最终图整体交付。改图时旧节点留在原处,只排新节点 try: @@ -1916,13 +1948,12 @@ async def generate_stream(payload: GenerateIn, session: AsyncSession = Depends(g except Exception as e: # noqa: BLE001 final = {"op": "error", "message": f"模型返回的工作流结构有误:{graph_error(e)}", "hint": "请重试,或把需求描述得更具体", "detail": raw_error(e)} - yield _sse(final) + yield final - return StreamingResponse( - event_stream(), - media_type="text/event-stream", - headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, - ) + spent = usage.meter() + async for op in usage.metered(steps(), spent): + yield op + yield {"op": "usage", **spent.summary()} # -------------------------------------------------------------------------- @@ -2774,13 +2805,31 @@ class ReviewIn(BaseModel): #: 默认只在扫出异常信号时才叫模型。打开它就每次都复核——排查时有用, #: 平时不开:干净的运行没什么可复核的,代价却是每次问数都多一次调用 always: bool = False + #: 这一轮已经自动试了几次(从 0 数,同 chat/turn.retry_after_review):结果里的 retry_instruction 据此给不给。 + #: 不传按第一次算 + attempt: int = Field(default=0, ge=0) @router.post("/review") async def review_run( payload: ReviewIn, session: AsyncSession = Depends(get_session) ) -> dict[str, Any]: - """接住一次编排的产出,判断它能不能就这么交给用户。 + """接住一次编排的产出,判断它能不能就这么交给用户(规则见 review_of_run)。这里只核对能不能读这次运行。""" + from app.api.deps import PRIVATE_RUN, can_read_run, current_principal + from app.db.models import Run + + run = await session.get(Run, payload.run_id) + if not run: + raise HTTPException(404, "运行记录不存在,可能已被删除") + if not await can_read_run(current_principal(), run): + raise HTTPException(403, PRIVATE_RUN) + return await review_of_run(session, run, question=payload.question, provider=payload.provider, + model=payload.model, always=payload.always, attempt=payload.attempt) + + +async def review_of_run(session: AsyncSession, run: Any, *, question: str, provider: str | None = None, + model: str | None = None, always: bool = False, attempt: int = 0) -> dict[str, Any]: + """复核一次运行(问数据页经 /copilot/review、评测直接调,同一个函数)。 「跑完了」和「答得对」是两回事。运行状态只说明流程走完了,而过程中检索 降级、工具报错、agent 步数用满这些事,原先一件都不会进到答案里——用户拿到 @@ -2788,47 +2837,57 @@ async def review_run( 规则层先扫一遍(engine/review.py:scan),没扫出东西就到此为止、直接返回, 不花一次模型调用;扫出来了才叫模型重组答案并把异常说破。 - """ - from app.api.deps import PRIVATE_RUN, can_read_run, current_principal - from app.db.models import Run, RunEvent - from app.engine import review as rv - run = await session.get(Run, payload.run_id) - if not run: - raise HTTPException(404, "运行记录不存在,可能已被删除") - if not await can_read_run(current_principal(), run): - raise HTTPException(403, PRIVATE_RUN) + 结果是 ReviewResult.as_dict(),另带两项: + - retry_instruction:要不要带着原图重来一次(chat/turn.retry_after_review,按 attempt 算这一轮还能不能再试)——要的话 + 是交给助手的那句话(chat/turn.retry_instruction),不要是 null。问数据页照它走; + - usage:复核这一次模型调用花了多少(providers/usage.py 的汇总,没调模型时是全 0),这一轮的花费 = 助手 + 运行 + 复核。 + """ + from app.chat import turn + from app.db.models import RunEvent + from app.engine import review as rv + from app.providers import usage rows = list((await session.execute( - select(RunEvent).where(RunEvent.run_id == payload.run_id).order_by(RunEvent.seq) + select(RunEvent).where(RunEvent.run_id == run.id).order_by(RunEvent.seq) )).scalars()) output = run.output if isinstance(run.output, dict) else None signals = rv.scan(rows, output) answer = rv.answer_text(output) + spent = usage.meter() - if not signals and not payload.always: - return rv.ReviewResult(verdict="ok", signals=[]).as_dict() + def done(result: Any) -> dict[str, Any]: + out = result.as_dict() + out["retry_instruction"] = ( + turn.retry_instruction(question, turn.review_reason(out), turn.has_nodes(run.graph)) + if turn.retry_after_review(out, attempt) else None) + out["usage"] = spent.summary() + return out + + if not signals and not always: + return done(rv.ReviewResult(verdict="ok", signals=[])) try: spec = await copilot_model_spec(session, GenerateIn( - instruction="review", provider=payload.provider, model=payload.model, + instruction="review", provider=provider, model=model, ), max_tokens=4096) - model, _ = await get_chat_model(session, spec) + chat_model, _ = await get_chat_model(session, spec) except (ProviderNotConfigured, HTTPException) as e: # 没配模型不该让这一轮挂掉:规则层已经知道出了什么事,照样说得出话 - return rv.ReviewResult( + return done(rv.ReviewResult( verdict="annotated", note=";".join(s.detail for s in signals) or ( _unconfigured(e) if isinstance(e, ProviderNotConfigured) else str(e.detail) ), retry=rv.should_retry(signals), signals=signals, severity=rv.worst(signals), - ).as_dict() + )) - result = await rv.review( - model, question=payload.question, answer=answer, signals=signals, - # 报告撰写节点产出的答案逐段对应着证据文档:只能加说明,不能改写 - locked=bool(output and output.get("_evidence")), - ) - return result.as_dict() + with usage.metering(spent): + result = await rv.review( + chat_model, question=question, answer=answer, signals=signals, + # 报告撰写节点产出的答案逐段对应着证据文档:只能加说明,不能改写 + locked=bool(output and output.get("_evidence")), + ) + return done(result) diff --git a/backend/app/chat/turn.py b/backend/app/chat/turn.py new file mode 100644 index 00000000..993c0ad5 --- /dev/null +++ b/backend/app/chat/turn.py @@ -0,0 +1,129 @@ +"""问数据一轮里的几个「决定」(细案 01 做法 1,总方案 2.1)。 + +一轮问数据是:建图(助手的生成过程,api/copilot.answer_ops)→ 要不要运行 → 运行(输入怎么填)→ 复核 → 要不要带着原图 +重来一次、重来时说什么。以前这几个决定都写在问数据页里(frontend/src/store/chat.ts 的 plan / launch / settle / retry), +后端跑不了一轮问数据:评测要么照着抄一份(迟早走样,量出来的不是用户拿到的那一轮),要么做不了;这也违反「界面层不放 +逻辑」。现在决定写在这里,是纯函数: + +- next_step:生成过程交出结局之后做什么; +- fill_input:用户的问题填进这张图的入口字段; +- retry_after_review:复核之后要不要带着原图重来一次; +- retry_instruction:重来时交给助手的那句话(两种措辞)。 + +接口把决定交给问数据页:/copilot/generate-stream 的 final 带 next(run | ready | fail),/copilot/review 的结果带 +retry_instruction(要重来时是那句话,不重来是 null)。问数据页照后端给的走;老后端没有这两个字段时退回它自己那份同样的 +规则(tests/test_chat_turn_policy.py 逐条对照两边的常量和措辞)。评测(第 3 波)用同一组函数把一轮串起来。 + +只搬「决定」:重连、断点续看、取消、复核等多久这些仍在问数据页(把整轮编排搬到后端要重做这些,单独立项)。 +""" +from __future__ import annotations + +import re +from collections.abc import Mapping +from typing import Any + +#: 一轮最多自动试几次(含第一次):第二次还不行多半是问题本身没说清,该让用户介入(同 chat.ts 的 MAX_ATTEMPTS) +MAX_ATTEMPTS = 2 + +#: final 之后的三种走法(final.next):运行、只建不跑(摆着等用户点运行)、判失败(没过检查的图不拿去跑) +NEXT_RUN, NEXT_READY, NEXT_FAIL = "run", "ready", "fail" +NEXT_STEPS = (NEXT_RUN, NEXT_READY, NEXT_FAIL) +#: 别的结局:直接回答了(reply)、助手反问了等用户答(asked)、没自查过的老服务端交来没过检查的图时重来一次(retry) +NEXT_REPLY, NEXT_ASKED, NEXT_RETRY = "reply", "asked", "retry" + +#: 问题往哪个入口字段放:名字像「问题」的优先(同 chat.ts fillInput 的正则,大小写不敏感) +QUESTION_FIELD = re.compile(r"question|query|topic|input|问题|需求|主题", re.IGNORECASE) + +#: 带着原图重来时交给助手的话:原图交回去改(调大步数、改参数这种事就是改一个字段,不是推倒重来) +RETRY_WITH_BASE = ("这个工作流上一次运行没有成功:{reason}。\n\n" + "请针对这个原因改它(比如调大最大步数、改正工具参数、换个查询方式)," + "改完继续回答原来的问题:{question}") +#: 没有图可交回(上一次连图都没建出来)时:原问题后面缀上原因,从零再建 +RETRY_FROM_SCRATCH = "{question}\n\n(上一次没有成功:{reason}。请针对这个原因调整,重新给出完整的工作流。)" + + +def _blockers(final: Mapping[str, Any]) -> list[str]: + return [str(i.get("message") or "") for i in final.get("issues") or [] + if isinstance(i, Mapping) and i.get("level") == "error"] + + +def next_step(terminal: Mapping[str, Any] | None, *, self_checked: bool = True) -> str: + """生成过程交出结局之后做什么。terminal 是最后那条结局操作(final / reply / error);流结束时只有反问(ask)、没有 + 别的结局的给 None。 + + - reply:直接回答,这一轮到此为止;error:判失败;只有反问:等用户回答; + - final:有 error 级问题时,服务端已经自查过(收到过 check 的结论)就判失败——没过的问题已经交回模型在同一轮里改过 + 了,再整轮重来是再烧一次完整生成、改的还是同一批问题;没自查过(老服务端)才重来一次(retry);模型说不运行 + (autorun=false:用户要的是流程本身、或者问了用户)只建不跑;其余运行。 + """ + if terminal is None: + return NEXT_ASKED + kind = terminal.get("op") + if kind == "reply": + return NEXT_REPLY + if kind != "final": + return NEXT_FAIL + if _blockers(terminal): + return NEXT_FAIL if self_checked else NEXT_RETRY + if terminal.get("autorun") is False: + return NEXT_READY + return NEXT_RUN + + +def blocked_reason(final: Mapping[str, Any]) -> str: + """没过检查的图重来时的原因(只有老服务端那条路用得到)。""" + return "生成的工作流没有通过校验:" + ";".join(_blockers(final)) + + +def fill_input(graph: Mapping[str, Any] | None, question: str, history: str = "") -> dict[str, Any]: + """用户的问题填进这张图的入口字段(同 chat.ts 的 fillInput)。 + + 入口没声明字段:{question, history}(下游可能用 input.question)。声明了:每个字段先填默认值(没有就空串),问题放进 + 名字像「问题」的那个,其次第一个必填的,再不然第一个;history(前几轮压成的文字)照样带上——图跑起来时模型得知道 + 「它们」指的是上一轮那批数据。 + """ + nodes = (graph or {}).get("nodes") or [] + entry = next((n for n in nodes if isinstance(n, Mapping) and n.get("type") == "input"), None) + config = ((entry or {}).get("data") or {}).get("config") or {} + fields = [f for f in (config.get("fields") or []) if isinstance(f, Mapping)] + if not fields: + return {"question": question, "history": history} + payload: dict[str, Any] = {"history": history} + for f in fields: + if f.get("name"): + payload[str(f["name"])] = f["default"] if f.get("default") is not None else "" + named = next((f for f in fields if QUESTION_FIELD.search(str(f.get("name") or ""))), None) + target = named or next((f for f in fields if f.get("required")), None) or fields[0] + if target.get("name"): + payload[str(target["name"])] = question + return payload + + +def review_reason(review: Mapping[str, Any]) -> str: + """复核判不可用时重来的原因:各条信号的说明,没有就用给用户看的那句说明。""" + details = [str(s.get("detail") or "") for s in review.get("signals") or [] if isinstance(s, Mapping)] + return ";".join(d for d in details if d) or str(review.get("note") or "") + + +def retry_after_review(review: Mapping[str, Any] | None, attempt: int) -> bool: + """复核之后要不要带着原图重来一次:复核说换个配置重跑大概率就好(retry)、答案不可信(severity=broken)、这一轮还没 + 试满(attempt 从 0 数,attempt + 1 < MAX_ATTEMPTS)。""" + if not review: + return False + return bool(review.get("retry")) and review.get("severity") == "broken" and attempt + 1 < MAX_ATTEMPTS + + +def retry_instruction(question: str, reason: str, has_base: bool) -> str: + """重来时交给助手的话。has_base:有上一次建出的图可以交回去改。""" + template = RETRY_WITH_BASE if has_base else RETRY_FROM_SCRATCH + return template.format(question=question, reason=reason) + + +def has_nodes(graph: Any) -> bool: + """一张图有没有节点(有才交回去改)。""" + return isinstance(graph, Mapping) and bool(graph.get("nodes")) + + +__all__ = ["MAX_ATTEMPTS", "NEXT_ASKED", "NEXT_FAIL", "NEXT_READY", "NEXT_REPLY", "NEXT_RETRY", "NEXT_RUN", "NEXT_STEPS", + "QUESTION_FIELD", "RETRY_FROM_SCRATCH", "RETRY_WITH_BASE", "blocked_reason", "fill_input", "has_nodes", + "next_step", "retry_after_review", "retry_instruction", "review_reason"] diff --git a/backend/app/providers/budget.py b/backend/app/providers/budget.py index 3d52402a..78b31f31 100644 --- a/backend/app/providers/budget.py +++ b/backend/app/providers/budget.py @@ -18,6 +18,7 @@ engine/nodes/llm.py(cite_fields 的抽取)、data/recipe_ai.py 各有一份 from __future__ import annotations from collections.abc import Awaitable, Callable, Hashable +from contextlib import aclosing from typing import Any #: 服务方嫌额度太大时报错里会出现的字样:超过模型的输出上限(DeepSeek chat 系只到 8K), @@ -167,19 +168,24 @@ class BudgetFallback: return await (await self._current()).ainvoke(messages, *args, **kwargs) async def astream(self, messages: Any, *args: Any, **kwargs: Any): + # 里面那条流用 aclosing 包着:调用方读到一半不读了(助手读到 done、look 就关流),关这一层时里面那条也当场关掉、 + # 模型那一次调用当场收尾(LangChain 的回调这时把已经收到的用量交出来,providers/usage.py)。不包的话里面那条要等 + # 垃圾回收才关,收尾落在调用方算完账之后 used = await self._current() started = False try: - async for chunk in used.astream(messages, *args, **kwargs): - started = True - yield chunk + async with aclosing(used.astream(messages, *args, **kwargs)) as stream: + async for chunk in stream: + started = True + yield chunk return except Exception as e: if started or not self._can_fall_back(used) or not self._switch.rejected(e): raise self._switch.fall_back() - async for chunk in (await self._current()).astream(messages, *args, **kwargs): - yield chunk + async with aclosing((await self._current()).astream(messages, *args, **kwargs)) as stream: + async for chunk in stream: + yield chunk def _derived(self, make: Callable[[Any], Any]) -> BudgetFallback: """对大小两个模型做同一件事(绑工具、要结构化输出),包成共用开关的一层。小的还没建时,建好再做。""" diff --git a/backend/app/providers/usage.py b/backend/app/providers/usage.py new file mode 100644 index 00000000..37563e6b --- /dev/null +++ b/backend/app/providers/usage.py @@ -0,0 +1,204 @@ +"""模型用量计量:助手建图(含挑表、接着写、自查修正)和复核这两段花了多少(细案 01 做法 3)。 + +**为什么要。** 运行按 llm.end 事件记用量,问数据一轮里另外两块——助手建图、复核——一直不记:这一轮的花费是低报的, +评测也算不出按角色的成本。在每个调用点手写累计(生成、挑表、退回小额度重来、复核各一处)迟早漏掉一处,所以用 LangChain +的回调在模型这一层统一截:和 langchain_core.callbacks.get_usage_metadata_callback 同一个机制(一个登记过的上下文变量, +模型每次调用时把它挂成回调),按模型累计 input / output tokens 和调用次数。 + +**为什么不直接用 get_usage_metadata_callback。** 它是一个 with 块,块里设上下文变量、出块时按 token 复原。生成过程是一个 +async generator:with 块跨着 yield 的话,变量在两次 yield 之间留在调用方的上下文里——调用方这时去发起运行,运行的任务 +会继承它,运行里的调用就被重复计一遍;调用方中途不迭代了、在别的任务里收尾,复原还会报「token 不属于这个上下文」。 +这里只在每一步 __anext__ 期间设上变量、交出操作之前复原(metered):生成过程里起的任务(心跳那一层每一步一个任务) +在创建时拷走上下文,照样计到;调用方自己的代码一行都不在计量里。 + +**金额。** 按 providers/catalog.estimate_cost 算,和运行的 llm.end 一个口径。估不出的调用记 unpriced_calls:模型不在价格 +目录里,或者用量没报全而目录价又不是 0。估不出的不当成 0 元,调用方据此写「≥」或「金额无法估算」。用量没报全有两种: +回复没带用量(有的网关流式不给);读到一半就不读了——生成过程读到 done、look、ask 那一行就关掉模型的流(后面的闲话不要、 +也不再为它付钱),用量在流的末尾才报全(Anthropic 的输出 token 在最后一条,OpenAI 兼容系开了 stream_usage 也在最后), +关掉的那一次只拿得到已经报了的部分。这一次照样计次数、计已报的 token,记成估不出,不悄悄少算。 + +模型名按发起调用时的参数取(invocation_params 的 model / model_name,就是接入里配的模型 id,价格目录按它查),取不到再 +看回复里的 model_name。 +""" +from __future__ import annotations + +import threading +from collections.abc import AsyncIterator, Iterator +from contextlib import contextmanager +from contextvars import ContextVar +from typing import Any +from uuid import UUID + +from langchain_core.callbacks import BaseCallbackHandler +from langchain_core.tracers.context import register_configure_hook + +#: 取不到模型名的调用记在这个名下 +UNKNOWN_MODEL = "unknown" + + +class Meter(BaseCallbackHandler): + """一次计量:按模型累计用量。线程安全(同步模型的回调可能在线程里来)。""" + + #: 同步模型在线程里调用时也要回调 + run_inline = True + + def __init__(self) -> None: + super().__init__() + self._lock = threading.Lock() + self._pending: dict[UUID, str] = {} + self._models: dict[str, dict[str, int]] = {} + + # ------------------------------------------------------------------ 回调 + + def _started(self, run_id: UUID, serialized: dict[str, Any] | None, kwargs: dict[str, Any]) -> None: + params = kwargs.get("invocation_params") or {} + model = params.get("model") or params.get("model_name") or params.get("model_id") + if not model and isinstance(serialized, dict): + model = (serialized.get("kwargs") or {}).get("model") or (serialized.get("kwargs") or {}).get("model_name") + with self._lock: + self._pending[run_id] = str(model) if model else "" + + def on_chat_model_start(self, serialized: dict[str, Any], messages: Any, *, run_id: UUID, + **kwargs: Any) -> None: + self._started(run_id, serialized, kwargs) + + def on_llm_start(self, serialized: dict[str, Any], prompts: Any, *, run_id: UUID, **kwargs: Any) -> None: + self._started(run_id, serialized, kwargs) + + def on_llm_end(self, response: Any, *, run_id: UUID, **kwargs: Any) -> None: + message = None + try: + message = response.generations[0][0].message + except (AttributeError, IndexError, TypeError): + pass + usage = getattr(message, "usage_metadata", None) or None + meta = getattr(message, "response_metadata", None) or {} + with self._lock: + model = self._pending.pop(run_id, "") or str(meta.get("model_name") or meta.get("model") or "") \ + or UNKNOWN_MODEL + row = self._models.setdefault(model, {"input_tokens": 0, "output_tokens": 0, "calls": 0, "unreported": 0}) + row["calls"] += 1 + if usage: + row["input_tokens"] += int(usage.get("input_tokens") or 0) + row["output_tokens"] += int(usage.get("output_tokens") or 0) + else: + row["unreported"] += 1 + + def on_llm_error(self, error: BaseException, *, run_id: UUID, **kwargs: Any) -> None: + """流读到一半停了(生成过程读到 done / look / ask 就关流,或者连接断了):LangChain 照样交来已经收到的那几段。 + 有收到东西的照计一次、记已报的 token,算用量没报全;一段都没收到的(请求刚发出去就被拒了)不计。""" + from langchain_core.outputs import ChatGenerationChunk + + response = kwargs.get("response") + partial = None + try: + first = response.generations[0][0] if response is not None and response.generations[0] else None + partial = first.message if isinstance(first, ChatGenerationChunk) else None + except (AttributeError, IndexError, TypeError): + partial = None + with self._lock: + model = self._pending.pop(run_id, "") + if partial is None: + return + meta = getattr(partial, "response_metadata", None) or {} + model = model or str(meta.get("model_name") or meta.get("model") or "") or UNKNOWN_MODEL + row = self._models.setdefault(model, {"input_tokens": 0, "output_tokens": 0, "calls": 0, "unreported": 0}) + row["calls"] += 1 + row["unreported"] += 1 + reported = getattr(partial, "usage_metadata", None) or {} + row["input_tokens"] += int(reported.get("input_tokens") or 0) + row["output_tokens"] += int(reported.get("output_tokens") or 0) + + # ------------------------------------------------------------------ 合并、汇总 + + def merge(self, other: Meter) -> None: + """把另一次计量并进来(嵌套的计量出块时并进外层)。""" + with other._lock: + rows = {k: dict(v) for k, v in other._models.items()} + with self._lock: + for model, row in rows.items(): + mine = self._models.setdefault(model, dict.fromkeys(row, 0)) + for key, value in row.items(): + mine[key] = mine.get(key, 0) + value + + def summary(self) -> dict[str, Any]: + """{by_model: {模型: {input_tokens, output_tokens, calls, cost_usd, unpriced_calls}}, input_tokens, output_tokens, + calls, cost_usd, unpriced_calls}。cost_usd 只是估得出的那部分。""" + from app.providers.catalog import estimate_cost, find_pricing + + with self._lock: + rows = {k: dict(v) for k, v in self._models.items()} + by_model: dict[str, dict[str, Any]] = {} + for model, row in sorted(rows.items()): + pricing = find_pricing(model) + free = pricing is not None and not any(pricing.values()) + if pricing is None: + unpriced = row["calls"] + else: + unpriced = 0 if free else row["unreported"] + by_model[model] = {"input_tokens": row["input_tokens"], "output_tokens": row["output_tokens"], + "calls": row["calls"], + "cost_usd": round(estimate_cost(model, row["input_tokens"], row["output_tokens"]), 6), + "unpriced_calls": unpriced} + total = {key: sum(r[key] for r in by_model.values()) + for key in ("input_tokens", "output_tokens", "calls", "unpriced_calls")} + return {"by_model": by_model, **total, "cost_usd": round(sum(r["cost_usd"] for r in by_model.values()), 6)} + + +#: 计量挂在这个上下文变量上(inheritable:模型调用时 LangChain 按它把 Meter 挂成回调,子任务拷走上下文时一并带走) +_CURRENT: ContextVar[Meter | None] = ContextVar("agentlab_usage_meter", default=None) +register_configure_hook(_CURRENT, inheritable=True) + + +def meter() -> Meter: + """新的一次计量。用 metering(一段代码)或 metered(一个生成过程)把它挂上。""" + return Meter() + + +def current() -> Meter | None: + return _CURRENT.get() + + +@contextmanager +def metering(m: Meter) -> Iterator[Meter]: + """这一段代码里的模型调用计进 m。只给不跨 yield 的一段用(复核那一次调用);生成过程用 metered。 + + 嵌套的:出块时并进外层的那一次,外层照样看得到。复原用 set 而不是按 token reset:在别的任务里收尾也不报错。""" + outer = _CURRENT.get() + _CURRENT.set(m) + try: + yield m + finally: + _CURRENT.set(outer) + if outer is not None and outer is not m: + outer.merge(m) + + +async def metered(source: AsyncIterator[Any], m: Meter) -> AsyncIterator[Any]: + """迭代一个生成过程,它的模型调用计进 m:每一步 __anext__ 期间挂上,交出操作之前复原(见模块开头)。 + + 开始时外层已经有一次计量的(评测把一整题的助手用量记成一份):结束时把 m 并进去,同 metering。""" + enclosing = _CURRENT.get() + iterator = source.__aiter__() + try: + while True: + outer = _CURRENT.get() + _CURRENT.set(m) + try: + item = await iterator.__anext__() + except StopAsyncIteration: + return + finally: + _CURRENT.set(outer) + yield item + finally: + if enclosing is not None and enclosing is not m: + enclosing.merge(m) + + +def empty() -> dict[str, Any]: + """一次调用都没有时的汇总(同 Meter().summary())。""" + return Meter().summary() + + +__all__ = ["Meter", "UNKNOWN_MODEL", "current", "empty", "meter", "metered", "metering"] diff --git a/backend/tests/fixtures/copilot_answer_ops_baseline.json b/backend/tests/fixtures/copilot_answer_ops_baseline.json new file mode 100644 index 00000000..f1de6562 --- /dev/null +++ b/backend/tests/fixtures/copilot_answer_ops_baseline.json @@ -0,0 +1,1030 @@ +{ + "graph_ok": [ + { + "op": "model", + "model": "mock-fast" + }, + { + "op": "plan", + "summary": "数一数" + }, + { + "op": "add_node", + "node": { + "id": "start", + "type": "input", + "label": "start", + "config": { + "fields": [ + { + "name": "question" + } + ] + } + } + }, + { + "op": "add_node", + "node": { + "id": "init", + "type": "transform", + "label": "init", + "config": { + "mode": "expression", + "expression": "[1, 2]", + "assign_to": "items" + } + } + }, + { + "op": "add_edge", + "edge": { + "source": "start", + "target": "init" + } + }, + { + "op": "add_node", + "node": { + "id": "lp", + "type": "loop", + "label": "lp", + "config": { + "mode": "while", + "condition": "len(vars.items) > 5", + "max_iterations": 2 + } + } + }, + { + "op": "add_edge", + "edge": { + "source": "init", + "target": "lp" + } + }, + { + "op": "add_node", + "node": { + "id": "body", + "type": "transform", + "label": "body", + "config": { + "mode": "expression", + "expression": "1" + } + } + }, + { + "op": "add_edge", + "edge": { + "source": "lp", + "target": "body", + "sourceHandle": "body" + } + }, + { + "op": "add_edge", + "edge": { + "source": "body", + "target": "lp" + } + }, + { + "op": "add_node", + "node": { + "id": "out", + "type": "output", + "label": "out", + "config": { + "fields": [ + { + "name": "结果", + "value": "{{ vars.items }}" + } + ] + } + } + }, + { + "op": "add_edge", + "edge": { + "source": "lp", + "target": "out", + "sourceHandle": "done" + } + }, + { + "op": "done", + "explanation": "数一数", + "run": true + }, + { + "op": "check", + "status": "passed", + "repaired": 0, + "warnings": [] + }, + { + "op": "final", + "graph": { + "nodes": [ + { + "id": "start", + "type": "input", + "position": { + "x": 80.0, + "y": 0.0 + }, + "data": { + "label": "start", + "description": "", + "config": { + "fields": [ + { + "name": "question" + } + ] + } + } + }, + { + "id": "init", + "type": "transform", + "position": { + "x": 414.0, + "y": 0.0 + }, + "data": { + "label": "init", + "description": "", + "config": { + "mode": "expression", + "expression": "[1, 2]", + "assign_to": "items" + } + } + }, + { + "id": "lp", + "type": "loop", + "position": { + "x": 748.0, + "y": 0.0 + }, + "data": { + "label": "lp", + "description": "", + "config": { + "mode": "while", + "condition": "len(vars.items) > 5", + "max_iterations": 2 + } + } + }, + { + "id": "body", + "type": "transform", + "position": { + "x": 1082.0, + "y": -66.0 + }, + "data": { + "label": "body", + "description": "", + "config": { + "mode": "expression", + "expression": "1" + } + } + }, + { + "id": "out", + "type": "output", + "position": { + "x": 1082.0, + "y": 66.0 + }, + "data": { + "label": "out", + "description": "", + "config": { + "fields": [ + { + "name": "结果", + "value": "{{ vars.items }}" + } + ] + } + } + } + ], + "edges": [ + { + "id": "start:->init", + "source": "start", + "target": "init", + "sourceHandle": null, + "targetHandle": null, + "label": "" + }, + { + "id": "init:->lp", + "source": "init", + "target": "lp", + "sourceHandle": null, + "targetHandle": null, + "label": "" + }, + { + "id": "lp:body->body", + "source": "lp", + "target": "body", + "sourceHandle": "body", + "targetHandle": null, + "label": "" + }, + { + "id": "body:->lp", + "source": "body", + "target": "lp", + "sourceHandle": null, + "targetHandle": null, + "label": "" + }, + { + "id": "lp:done->out", + "source": "lp", + "target": "out", + "sourceHandle": "done", + "targetHandle": null, + "label": "" + } + ], + "viewport": {}, + "defaults": {} + }, + "issues": [], + "explanation": "数一数", + "autorun": true, + "layout": { + "mode": "full", + "placed": [ + "start", + "init", + "lp", + "body", + "out" + ] + }, + "tool_changes": [] + } + ], + "graph_fixed": [ + { + "op": "model", + "model": "mock-fast" + }, + { + "op": "plan", + "summary": "数一数" + }, + { + "op": "add_node", + "node": { + "id": "start", + "type": "input", + "label": "start", + "config": { + "fields": [ + { + "name": "question" + } + ] + } + } + }, + { + "op": "add_node", + "node": { + "id": "init", + "type": "transform", + "label": "init", + "config": { + "mode": "expression", + "expression": "[1, 2]", + "assign_to": "items" + } + } + }, + { + "op": "add_edge", + "edge": { + "source": "start", + "target": "init" + } + }, + { + "op": "add_node", + "node": { + "id": "lp", + "type": "loop", + "label": "lp", + "config": { + "mode": "while", + "condition": "{{ vars.items | length }} > 5", + "max_iterations": 2 + } + } + }, + { + "op": "add_edge", + "edge": { + "source": "init", + "target": "lp" + } + }, + { + "op": "add_node", + "node": { + "id": "body", + "type": "transform", + "label": "body", + "config": { + "mode": "expression", + "expression": "1" + } + } + }, + { + "op": "add_edge", + "edge": { + "source": "lp", + "target": "body", + "sourceHandle": "body" + } + }, + { + "op": "add_edge", + "edge": { + "source": "body", + "target": "lp" + } + }, + { + "op": "add_node", + "node": { + "id": "out", + "type": "output", + "label": "out", + "config": { + "fields": [ + { + "name": "结果", + "value": "{{ vars.items }}" + } + ] + } + } + }, + { + "op": "add_edge", + "edge": { + "source": "lp", + "target": "out", + "sourceHandle": "done" + } + }, + { + "op": "done", + "explanation": "数一数", + "run": true + }, + { + "op": "check", + "status": "repairing", + "round": 1, + "issues": [ + { + "level": "error", + "node_id": "lp", + "edge_id": null, + "message": "循环条件有误:表达式中不支持 | 过滤器(那是模板的写法):取长度请写 len(x),转换大小写请写 upper(x) / lower(x)(原文:{{ vars.items | length }} > 5)", + "field": "condition", + "code": null, + "fix": null + } + ] + }, + { + "op": "update_node", + "id": "lp", + "config": { + "mode": "while", + "condition": "len(vars.items) > 5", + "max_iterations": 2 + } + }, + { + "op": "check", + "status": "passed", + "repaired": 1, + "warnings": [] + }, + { + "op": "final", + "graph": { + "nodes": [ + { + "id": "start", + "type": "input", + "position": { + "x": 80.0, + "y": 0.0 + }, + "data": { + "label": "start", + "description": "", + "config": { + "fields": [ + { + "name": "question" + } + ] + } + } + }, + { + "id": "init", + "type": "transform", + "position": { + "x": 414.0, + "y": 0.0 + }, + "data": { + "label": "init", + "description": "", + "config": { + "mode": "expression", + "expression": "[1, 2]", + "assign_to": "items" + } + } + }, + { + "id": "lp", + "type": "loop", + "position": { + "x": 748.0, + "y": 0.0 + }, + "data": { + "label": "lp", + "description": "", + "config": { + "mode": "while", + "condition": "len(vars.items) > 5", + "max_iterations": 2 + } + } + }, + { + "id": "body", + "type": "transform", + "position": { + "x": 1082.0, + "y": -66.0 + }, + "data": { + "label": "body", + "description": "", + "config": { + "mode": "expression", + "expression": "1" + } + } + }, + { + "id": "out", + "type": "output", + "position": { + "x": 1082.0, + "y": 66.0 + }, + "data": { + "label": "out", + "description": "", + "config": { + "fields": [ + { + "name": "结果", + "value": "{{ vars.items }}" + } + ] + } + } + } + ], + "edges": [ + { + "id": "start:->init", + "source": "start", + "target": "init", + "sourceHandle": null, + "targetHandle": null, + "label": "" + }, + { + "id": "init:->lp", + "source": "init", + "target": "lp", + "sourceHandle": null, + "targetHandle": null, + "label": "" + }, + { + "id": "lp:body->body", + "source": "lp", + "target": "body", + "sourceHandle": "body", + "targetHandle": null, + "label": "" + }, + { + "id": "body:->lp", + "source": "body", + "target": "lp", + "sourceHandle": null, + "targetHandle": null, + "label": "" + }, + { + "id": "lp:done->out", + "source": "lp", + "target": "out", + "sourceHandle": "done", + "targetHandle": null, + "label": "" + } + ], + "viewport": {}, + "defaults": {} + }, + "issues": [], + "explanation": "数一数", + "autorun": true, + "layout": { + "mode": "full", + "placed": [ + "start", + "init", + "lp", + "body", + "out" + ] + }, + "tool_changes": [] + } + ], + "check_failed": [ + { + "op": "model", + "model": "mock-fast" + }, + { + "op": "plan", + "summary": "数一数" + }, + { + "op": "add_node", + "node": { + "id": "start", + "type": "input", + "label": "start", + "config": { + "fields": [ + { + "name": "question" + } + ] + } + } + }, + { + "op": "add_node", + "node": { + "id": "init", + "type": "transform", + "label": "init", + "config": { + "mode": "expression", + "expression": "[1, 2]", + "assign_to": "items" + } + } + }, + { + "op": "add_edge", + "edge": { + "source": "start", + "target": "init" + } + }, + { + "op": "add_node", + "node": { + "id": "lp", + "type": "loop", + "label": "lp", + "config": { + "mode": "while", + "condition": "foo(vars.items)", + "max_iterations": 2 + } + } + }, + { + "op": "add_edge", + "edge": { + "source": "init", + "target": "lp" + } + }, + { + "op": "add_node", + "node": { + "id": "body", + "type": "transform", + "label": "body", + "config": { + "mode": "expression", + "expression": "1" + } + } + }, + { + "op": "add_edge", + "edge": { + "source": "lp", + "target": "body", + "sourceHandle": "body" + } + }, + { + "op": "add_edge", + "edge": { + "source": "body", + "target": "lp" + } + }, + { + "op": "add_node", + "node": { + "id": "out", + "type": "output", + "label": "out", + "config": { + "fields": [ + { + "name": "结果", + "value": "{{ vars.items }}" + } + ] + } + } + }, + { + "op": "add_edge", + "edge": { + "source": "lp", + "target": "out", + "sourceHandle": "done" + } + }, + { + "op": "done", + "explanation": "数一数", + "run": true + }, + { + "op": "check", + "status": "repairing", + "round": 1, + "issues": [ + { + "level": "error", + "node_id": "lp", + "edge_id": null, + "message": "循环条件有误:不支持函数 foo(),可用的函数:len、str、int、float、bool、abs、min、max、sum、round、sorted、any、all、lower、upper、contains、startswith、endswith、matches、get、cell(原文:foo(vars.items))", + "field": "condition", + "code": null, + "fix": null + } + ] + }, + { + "op": "update_node", + "id": "lp", + "config": { + "mode": "while", + "condition": "bar(vars.items)", + "max_iterations": 2 + } + }, + { + "op": "check", + "status": "repairing", + "round": 2, + "issues": [ + { + "level": "error", + "node_id": "lp", + "edge_id": null, + "message": "循环条件有误:不支持函数 bar(),可用的函数:len、str、int、float、bool、abs、min、max、sum、round、sorted、any、all、lower、upper、contains、startswith、endswith、matches、get、cell(原文:bar(vars.items))", + "field": "condition", + "code": null, + "fix": null + } + ] + }, + { + "op": "update_node", + "id": "lp", + "config": { + "mode": "while", + "condition": "bar(vars.items)", + "max_iterations": 2 + } + }, + { + "op": "check", + "status": "failed", + "issues": [ + { + "level": "error", + "node_id": "lp", + "edge_id": null, + "message": "循环条件有误:不支持函数 bar(),可用的函数:len、str、int、float、bool、abs、min、max、sum、round、sorted、any、all、lower、upper、contains、startswith、endswith、matches、get、cell(原文:bar(vars.items))", + "field": "condition", + "code": null, + "fix": null + } + ], + "warnings": [] + }, + { + "op": "final", + "graph": { + "nodes": [ + { + "id": "start", + "type": "input", + "position": { + "x": 80.0, + "y": 0.0 + }, + "data": { + "label": "start", + "description": "", + "config": { + "fields": [ + { + "name": "question" + } + ] + } + } + }, + { + "id": "init", + "type": "transform", + "position": { + "x": 414.0, + "y": 0.0 + }, + "data": { + "label": "init", + "description": "", + "config": { + "mode": "expression", + "expression": "[1, 2]", + "assign_to": "items" + } + } + }, + { + "id": "lp", + "type": "loop", + "position": { + "x": 748.0, + "y": 0.0 + }, + "data": { + "label": "lp", + "description": "", + "config": { + "mode": "while", + "condition": "bar(vars.items)", + "max_iterations": 2 + } + } + }, + { + "id": "body", + "type": "transform", + "position": { + "x": 1082.0, + "y": -66.0 + }, + "data": { + "label": "body", + "description": "", + "config": { + "mode": "expression", + "expression": "1" + } + } + }, + { + "id": "out", + "type": "output", + "position": { + "x": 1082.0, + "y": 66.0 + }, + "data": { + "label": "out", + "description": "", + "config": { + "fields": [ + { + "name": "结果", + "value": "{{ vars.items }}" + } + ] + } + } + } + ], + "edges": [ + { + "id": "start:->init", + "source": "start", + "target": "init", + "sourceHandle": null, + "targetHandle": null, + "label": "" + }, + { + "id": "init:->lp", + "source": "init", + "target": "lp", + "sourceHandle": null, + "targetHandle": null, + "label": "" + }, + { + "id": "lp:body->body", + "source": "lp", + "target": "body", + "sourceHandle": "body", + "targetHandle": null, + "label": "" + }, + { + "id": "body:->lp", + "source": "body", + "target": "lp", + "sourceHandle": null, + "targetHandle": null, + "label": "" + }, + { + "id": "lp:done->out", + "source": "lp", + "target": "out", + "sourceHandle": "done", + "targetHandle": null, + "label": "" + } + ], + "viewport": {}, + "defaults": {} + }, + "issues": [ + { + "level": "error", + "node_id": "lp", + "edge_id": null, + "message": "循环条件有误:不支持函数 bar(),可用的函数:len、str、int、float、bool、abs、min、max、sum、round、sorted、any、all、lower、upper、contains、startswith、endswith、matches、get、cell(原文:bar(vars.items))", + "field": "condition", + "code": null, + "fix": null + } + ], + "explanation": "数一数", + "autorun": false, + "layout": { + "mode": "full", + "placed": [ + "start", + "init", + "lp", + "body", + "out" + ] + }, + "tool_changes": [] + } + ], + "reply": [ + { + "op": "model", + "model": "mock-fast" + }, + { + "op": "reply", + "text": "上一轮已经查过了:一共 3 家门店。" + } + ], + "ask_only": [ + { + "op": "model", + "model": "mock-fast" + }, + { + "op": "plan", + "summary": "先问清楚" + }, + { + "op": "ask", + "question": "要看哪个月的?", + "options": [ + "九月", + "十月" + ] + }, + { + "op": "final", + "graph": { + "nodes": [], + "edges": [] + }, + "issues": [], + "explanation": "", + "autorun": false, + "layout": { + "mode": "keep", + "placed": [] + }, + "tool_changes": [], + "asked": { + "question": "要看哪个月的?", + "options": [ + "九月", + "十月" + ] + } + } + ], + "error": [ + { + "op": "model", + "model": "mock-fast" + }, + { + "op": "error", + "message": "助手本轮未完成:接入返回了 502", + "hint": "", + "detail": "RuntimeError: 接入返回了 502" + } + ], + "no_ops": [ + { + "op": "model", + "model": "mock-fast" + }, + { + "op": "error", + "message": "助手本轮未完成:模型没有给出任何修改,也没有说明原因", + "hint": "画布未修改。可以重试,或把需求描述得更具体", + "detail": "" + } + ], + "truncated": [ + { + "op": "model", + "model": "mock-fast" + }, + { + "op": "plan", + "summary": "想了很久" + }, + { + "op": "error", + "message": "助手本轮未完成:模型的输出超出了长度上限,修改没有写完", + "hint": "画布未修改。可以重试,或把需求拆成几步分别提出", + "detail": "finish_reason=length" + } + ] +} diff --git a/backend/tests/test_catalog_patch.py b/backend/tests/test_catalog_patch.py index e9c1a49d..15fb7c1b 100644 --- a/backend/tests/test_catalog_patch.py +++ b/backend/tests/test_catalog_patch.py @@ -268,7 +268,8 @@ async def _stream(monkeypatch, model, source, base_graph=None): async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: async with client.stream("POST", "/api/copilot/generate-stream", json=body) as r: async for line in r.aiter_lines(): - if line.startswith("data: "): + # 末尾那条 usage 是这一轮助手的用量(answer_ops),不是生成的操作 + if line.startswith("data: ") and not line.startswith('data: {"op": "usage"'): events.append(json.loads(line[6:])) return events diff --git a/backend/tests/test_conversations.py b/backend/tests/test_conversations.py index df0d1fb9..55f03d55 100644 --- a/backend/tests/test_conversations.py +++ b/backend/tests/test_conversations.py @@ -215,7 +215,8 @@ async def _stream_graph(client, monkeypatch, conv_id: str, lines: list[str]) -> }) as r: last = {} async for line in r.aiter_lines(): - if line.startswith("data: "): + # 末尾那条 usage 是这一轮助手的用量(answer_ops),不是生成的操作 + if line.startswith("data: ") and not line.startswith('data: {"op": "usage"'): last = json.loads(line[6:]) return last diff --git a/backend/tests/test_copilot_answer_ops.py b/backend/tests/test_copilot_answer_ops.py new file mode 100644 index 00000000..4bbd5566 --- /dev/null +++ b/backend/tests/test_copilot_answer_ops.py @@ -0,0 +1,165 @@ +"""生成过程抽成可直接调用的 answer_ops(细案 01 做法 2):SSE 路由只包一层,同一个剧本下输出逐条不变。 + +评测和问数据页要走同一条生成路径:评测直接迭代 answer_ops 拿 model / look / check / final / reply / error 这些操作, +问数据页照旧收 SSE。这里守: + +- 几种结局(建图交付、直接回答、只问不改、自查改不好、模型出错、没给操作、输出被截断)的 SSE 输出,和抽出来之前 + 逐条一样(剧本见 SCENARIOS,抽出来之前录下的输出在 fixtures/copilot_answer_ops_baseline.json),末尾只多一条 usage; +- 直接迭代 answer_ops 和 SSE 收到的是同一串; +- 开流前的拒绝(模型没配好、限定的数据源都不在)照旧是 HTTP 错误,不进流。 + +剧本、需求都是虚构的。 +""" +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +from httpx import ASGITransport, AsyncClient +from langchain_core.messages import AIMessageChunk + +from app.main import app + +BASELINE = Path(__file__).parent / "fixtures" / "copilot_answer_ops_baseline.json" +#: 和时间有关、每次都不一样的字段:比较前去掉 +_TIMING = ("elapsed_ms", "ms", "duration_ms") + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +def _node(nid: str, ntype: str, config: dict) -> dict: + return {"op": "add_node", "node": {"id": nid, "type": ntype, "label": nid, "config": config}} + + +def _edge(a: str, b: str, **extra) -> dict: + return {"op": "add_edge", "edge": {"source": a, "target": b, **extra}} + + +def _loop_graph(condition: str) -> list[dict]: + return [ + {"op": "plan", "summary": "数一数"}, + _node("start", "input", {"fields": [{"name": "question"}]}), + _node("init", "transform", {"mode": "expression", "expression": "[1, 2]", "assign_to": "items"}), + _edge("start", "init"), + _node("lp", "loop", {"mode": "while", "condition": condition, "max_iterations": 2}), + _edge("init", "lp"), + _node("body", "transform", {"mode": "expression", "expression": "1"}), + _edge("lp", "body", sourceHandle="body"), + _edge("body", "lp"), + _node("out", "output", {"fields": [{"name": "结果", "value": "{{ vars.items }}"}]}), + _edge("lp", "out", sourceHandle="done"), + {"op": "done", "explanation": "数一数", "run": True}, + ] + + +def _fix(condition: str) -> list[dict]: + return [{"op": "plan", "summary": "改条件"}, + {"op": "update_node", "id": "lp", "config": {"mode": "while", "condition": condition, "max_iterations": 2}}, + {"op": "done", "explanation": "改好了"}] + + +def _chunks(ops: list) -> list[AIMessageChunk]: + return [op if isinstance(op, AIMessageChunk) else AIMessageChunk(content=json.dumps(op, ensure_ascii=False) + "\n") + for op in ops] + + +#: 剧本:(第一次调用吐的, 自查修正时吐的)。None 表示这一次调用直接抛错 +SCENARIOS: dict[str, tuple[list | None, list | None]] = { + "graph_ok": (_loop_graph("len(vars.items) > 5"), []), + "graph_fixed": (_loop_graph("{{ vars.items | length }} > 5"), _fix("len(vars.items) > 5")), + "check_failed": (_loop_graph("foo(vars.items)"), _fix("bar(vars.items)")), + "reply": ([{"op": "reply", "text": "上一轮已经查过了:一共 3 家门店。"}], []), + "ask_only": ([{"op": "plan", "summary": "先问清楚"}, + {"op": "ask", "question": "要看哪个月的?", "options": ["九月", "十月"]}], []), + "error": (None, []), + "no_ops": (["模型的随口注释,不是 JSON"], []), + "truncated": ([{"op": "plan", "summary": "想了很久"}, + AIMessageChunk(content="", response_metadata={"finish_reason": "length"})], []), +} + + +class _Scripted: + """照 SCENARIOS 吐:human 里有「上线前自查」时吐修正那一份。""" + + def __init__(self, first: list | None, fix: list | None) -> None: + self.first, self.fix = first, fix + + async def astream(self, messages): + human = next(text for role, text in messages if role == "human") + lines = self.fix if "上线前自查" in human else self.first + if lines is None: + raise RuntimeError("接入返回了 502") + for chunk in _chunks(lines): + yield chunk + + +def _use(monkeypatch, model) -> None: + import app.api.copilot as copilot + + async def _model(*_a, **_k): + return model, "mock-fast" + + monkeypatch.setattr(copilot, "get_chat_model", _model) + # 心跳和时间有关:拍子调大,剧本里不会有 + monkeypatch.setattr(copilot, "_HEARTBEAT_SECONDS", 3600.0) + + +def _clean(value): + if isinstance(value, dict): + return {k: _clean(v) for k, v in value.items() if k not in _TIMING} + if isinstance(value, list): + return [_clean(v) for v in value] + return value + + +async def _sse(client, body: dict) -> list[dict]: + out: list[dict] = [] + async with client.stream("POST", "/api/copilot/generate-stream", json=body) as r: + assert r.status_code == 200, await r.aread() + async for line in r.aiter_lines(): + if line.startswith("data: "): + out.append(json.loads(line[6:])) + return [_clean(op) for op in out if op.get("op") != "heartbeat"] + + +BODY = {"instruction": "数一数门店", "intent": "answer"} + + +@pytest.mark.parametrize("name", list(SCENARIOS)) +async def test_the_sse_output_is_unchanged_but_for_a_trailing_usage(client, monkeypatch, name): + _use(monkeypatch, _Scripted(*SCENARIOS[name])) + got = await _sse(client, BODY) + expected = json.loads(BASELINE.read_text(encoding="utf-8"))[name] + assert got[-1]["op"] == "usage", got[-1] + assert got[:-1] == expected + + +@pytest.mark.parametrize("name", ["graph_ok", "reply", "error"]) +async def test_iterating_answer_ops_directly_gives_the_same_ops(client, monkeypatch, name): + from app.api.copilot import GenerateIn, answer_ops + + _use(monkeypatch, _Scripted(*SCENARIOS[name])) + sse = await _sse(client, BODY) + direct = [_clean(op) async for op in answer_ops(GenerateIn(**BODY)) if op.get("op") != "heartbeat"] + assert direct == sse + + +async def test_refusals_before_the_stream_stay_http_errors(client, monkeypatch): + import app.api.copilot as copilot + from app.providers.factory import ProviderNotConfigured + + async def _unconfigured(*_a, **_k): + raise ProviderNotConfigured("provider 'gw' 需要填 base_url") + + monkeypatch.setattr(copilot, "get_chat_model", _unconfigured) + r = await client.post("/api/copilot/generate-stream", json=BODY) + assert r.status_code == 400 and "模型接入「gw」" in r.json()["detail"] + + _use(monkeypatch, _Scripted(*SCENARIOS["reply"])) + r = await client.post("/api/copilot/generate-stream", json={**BODY, "datasource_ids": ["no-such-source"]}) + assert r.status_code == 400 and r.json()["code"] == "datasource_scope_empty" diff --git a/backend/tests/test_copilot_evidence_guide.py b/backend/tests/test_copilot_evidence_guide.py index 9473da07..8a2a184e 100644 --- a/backend/tests/test_copilot_evidence_guide.py +++ b/backend/tests/test_copilot_evidence_guide.py @@ -213,7 +213,8 @@ async def test_guessed_columns_are_sent_back_and_fixed_before_running(sales, mon async with client.stream("POST", "/api/copilot/generate-stream", json={"instruction": "上周销售额多少", "intent": "answer"}) as r: async for line in r.aiter_lines(): - if line.startswith("data: "): + # 末尾那条 usage 是这一轮助手的用量(answer_ops),不是生成的操作 + if line.startswith("data: ") and not line.startswith('data: {"op": "usage"'): events.append(json.loads(line[6:])) checks = [e for e in events if e["op"] == "check"] assert [c["status"] for c in checks] == ["repairing", "passed"], checks diff --git a/backend/tests/test_copilot_look.py b/backend/tests/test_copilot_look.py index 86ade6a8..fdc96299 100644 --- a/backend/tests/test_copilot_look.py +++ b/backend/tests/test_copilot_look.py @@ -99,7 +99,8 @@ async def _events(client, monkeypatch, model, body: dict, path: str = "/api/copi async with client.stream("POST", path, json=body) as r: assert r.status_code == 200, await r.aread() async for line in r.aiter_lines(): - if line.startswith("data: "): + # 末尾那条 usage 是这一轮助手的用量(answer_ops),不是生成的操作 + if line.startswith("data: ") and not line.startswith('data: {"op": "usage"'): out.append(json.loads(line[6:])) return out diff --git a/backend/tests/test_copilot_merge.py b/backend/tests/test_copilot_merge.py index 9f8c97da..e425652f 100644 --- a/backend/tests/test_copilot_merge.py +++ b/backend/tests/test_copilot_merge.py @@ -110,7 +110,8 @@ async def _stream(client, monkeypatch, model, body: dict) -> list[dict]: async with client.stream("POST", "/api/copilot/generate-stream", json=body) as r: assert r.status_code == 200, await r.aread() async for line in r.aiter_lines(): - if line.startswith("data: "): + # 末尾那条 usage 是这一轮助手的用量(answer_ops),不是生成的操作 + if line.startswith("data: ") and not line.startswith('data: {"op": "usage"'): events.append(json.loads(line[6:])) return events diff --git a/backend/tests/test_copilot_patch_layout.py b/backend/tests/test_copilot_patch_layout.py index 27525fbd..647d9e01 100644 --- a/backend/tests/test_copilot_patch_layout.py +++ b/backend/tests/test_copilot_patch_layout.py @@ -64,7 +64,8 @@ async def _events(client, monkeypatch, ops: list[dict], base: dict | None) -> li out: list[dict] = [] async with client.stream("POST", "/api/copilot/generate-stream", json=body) as r: async for line in r.aiter_lines(): - if line.startswith("data: "): + # 末尾那条 usage 是这一轮助手的用量(answer_ops),不是生成的操作 + if line.startswith("data: ") and not line.startswith('data: {"op": "usage"'): out.append(json.loads(line[6:])) return out diff --git a/backend/tests/test_copilot_prompt_evidence.py b/backend/tests/test_copilot_prompt_evidence.py index fe4250e1..ee6f9beb 100644 --- a/backend/tests/test_copilot_prompt_evidence.py +++ b/backend/tests/test_copilot_prompt_evidence.py @@ -59,7 +59,8 @@ async def generate(client, monkeypatch, model: Scripted) -> list[dict]: events: list[dict] = [] async with client.stream("POST", "/api/copilot/generate-stream", json={"instruction": "写一份订单周报"}) as r: async for line in r.aiter_lines(): - if line.startswith("data: "): + # 末尾那条 usage 是这一轮助手的用量(answer_ops),不是生成的操作 + if line.startswith("data: ") and not line.startswith('data: {"op": "usage"'): events.append(json.loads(line[6:])) return events diff --git a/backend/tests/test_copilot_scope.py b/backend/tests/test_copilot_scope.py index 0a6f9322..48d738a5 100644 --- a/backend/tests/test_copilot_scope.py +++ b/backend/tests/test_copilot_scope.py @@ -96,7 +96,8 @@ async def _stream(client, monkeypatch, model, body: dict) -> tuple[int, list[dic if r.status_code != 200: return r.status_code, [json.loads(await r.aread())] async for line in r.aiter_lines(): - if line.startswith("data: "): + # 末尾那条 usage 是这一轮助手的用量(answer_ops),不是生成的操作 + if line.startswith("data: ") and not line.startswith('data: {"op": "usage"'): events.append(json.loads(line[6:])) return 200, events diff --git a/backend/tests/test_copilot_selfcheck.py b/backend/tests/test_copilot_selfcheck.py index 6e56e193..add5b950 100644 --- a/backend/tests/test_copilot_selfcheck.py +++ b/backend/tests/test_copilot_selfcheck.py @@ -85,7 +85,8 @@ async def _generate(client, monkeypatch, model: _Scripted) -> list[dict]: async with client.stream("POST", "/api/copilot/generate-stream", json={"instruction": "数一数"}) as r: async for line in r.aiter_lines(): - if line.startswith("data: "): + # 末尾那条 usage 是这一轮助手的用量(answer_ops),不是生成的操作 + if line.startswith("data: ") and not line.startswith('data: {"op": "usage"'): events.append(json.loads(line[6:])) return events diff --git a/backend/tests/test_copilot_sqlcheck.py b/backend/tests/test_copilot_sqlcheck.py index 2263ccc3..2dd9f809 100644 --- a/backend/tests/test_copilot_sqlcheck.py +++ b/backend/tests/test_copilot_sqlcheck.py @@ -195,7 +195,8 @@ async def _stream(monkeypatch, model, source): json={"instruction": "上周订单金额多少", "intent": "answer", "datasource_ids": [source]}) as r: async for line in r.aiter_lines(): - if line.startswith("data: "): + # 末尾那条 usage 是这一轮助手的用量(answer_ops),不是生成的操作 + if line.startswith("data: ") and not line.startswith('data: {"op": "usage"'): events.append(json.loads(line[6:])) return events diff --git a/backend/tests/test_copilot_truncation.py b/backend/tests/test_copilot_truncation.py index c81a73e0..b3f0733c 100644 --- a/backend/tests/test_copilot_truncation.py +++ b/backend/tests/test_copilot_truncation.py @@ -87,7 +87,8 @@ async def _events(client, monkeypatch, model, base: dict | None = BASE) -> list[ async with client.stream("POST", "/api/copilot/generate-stream", json={"instruction": "并行算同比和环比", "base_graph": base}) as r: async for line in r.aiter_lines(): - if line.startswith("data: "): + # 末尾那条 usage 是这一轮助手的用量(answer_ops),不是生成的操作 + if line.startswith("data: ") and not line.startswith('data: {"op": "usage"'): out.append(json.loads(line[6:])) return out @@ -263,7 +264,8 @@ async def test_a_rejected_budget_falls_back_to_the_old_one(client, monkeypatch, async with client.stream("POST", "/api/copilot/generate-stream", json={"instruction": "加一支同比", "base_graph": BASE}) as r: async for line in r.aiter_lines(): - if line.startswith("data: "): + # 末尾那条 usage 是这一轮助手的用量(answer_ops),不是生成的操作 + if line.startswith("data: ") and not line.startswith('data: {"op": "usage"'): events.append(json.loads(line[6:])) assert 8192 in asked assert big.calls == 1 and small.calls == 1 @@ -284,7 +286,8 @@ async def test_other_rejections_are_not_retried(client, monkeypatch): async with client.stream("POST", "/api/copilot/generate-stream", json={"instruction": "改一下", "base_graph": BASE}) as r: async for line in r.aiter_lines(): - if line.startswith("data: "): + # 末尾那条 usage 是这一轮助手的用量(answer_ops),不是生成的操作 + if line.startswith("data: ") and not line.startswith('data: {"op": "usage"'): events.append(json.loads(line[6:])) assert small.calls == 0 assert events[-1]["op"] == "error" diff --git a/backend/tests/test_usage_meter.py b/backend/tests/test_usage_meter.py new file mode 100644 index 00000000..c69259f6 --- /dev/null +++ b/backend/tests/test_usage_meter.py @@ -0,0 +1,283 @@ +"""用量计量(providers/usage.py;细案 01 做法 3):助手建图(含挑表、接着写、自查修正)和复核的用量也记。 + +守的是: +- 按模型累计 tokens 和调用次数,金额按价格目录算;目录外的模型、回复没带用量(而目录价又不是 0)的调用记 unpriced_calls, + 不当成 0 元; +- metered 只计生成过程自己的调用(含它在一步里起的任务);调用方在两次 yield 之间做的事(发起运行)不计——运行的用量 + 另有 llm.end,包进来就重复了;嵌套时内层并进外层; +- answer_ops 末尾的 usage 记下这一轮建图的每一次调用(起草、自查修正各一次);复核的结果带 usage。 + +模型都是剧本,不调真接口;价格用目录里登记过的模型 id,名字之外都是虚构的。 +""" +from __future__ import annotations + +import asyncio +import json +from typing import Any + +import pytest +from httpx import ASGITransport, AsyncClient +from langchain_core.language_models.chat_models import BaseChatModel +from langchain_core.messages import AIMessage, AIMessageChunk +from langchain_core.outputs import ChatGeneration, ChatGenerationChunk, ChatResult + +from app.main import app +from app.providers import usage +from app.providers.catalog import estimate_cost + +PRICED = "claude-opus-5" # 价格目录里有 +UNPRICED = "made-up-model-x" # 价格目录里没有 +FREE = "mock-fast" # 目录价是 0 + + +class Scripted(BaseChatModel): + """真的 LangChain 模型:回调照常触发。每次调用吐 replies 里的下一份(用完重复最后一份),用量按 tokens 报; + report=False 时回复不带用量(有的网关流式不给)。""" + + model: str = PRICED + replies: list[list[str]] = [["好的"]] + tokens: tuple[int, int] = (1000, 200) + report: bool = True + calls: int = 0 + + @property + def _llm_type(self) -> str: + return "scripted" + + @property + def _identifying_params(self) -> dict[str, Any]: + return {"model": self.model} + + def _next(self) -> list[str]: + lines = self.replies[min(self.calls, len(self.replies) - 1)] + self.calls += 1 + return lines + + def _usage(self) -> dict[str, int] | None: + if not self.report: + return None + inp, out = self.tokens + return {"input_tokens": inp, "output_tokens": out, "total_tokens": inp + out} + + def _generate(self, messages, stop=None, run_manager=None, **kwargs) -> ChatResult: + message = AIMessage(content="\n".join(self._next())) + if (meta := self._usage()) is not None: + message.usage_metadata = meta + return ChatResult(generations=[ChatGeneration(message=message)]) + + async def _astream(self, messages, stop=None, run_manager=None, **kwargs): + """像 Anthropic 那样报用量:第一段带输入 token,流末尾单独一段带输出 token。""" + lines = self._next() + inp, out = self.tokens + for i, line in enumerate(lines): + meta = {"input_tokens": inp, "output_tokens": 0, "total_tokens": inp} if self.report and i == 0 else None + yield ChatGenerationChunk(message=AIMessageChunk(content=line + "\n", **({"usage_metadata": meta} if meta + else {}))) + if self.report: + yield ChatGenerationChunk(message=AIMessageChunk(content="", usage_metadata={ + "input_tokens": 0, "output_tokens": out, "total_tokens": out})) + + +# -------------------------------------------------------------------------- +# 计量本身 +# -------------------------------------------------------------------------- + + +async def test_metering_counts_by_model_and_prices_from_the_catalog(): + m = usage.meter() + with usage.metering(m): + await Scripted(model=PRICED).ainvoke("你好") + await Scripted(model=PRICED).ainvoke("再来") + await Scripted(model=UNPRICED).ainvoke("你好") + await Scripted(model=PRICED + "-silent", report=False).ainvoke("你好") + await Scripted(model=FREE, report=False).ainvoke("你好") + got = m.summary() + assert got["by_model"][PRICED] == {"input_tokens": 2000, "output_tokens": 400, "calls": 2, + "cost_usd": round(estimate_cost(PRICED, 2000, 400), 6), "unpriced_calls": 0} + assert got["by_model"][UNPRICED]["unpriced_calls"] == 1 and got["by_model"][UNPRICED]["cost_usd"] == 0 + # 目录外的模型名(多了后缀)照样算估不出;目录价是 0 的本地模型没报用量也不算估不出 + assert got["by_model"][PRICED + "-silent"]["unpriced_calls"] == 1 + assert got["by_model"][FREE]["unpriced_calls"] == 0 + assert got["calls"] == 5 and got["unpriced_calls"] == 2 + assert got["cost_usd"] == pytest.approx(estimate_cost(PRICED, 2000, 400)) + + +async def test_outside_the_meter_nothing_is_counted(): + m = usage.meter() + with usage.metering(m): + pass + await Scripted().ainvoke("你好") + assert m.summary()["calls"] == 0 + assert usage.current() is None + + +async def test_metered_counts_the_generator_and_its_tasks_but_not_the_caller(): + """生成过程里的调用(含它起的任务)计进来;调用方在两次 yield 之间发起的「运行」不计。""" + m = usage.meter() + leaked: list[asyncio.Task] = [] + + async def generation(): + await Scripted(model=PRICED).ainvoke("起草") + # 心跳那一层每一步起一个任务:任务在步里创建,拷走的上下文带着计量 + await asyncio.ensure_future(Scripted(model=PRICED).ainvoke("挑表")) + yield "first" + await Scripted(model=PRICED).ainvoke("自查修正") + yield "second" + + seen = [] + async for item in usage.metered(generation(), m): + seen.append(item) + # 调用方拿到一条操作后自己调模型(相当于发起运行):它的上下文里没有计量 + await Scripted(model=UNPRICED).ainvoke("运行里的调用") + leaked.append(asyncio.ensure_future(Scripted(model=UNPRICED).ainvoke("运行任务"))) + await asyncio.gather(*leaked) + got = m.summary() + assert seen == ["first", "second"] + assert got["by_model"] == {PRICED: got["by_model"][PRICED]} and got["calls"] == 3 + assert usage.current() is None + + +async def test_nested_meters_fold_into_the_outer_one(): + outer, inner = usage.meter(), usage.meter() + + async def generation(): + await Scripted(model=PRICED).ainvoke("起草") + yield 1 + + with usage.metering(outer): + await Scripted(model=PRICED).ainvoke("外层自己的") + async for _ in usage.metered(generation(), inner): + pass + with usage.metering(usage.meter()): + await Scripted(model=PRICED).ainvoke("复核") + assert inner.summary()["calls"] == 1 + assert outer.summary()["calls"] == 3 + + +# -------------------------------------------------------------------------- +# answer_ops 末尾的 usage、复核的 usage +# -------------------------------------------------------------------------- + + +def _ops(*ops: dict) -> list[str]: + return [json.dumps(o, ensure_ascii=False) for o in ops] + + +GRAPH = _ops( + {"op": "plan", "summary": "数一数"}, + {"op": "add_node", "node": {"id": "start", "type": "input", "label": "输入", + "config": {"fields": [{"name": "question"}]}}}, + {"op": "add_node", "node": {"id": "lp", "type": "loop", "label": "循环", + "config": {"mode": "while", "condition": "foo(vars.x)", "max_iterations": 2}}}, + {"op": "add_edge", "edge": {"source": "start", "target": "lp"}}, + {"op": "add_node", "node": {"id": "body", "type": "transform", "label": "体", + "config": {"mode": "expression", "expression": "1"}}}, + {"op": "add_edge", "edge": {"source": "lp", "target": "body", "sourceHandle": "body"}}, + {"op": "add_edge", "edge": {"source": "body", "target": "lp"}}, + {"op": "add_node", "node": {"id": "out", "type": "output", "label": "成果", + "config": {"fields": [{"name": "r", "value": "1"}]}}}, + {"op": "add_edge", "edge": {"source": "lp", "target": "out", "sourceHandle": "done"}}, + {"op": "done", "explanation": "数一数", "run": True}, +) +FIX = _ops({"op": "update_node", "id": "lp", "config": {"mode": "while", "condition": "len(vars.x) > 1", + "max_iterations": 2}}, + {"op": "done", "explanation": "改好了"}) + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +async def test_answer_ops_ends_with_the_usage_of_every_build_call(client, monkeypatch): + """起草、自查修正各一次。生成过程读到 done 就关流(后面的闲话不要),流末尾才报的输出 token 拿不到:这两次照样计次数、 + 计已报的输入 token,记成估不出(unpriced_calls),金额只是估得出的那部分——不悄悄少算。""" + import app.api.copilot as copilot + + model = Scripted(model=PRICED, replies=[GRAPH, FIX], tokens=(3000, 500)) + + async def _model(*_a, **_k): + return model, PRICED + + monkeypatch.setattr(copilot, "get_chat_model", _model) + events = [] + async with client.stream("POST", "/api/copilot/generate-stream", json={"instruction": "数一数", "intent": "answer"}) as r: + async for line in r.aiter_lines(): + if line.startswith("data: "): + events.append(json.loads(line[6:])) + assert [e["op"] for e in events if e["op"] == "check"] and events[-2]["op"] == "final" + spent = events[-1] + assert spent["op"] == "usage", spent + assert model.calls == 2, spent # 起草一次、自查修正一次 + assert spent["by_model"][PRICED]["calls"] == 2, spent + assert spent["by_model"][PRICED]["input_tokens"] == 6000 and spent["by_model"][PRICED]["output_tokens"] == 0 + assert spent["cost_usd"] == pytest.approx(estimate_cost(PRICED, 6000, 0), abs=1e-6) + assert spent["unpriced_calls"] == 2, "关流前没报全的两次记成估不出" + + +async def test_a_build_on_an_unpriced_model_says_it_cannot_be_priced(client, monkeypatch): + import app.api.copilot as copilot + + model = Scripted(model=UNPRICED, replies=[_ops({"op": "reply", "text": "已经查过了"})]) + + async def _model(*_a, **_k): + return model, UNPRICED + + monkeypatch.setattr(copilot, "get_chat_model", _model) + events = [] + async with client.stream("POST", "/api/copilot/generate-stream", json={"instruction": "刚才那个数是多少"}) as r: + async for line in r.aiter_lines(): + if line.startswith("data: "): + events.append(json.loads(line[6:])) + assert [e["op"] for e in events] == ["model", "reply", "usage"] + assert events[-1]["calls"] == 1 and events[-1]["unpriced_calls"] == 1 and events[-1]["cost_usd"] == 0 + + +async def test_a_stream_read_to_the_end_is_fully_priced(): + """读完的流(直接回答那一行之后模型自己停了):输入、输出 token 都报全了,金额估得出。""" + m = usage.meter() + + async def generation(): + async for _ in Scripted(model=PRICED, tokens=(400, 60)).astream("问"): + pass + yield 1 + + async for _ in usage.metered(generation(), m): + pass + row = m.summary()["by_model"][PRICED] + assert (row["input_tokens"], row["output_tokens"], row["unpriced_calls"]) == (400, 60, 0) + assert row["cost_usd"] == round(estimate_cost(PRICED, 400, 60), 6) + + +async def test_the_review_carries_its_own_usage(client, monkeypatch): + import app.api.copilot as copilot + from app.core.events import EventType + from app.db.base import SessionLocal + from app.db.models import Run, RunEvent + + reply = json.dumps({"note": "agent 用满了步数,结论不完整。", "answer": "", "retry": True}, ensure_ascii=False) + model = Scripted(model=PRICED, replies=[[reply]], tokens=(800, 100)) + + async def _model(*_a, **_k): + return model, PRICED + + monkeypatch.setattr(copilot, "get_chat_model", _model) + async with SessionLocal() as session: + run = Run(status="succeeded", output={"answer": "大概是 3 家"}, graph={}) + session.add(run) + await session.flush() + session.add(RunEvent(run_id=run.id, seq=1, type=EventType.LOG, node_id="a", ts=0.0, + data={"level": "warn", "code": "step_limit", "message": "用满了步数"})) + await session.commit() + run_id = run.id + body = (await client.post("/api/copilot/review", json={"run_id": run_id, "question": "有几家门店"})).json() + assert body["usage"]["calls"] == 1 and body["usage"]["by_model"][PRICED]["input_tokens"] == 800 + # 干净的运行不调模型:usage 全 0 + async with SessionLocal() as session: + clean = Run(status="succeeded", output={"answer": "3 家"}, graph={}) + session.add(clean) + await session.commit() + clean_id = clean.id + body = (await client.post("/api/copilot/review", json={"run_id": clean_id, "question": "有几家门店"})).json() + assert body["verdict"] == "ok" and body["usage"]["calls"] == 0 and body["usage"]["by_model"] == {} From 2fa2db3ea013e237e5aed944054f3298ebb6f52a Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Fri, 16 Oct 2026 10:54:02 +0800 Subject: [PATCH 18/39] =?UTF-8?q?=E9=A9=B3=E5=9B=9E=E4=B8=8D=E5=86=8D?= =?UTF-8?q?=E8=AE=B0=E6=88=90=E5=A4=B1=E8=B4=A5=EF=BC=88=E5=89=8D=E7=AB=AF?= =?UTF-8?q?=EF=BC=89=EF=BC=9A=E7=BB=88=E6=80=81=E6=8C=89=20end=5Freason=20?= =?UTF-8?q?=E5=86=99=E3=80=8C=E8=A2=AB=E9=A9=B3=E5=9B=9E=E3=80=8D=EF=BC=8C?= =?UTF-8?q?=E5=89=AF=E6=A0=87=E9=A2=98=E5=86=99=E8=B0=81=E3=80=81=E5=9C=A8?= =?UTF-8?q?=E5=93=AA=E4=B8=80=E6=AD=A5=E3=80=81=E5=A4=87=E6=B3=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 为什么:服务端已把人的驳回收成 cancelled + end_reason=rejected(上一个提交)。前端以前靠正则解析报错原文认驳回,而且只认 「人工驳回」一种,动作确认、工具调用审批、代码执行审批的三种原话都落到兜底、给出注定再失败的「继续运行」(05-已知遗留 (a))。 做了什么: - types.ts:Run.end_reason(rejected / stopped)。 - lib/status:新增状态 rejected(「被驳回」,同取消的剪影、不画红、终态),resolveStatus 按 endReason 区分;记录页列表、详情、 航迹的终点按它写「被驳回 · 原话」。 - run/decode:run.cancelled 带 reason=rejected 时标题写「被驳回」,副标题「张工 驳回了「付款」:金额不对」(超时驳回写按设置 驳回、不重复备注),指向那一步;手动停止、老事件照旧「已取消」。 - store/chat + ChatPage:终态里被驳回的一轮照 cancelled 收(不给继续运行),徽标写「被驳回」,补救区说明这是审批人的决定、 副标题写谁和备注;刷新之后从 meta.rejected 恢复;核对状态时按运行的 end_reason 认。 - lib/explain:正则只留给存量 failed+驳回,补上「未确认(被驳回)」「的调用已被驳回」「代码执行已被驳回」三种原话,一律不给继续运行。 - 继续运行被拒的说法改成「被驳回的运行不能继续运行:改好内容后重新发起」(check-copy 不收「接着跑」)。 - check-decode 加一节:reason=rejected 的标题、副标题、没署名没备注、超时驳回、相位、手动停止和老事件。 偏离:问数据的「之前的尝试」(retry 留档,W1-a 的地盘)里被驳回的那一次仍写「已手动停止」,没动。 --- backend/app/engine/runner.py | 4 +-- backend/tests/test_run_rejected.py | 2 +- frontend/src/lib/explain.ts | 10 +++++--- frontend/src/lib/status.ts | 23 ++++++++++++++--- frontend/src/lib/terms.ts | 19 ++++++++++++++ frontend/src/pages/ChatPage.tsx | 6 +++-- frontend/src/pages/runs/RunDetailView.tsx | 7 ++++-- frontend/src/pages/runs/RunList.tsx | 10 +++++++- frontend/src/pages/runs/TracePane.tsx | 4 ++- frontend/src/pages/runs/model.ts | 4 +-- frontend/src/run/decode.ts | 8 +++++- frontend/src/store/chat.ts | 30 +++++++++++++++++++---- frontend/src/types.ts | 10 ++++++++ scripts/check-decode.mjs | 27 ++++++++++++++++++++ 14 files changed, 141 insertions(+), 23 deletions(-) diff --git a/backend/app/engine/runner.py b/backend/app/engine/runner.py index 7fcd0a62..25c5e5e1 100644 --- a/backend/app/engine/runner.py +++ b/backend/app/engine/runner.py @@ -50,8 +50,8 @@ RUN_REJECTED = "run_rejected" #: 运行为什么结束在 cancelled(Run.end_reason,docs/design/next-round/00-总方案.md X9) END_REJECTED = "rejected" END_STOPPED = "stopped" -#: 被驳回的运行不能接着跑:从断点重跑会读到同一个驳回答复、再停在同一处 -REJECTED_CONTINUE = "被驳回的运行不能接着跑:改好内容后重新发起" +#: 被驳回的运行不能继续运行:从断点重跑会读到同一个驳回答复、再停在同一处 +REJECTED_CONTINUE = "被驳回的运行不能继续运行:改好内容后重新发起" #: 服务重启时还在排队的运行改成失败时写的话(run.error 和补发的 run.failed 是同一句) QUEUED_AT_RESTART = "服务重启时这次运行仍在排队,没有可恢复的进度,请重新发起运行" diff --git a/backend/tests/test_run_rejected.py b/backend/tests/test_run_rejected.py index 520d2d4f..690db845 100644 --- a/backend/tests/test_run_rejected.py +++ b/backend/tests/test_run_rejected.py @@ -132,7 +132,7 @@ async def test_the_api_carries_end_reason_and_refuses_to_continue_a_rejected_run assert (body["status"], body["end_reason"]) == ("cancelled", "rejected") r = await client.post(f"/api/runs/{run_id}/continue", json={}) assert r.status_code == 409 and r.json()["code"] == "run_rejected" - assert r.json()["detail"] == "被驳回的运行不能接着跑:改好内容后重新发起" + assert r.json()["detail"] == "被驳回的运行不能继续运行:改好内容后重新发起" stop = await client.post(f"/api/runs/{run_id}/cancel") assert stop.status_code == 409 and "被驳回" in stop.json()["detail"] diff --git a/frontend/src/lib/explain.ts b/frontend/src/lib/explain.ts index 4e6e06e5..d9c992fe 100644 --- a/frontend/src/lib/explain.ts +++ b/frontend/src/lib/explain.ts @@ -295,11 +295,15 @@ export function explainRunError(error: string | null | undefined, detail?: strin raw, } } - // 后端有一条路径以前写「人工拒绝:」(现在统一为「人工驳回:」),历史运行里两种都有 - if (/人工(?:驳回|拒绝)/.test(text)) { + // 只留给存量:加 end_reason 之前被驳回的运行记的是 failed,报错原文是这几种说法。新的驳回结束在 cancelled + + // end_reason=rejected,界面按那个字段说(lib/status、REJECTED_TEXT),不走这里(05-已知遗留 (a))。后端有一条路径 + // 以前写「人工拒绝:」(现在统一为「人工驳回:」);动作确认、工具调用审批、代码执行审批的驳回各有一种原话。 + // 接着跑只会读到同一个驳回答复、再停一次,所以都不给「继续运行」 + if (/人工(?:驳回|拒绝)|未确认(被驳回)|的调用已被驳回|代码执行已被驳回/.test(text)) { + const human = /^人工(?:驳回|拒绝)/.test(plain) return { title: '人工审批驳回,运行终止', - reason: plain.replace(/^人工(?:驳回|拒绝)[::]?\s*/, '') || undefined, + reason: (human ? plain.replace(/^人工(?:驳回|拒绝)[::]?\s*/, '') : plain) || undefined, action: '这是审批人的决定,不是故障。如有需要,请修改内容后重新发起运行。', continuable: false, raw, diff --git a/frontend/src/lib/status.ts b/frontend/src/lib/status.ts index c817fbf3..96a6eb16 100644 --- a/frontend/src/lib/status.ts +++ b/frontend/src/lib/status.ts @@ -22,6 +22,7 @@ export type StatusCode = | 'succeeded' // 运行跑完 | 'failed' | 'cancelled' + | 'rejected' // 运行:被人驳回结束(后端是 cancelled + end_reason=rejected,05-已知遗留 (a)) | 'suspended' // 服务重启后挂起,可续跑 | 'held' // 后端的 interrupted 但没有待审批:已挂起 · 可续跑 | 'skipped' // 被 skip_if 跳过 @@ -104,6 +105,11 @@ export const STATUS: Record = { code: 'cancelled', label: '已取消', short: '已取消', shape: 'stop', alert: false, terminal: true, ...tok('cancelled', 'var(--text-faint)'), }, + // 人的决定,不是故障:不画红、不给继续运行;和手动停止同一个剪影,叫法分开 + rejected: { + code: 'rejected', label: '被驳回', short: '被驳回', shape: 'stop', alert: false, terminal: true, + hint: '审批人驳回了这次运行,运行到这里结束;改好内容后重新发起', ...tok('cancelled', 'var(--text-faint)'), + }, suspended: { code: 'suspended', label: '已中断(服务重启)', short: '已中断', shape: 'pause', alert: true, terminal: true, hint: '服务重启时本次运行尚未结束,已在断点处挂起,可继续运行', ...tok('suspended', 'var(--warn)'), @@ -141,15 +147,25 @@ const ALIAS: Record = { * 才分得开——传 pendingApproval。不知道时(没传)按等待审批处理,这是 * interrupted 最常见的来源,也是改造前的说法,不会比原来更错。 */ -export function resolveStatus(status: string | null | undefined, opts?: { pendingApproval?: boolean }): StatusCode { +export function resolveStatus(status: string | null | undefined, opts?: StatusOpts): StatusCode { if (!status) return 'idle' if (status === 'interrupted') return opts?.pendingApproval === false ? 'held' : 'waiting' + // 被驳回只看服务端给的 end_reason,不解析报错原文(界面层不放逻辑) + if (status === 'cancelled' && opts?.endReason === 'rejected') return 'rejected' if (status in STATUS) return status as StatusCode return ALIAS[status] ?? 'idle' } +/** + * 落状态要的旁证:pendingApproval 分 interrupted 的两种;endReason 是运行的 end_reason(cancelled 时分被驳回和手动停止) + */ +export interface StatusOpts { + pendingApproval?: boolean + endReason?: string | null +} + /** 状态的完整外观。未知状态码回落到 idle 的外观,文字写原码,免得静默吞掉 */ -export function statusMeta(status: string | null | undefined, opts?: { pendingApproval?: boolean }): StatusMeta { +export function statusMeta(status: string | null | undefined, opts?: StatusOpts): StatusMeta { const code = resolveStatus(status, opts) const meta = STATUS[code] if (status && code === 'idle' && status !== 'idle' && !(status in ALIAS)) { @@ -159,7 +175,7 @@ export function statusMeta(status: string | null | undefined, opts?: { pendingAp } /** 只要文字 */ -export function statusLabel(status: string | null | undefined, opts?: { pendingApproval?: boolean; short?: boolean }): string { +export function statusLabel(status: string | null | undefined, opts?: StatusOpts & { short?: boolean }): string { const meta = statusMeta(status, opts) return opts?.short ? meta.short : meta.label } @@ -186,6 +202,7 @@ export function serverStatusOf(code: StatusCode): RunStatus | null { case 'waiting': case 'held': case 'suspended': return 'interrupted' case 'done': case 'succeeded': return 'succeeded' case 'queued': case 'running': case 'failed': case 'cancelled': return code + case 'rejected': return 'cancelled' default: return null } } diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index 51fb6dba..72bd4212 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -5135,6 +5135,25 @@ export const ORIGIN_TEXT = { actor: (who: string, verified: boolean) => `${verified ? '已登录用户' : '署名'}「${who}」`, } +/** + * 驳回 ≠ 失败(docs/design/next-round/05-已知遗留.md (a)):人的驳回让运行结束在 cancelled + end_reason=rejected, + * run.cancelled 带着在哪一步(label)、谁(actor)、结论(decision)、备注(note)。时间线、问数据的终态、记录页都照这一份说, + * 不再按报错原文猜 + */ +export const REJECTED_TEXT = { + title: '被驳回', + /** 副标题:谁在哪一步驳回、备注;超时驳回写按设置驳回(备注是系统写的那句,不重复) */ + sub: (d: { actor?: unknown; label?: unknown; note?: unknown; decision?: unknown }): string => { + const who = typeof d.actor === 'string' ? d.actor.trim() : '' + const where = typeof d.label === 'string' && d.label.trim() ? `「${d.label.trim()}」` : '这一步' + const note = typeof d.note === 'string' ? d.note.trim() : '' + if (d.decision === 'expired_reject') return `${where}超过时限,按设置驳回` + return `${who ? `${who} 驳回了${where}` : `${where}被驳回`}${note ? `:${note}` : ''}` + }, + /** 问数据:这一轮的运行被驳回结束时,补救区的那一句 */ + chatReason: '这一轮的运行被审批人驳回,已经结束。这是人的决定,不是故障;改好内容后可以重新运行。', +} + /** 来源标签:老运行没有来源时写「来源未记录」 */ export const originLabel = (kind: string | null | undefined): string => kind && kind in ORIGIN_LABEL ? ORIGIN_LABEL[kind as OriginKind] : ORIGIN_TEXT.unknown diff --git a/frontend/src/pages/ChatPage.tsx b/frontend/src/pages/ChatPage.tsx index 8bae6511..a93803b7 100644 --- a/frontend/src/pages/ChatPage.tsx +++ b/frontend/src/pages/ChatPage.tsx @@ -17,7 +17,7 @@ import { humanizeError } from '../lib/errors' import { explainRunError } from '../lib/explain' import { canLeave, leavePass } from '../lib/leave' import { formatNumber, formatTime } from '../lib/format' -import { CHAT_BLUEPRINT_TEXT } from '../lib/terms' +import { CHAT_BLUEPRINT_TEXT, REJECTED_TEXT } from '../lib/terms' import { useCatalog, useCatalogListState, useDatasources, useOnReconnect, type CatalogListState } from '../store/catalog' import { useAtLeast } from '../store/auth' import { useBlueprint } from '../store/blueprint' @@ -383,7 +383,8 @@ function toStreamTurn(turn: ChatTurn): StreamTurn { : turn.phase === 'error' ? 'error' : live || turn.phase === 'checking' ? 'running' : 'done', // 没核对上的不知道结局,用「未运行」那个空心点,不画成完成也不画成失败 - statusCode: turn.phase === 'cancelled' ? 'cancelled' + // 被人驳回的(turn.rejected,服务端 end_reason=rejected)照 lib/status 写「被驳回」,不写「已取消」 + statusCode: turn.phase === 'cancelled' ? (turn.rejected ? 'rejected' : 'cancelled') : turn.phase === 'suspended' ? 'suspended' // 助手反问:这一轮结束了,在等用户回答 : turn.asked && turn.phase === 'done' ? 'waiting' @@ -729,6 +730,7 @@ function Remedies({ conversationId, turnId, last, onRephrase, readOnly }: { // 中断、取消、没核对上在流里不出红框(它们不是故障),为什么停、停了意味着什么在这里说 const note = kind === 'suspended' || kind === 'unknown' ? turn.failure + : kind === 'cancelled' && turn.rejected ? { title: '', reason: REJECTED_TEXT.chatReason, hint: turn.rejected.sub } : kind === 'cancelled' ? { title: '', reason: turn.run ? '你已停止本轮,对应的运行也已一并取消。' : '你在搭建工作流时停止了本轮。' } : null diff --git a/frontend/src/pages/runs/RunDetailView.tsx b/frontend/src/pages/runs/RunDetailView.tsx index 5769acb3..74073354 100644 --- a/frontend/src/pages/runs/RunDetailView.tsx +++ b/frontend/src/pages/runs/RunDetailView.tsx @@ -205,7 +205,7 @@ export function RunDetailView({ runId, onChange, onDeleted }: { const aimed = focus && !goneFromCanvas(focus) ? focus : null const canvasHref = hrefFor(aimed) // 还在往前走的叫「打开」(接着看实时的、去处理),停下的叫「回放」 - const replayable = code === 'succeeded' || code === 'failed' || code === 'cancelled' + const replayable = code === 'succeeded' || code === 'failed' || code === 'cancelled' || code === 'rejected' const dup = !!run.workflow_id && duplicateNames(workflows.map((w) => ({ id: w.id, name: w.name }))).has(run.workflow_name) // ---- 动作 ---- @@ -730,7 +730,10 @@ export function RunDetailView({ runId, onChange, onDeleted }: { /** 事件之外的事实(查到的状态、有没有审批)和事件推出的相位合成一个显示码 */ function displayCode(run: Run, phase: RunPhase, pending: boolean | undefined): StatusCode { switch (phase) { - case 'running': case 'queued': case 'waiting': case 'succeeded': case 'failed': case 'cancelled': + case 'cancelled': + // 被驳回只看服务端的 end_reason(05-已知遗留 (a)) + return run.end_reason === 'rejected' ? 'rejected' : 'cancelled' + case 'running': case 'queued': case 'waiting': case 'succeeded': case 'failed': return phase case 'suspended': // 后端写的是 interrupted:服务重启打断、没有审批,叫「已挂起 · 可续跑」 diff --git a/frontend/src/pages/runs/RunList.tsx b/frontend/src/pages/runs/RunList.tsx index 01e9e699..0958caf6 100644 --- a/frontend/src/pages/runs/RunList.tsx +++ b/frontend/src/pages/runs/RunList.tsx @@ -7,7 +7,8 @@ import { formatCost, formatDateTime, formatDay, formatSpan, formatTime, formatTokens, parseServerTime, shortId, } from '../../lib/format' import { - ORIGIN_LABEL, ORIGIN_TEXT, RUN_CLASS_LABEL, TASK_KIND_LABEL, TASK_LOG_TEXT, UNSAVED_HINT, UNSAVED_NAME, isUnsaved, originLabel, + ORIGIN_LABEL, ORIGIN_TEXT, REJECTED_TEXT, RUN_CLASS_LABEL, TASK_KIND_LABEL, TASK_LOG_TEXT, UNSAVED_HINT, UNSAVED_NAME, isUnsaved, + originLabel, runName, } from '../../lib/terms' import { summarizeRun } from '../../run/decode' @@ -466,6 +467,13 @@ function RowContext({ run, code, approval, now }: { const reason = cancelReason(run.error) return
    已取消{reason ? ` · ${reason}` : ''}
    } + case 'rejected': + // 人的驳回(end_reason=rejected):报错原文就是给人看的那一句(「「付款」未确认(被驳回),没有发送:…」) + return ( +
    + {REJECTED_TEXT.title}{run.error ? ` · ${run.error}` : ''} +
    + ) default: { const summary = summarizeRun(run.input, run.output) return summary diff --git a/frontend/src/pages/runs/TracePane.tsx b/frontend/src/pages/runs/TracePane.tsx index 8c6523a0..73dbbbdb 100644 --- a/frontend/src/pages/runs/TracePane.tsx +++ b/frontend/src/pages/runs/TracePane.tsx @@ -11,7 +11,7 @@ import { explainRunError } from '../../lib/explain' import { NONE, formatCost, formatDateTime, formatNumber, formatOffset, formatSpan, formatTokens, shortId, } from '../../lib/format' -import { COST_TEXT, runClassLabel, nodeTypeLabel, runCostText } from '../../lib/terms' +import { COST_TEXT, REJECTED_TEXT, runClassLabel, nodeTypeLabel, runCostText } from '../../lib/terms' import { topology, type GraphLike } from '../../run/derive' import { isActivePhase, isSettled, liveAt, project, type NodeState, type Projection, type RunPhase, type Trace, @@ -449,6 +449,7 @@ function momentsOf( const failed = trace.phase === 'failed' ? trace.failedNodeId : undefined const why = trace.phase === 'failed' ? explainRunError((failed && trace.nodes[failed]?.error) || error).title + : code === 'rejected' ? error?.trim() || undefined : trace.phase === 'cancelled' ? cancelReason(error) ?? undefined : trace.phase === 'suspended' ? error?.trim() || undefined : undefined out.push({ @@ -458,6 +459,7 @@ function momentsOf( text: trace.phase === 'succeeded' ? '运行完成' : failed ? `失败于${name(failed)}` : trace.phase === 'failed' ? '运行失败' + : code === 'rejected' ? REJECTED_TEXT.title : trace.phase === 'cancelled' ? '运行取消' : '运行挂起', sub: why, }) diff --git a/frontend/src/pages/runs/model.ts b/frontend/src/pages/runs/model.ts index 61674490..9236665c 100644 --- a/frontend/src/pages/runs/model.ts +++ b/frontend/src/pages/runs/model.ts @@ -49,8 +49,8 @@ export const SEGMENT_CODES: StatusCode[] = RUN_STATUS_ORDER * 打断后没有审批——要看待审批列表才分得开。approvals 还没加载时传 null,按 * 等待审批算(最常见的来源),不会在加载的那一瞬间闪成「已挂起」。 */ -export function runCode(run: Pick, approvals: Approval[] | null): StatusCode { - if (run.status !== 'interrupted') return resolveStatus(run.status) +export function runCode(run: Pick, approvals: Approval[] | null): StatusCode { + if (run.status !== 'interrupted') return resolveStatus(run.status, { endReason: run.end_reason }) return resolveStatus(run.status, { pendingApproval: approvals == null ? undefined : hasPendingApproval(approvals, run.id), }) diff --git a/frontend/src/run/decode.ts b/frontend/src/run/decode.ts index 427e24bc..185c3d52 100644 --- a/frontend/src/run/decode.ts +++ b/frontend/src/run/decode.ts @@ -6,7 +6,7 @@ import { ABSENCE_CLAIM_TEXT, ACTION_CONFIRM_TEXT, ACTION_ENV_LABEL, ACTION_LOG_TEXT, CATALOG_DRIFT_TEXT, CATALOG_PATCH_TEXT, COPILOT_CONTEXT_TEXT, JUDGE_TEXT, LOOK_TEXT, MERGE_TEXT, ORIGIN_TEXT, RUN_SQL_CHECK_TEXT, SQL_CHECK_TEXT, TASK_KIND_LABEL, TASK_LOG_TEXT, TASK_STATUS_LABEL, TASK_TEXT, TYPE_LABEL, originLabel, CATALOG_TABLE_FIELD_LABEL, CHALLENGE_TEXT, DUAL_CHECK_TEXT, INQUIRY_TEXT, JUDGE_TRUNCATED_TEXT, MASKED_CELL_TEXT, NODE_TYPE_LABEL, - VERIFY_TRUNCATED_TEXT, BP_OP_TEXT, TEMPLATE_LABEL, bpPathText, FILE_TEXT, + VERIFY_TRUNCATED_TEXT, BP_OP_TEXT, TEMPLATE_LABEL, bpPathText, FILE_TEXT, REJECTED_TEXT, claimTally, evidenceTally, issuanceLabel, nodeTypeLabel, } from '../lib/terms' @@ -2673,6 +2673,12 @@ export function decodeRun(events: RunEvent[], final?: RunFinal, opts?: DecodeOpt // 用户主动停下的,不是出错:收成中性的"已取消",不画红色的失败。谁停的、放弃等 // 审批时一并关了几条,写在副标题上——多人用时「怎么没了」得有个交代 settle(endingOf(event, awaiting)!) + if (d.reason === 'rejected') { + // 人的驳回(05-已知遗留 (a)):同样不是出错,标题写「被驳回」,副标题写谁在哪一步驳回、备注 + out.push({ id: `rc-${seq}`, seq, kind: 'lifecycle', status: 'cancelled', title: REJECTED_TEXT.title, + sub: REJECTED_TEXT.sub(d), ...(d.node_id ? { nodeId: String(d.node_id) } : {}) }) + break + } // 说法和画布胶囊的那一句一致:「张工 放弃了这次运行,1 条待审批一并关闭」 const who = typeof d.actor === 'string' && d.actor.trim() ? d.actor.trim() : '' const said = String(d.message ?? '').trim() diff --git a/frontend/src/store/chat.ts b/frontend/src/store/chat.ts index 6ed4255c..7983e5bf 100644 --- a/frontend/src/store/chat.ts +++ b/frontend/src/store/chat.ts @@ -2,7 +2,7 @@ import { create } from 'zustand' import { ApiError, api, streamCopilot, streamRun } from '../api/client' import type { CopilotOp, FixKind, RunFinal } from '../run/decode' import { copilotContext, copilotLook, decodeRun, looksOf } from '../run/decode' -import { CHAT_BLUEPRINT_TEXT, LOOK_TEXT } from '../lib/terms' +import { CHAT_BLUEPRINT_TEXT, LOOK_TEXT, REJECTED_TEXT } from '../lib/terms' import type { LookEvent } from '../typesApp' import { errorMessage, humanizeError, isNetworkError } from '../lib/errors' import { guardReview, hasEvidence } from '../lib/evidence' @@ -87,6 +87,11 @@ export interface Failure { fixFirst?: true } +/** 被驳回的那一句:事件流里来的按 run.cancelled 的谁、哪一步、备注拼(REJECTED_TEXT.sub);核对状态时只有报错原文 */ +export interface Rejected { + sub: string +} + /** 限定这一问只查哪些数据源。名字跟着存:库删了之后,这一轮查的是谁还得说得出来 */ export interface ScopeSource { id: string @@ -127,6 +132,8 @@ export interface TurnMeta { issuance?: Record noQuery?: boolean outcome?: 'cancelled' | 'suspended' + /** 运行被人驳回结束(outcome 是 cancelled):副标题那一句,刷新之后照样写「被驳回」,不退回成「你已停止本轮」 */ + rejected?: Rejected failure?: Failure attempts?: Attempt[] /** 这一轮从开问到交付的墙钟毫秒 */ @@ -180,6 +187,11 @@ export interface ChatTurn { /** 一行错误文案(兼容老数据和流式头部)。结构化的在 failure 里 */ error: string failure?: Failure | null + /** + * 这一轮的运行被人驳回结束(服务端 cancelled + end_reason=rejected,05-已知遗留 (a))。phase 照旧是 cancelled(不是故障、 + * 不给继续运行),说法另写「被驳回」和谁、在哪一步、备注 + */ + rejected?: Rejected | null startedAt: number /** 交付、失败、取消或中断的时刻。计时器停在这里 */ endedAt?: number @@ -440,6 +452,7 @@ function buildMeta(t: ChatTurn): TurnMeta { issuance: slimIssuance(t.output?._issuance ?? t.rawOutput?._issuance) ?? t.meta?.issuance, noQuery: t.noQuery || undefined, outcome: t.phase === 'cancelled' ? 'cancelled' : t.phase === 'suspended' ? 'suspended' : undefined, + rejected: t.phase === 'cancelled' && t.rejected ? t.rejected : undefined, failure: t.failure ?? undefined, attempts: t.attempts?.length ? t.attempts : undefined, ms: t.endedAt ? t.endedAt - t.startedAt : t.meta?.ms, @@ -678,6 +691,7 @@ function restoreTurn(t: ConversationTurn): ChatTurn { output, error: phase === 'error' ? (failure ? lineOf(failure) : t.status === 'running' ? buildInterrupted.title : '') : '', failure: phase === 'error' ? (t.status === 'running' ? buildInterrupted : failure) : phase === 'suspended' ? SUSPENDED : null, + rejected: phase === 'cancelled' ? meta?.rejected ?? null : null, startedAt: Date.parse(t.created_at ?? '') || Date.now(), endedAt: meta?.ms != null ? (Date.parse(t.created_at ?? '') || 0) + meta.ms : undefined, lastSeq: 0, @@ -1043,7 +1057,10 @@ export const useChat = create((set, get) => ({ } if (run.status === 'cancelled') { if (cur.phase !== 'done' && cur.phase !== 'cancelled') { - conclude({ phase: 'cancelled', status: PHASE_TEXT.cancelled, failure: null, error: '' }, '已取消') + // 被驳回只看服务端的 end_reason:核对时手上只有运行记录,副标题用报错原文(它就是给人看的那句) + const rejected = run.end_reason === 'rejected' ? { sub: run.error || REJECTED_TEXT.title } : null + conclude({ phase: 'cancelled', status: rejected ? REJECTED_TEXT.title : PHASE_TEXT.cancelled, failure: null, + error: '', rejected }, '已取消') } return } @@ -1673,15 +1690,18 @@ function watch( // 服务在关停:这一轮会停在断点上。停在审批上的仍然是在等人 if (d.code === 'server_shutdown' && current()?.phase !== 'waiting') suspend() break - case 'run.cancelled': + case 'run.cancelled': { ended = true batch.now() + // 人的驳回(reason=rejected)也收成 cancelled:不是故障、不给继续运行,说法写「被驳回」和谁、哪一步、备注 + const rejected = d.reason === 'rejected' ? { sub: REJECTED_TEXT.sub(d) } : null patch(() => ({ - phase: 'cancelled', status: PHASE_TEXT.cancelled, cancel: null, endedAt: Date.now(), - runStatus: 'cancelled', final: { status: 'cancelled' }, + phase: 'cancelled', status: rejected ? REJECTED_TEXT.title : PHASE_TEXT.cancelled, cancel: null, + endedAt: Date.now(), runStatus: 'cancelled', final: { status: 'cancelled' }, rejected, })) void flow.persist({ status: 'error', error: '已取消' }) break + } case 'run.failed': { ended = true batch.now() diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 0bd204f6..8381edd5 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -4471,3 +4471,13 @@ export interface DataSourceName { kind?: string enabled?: boolean } + +// --------------------------------------------------------------------------- +// 驳回 ≠ 失败(docs/design/next-round/05-已知遗留.md (a),总方案 X9) + +/** 运行为什么结束在 cancelled(服务端 Run.end_reason):rejected 人工驳回,stopped 手动停止。其余终态、老运行为空 */ +export type RunEndReason = 'rejected' | 'stopped' + +export interface Run { + end_reason?: RunEndReason | null +} diff --git a/scripts/check-decode.mjs b/scripts/check-decode.mjs index 98dd2172..f079d849 100644 --- a/scripts/check-decode.mjs +++ b/scripts/check-decode.mjs @@ -2724,5 +2724,32 @@ await section('读入附件 file.extracted:读入多少字、全文多少字 && odd[0]?.title === '读入附件' && odd[2]?.title === '附件抽不出文字', odd.map((s) => `${s.title}|${s.sub ?? ''}`).join(' ; ')) }) +await section('人的驳回 run.cancelled reason=rejected(05-已知遗留 (a)):终态是「被驳回」,副标题写谁、哪一步、备注;手动停止照旧「已取消」', async () => { + const E = (seq, type, node_id, data = {}) => ({ seq, type, node_id, ts: seq, data }) + const head = [E(1, 'run.started', null, { nodes: 2 }), E(2, 'node.started', 'pay', { label: '付款' }), + E(3, 'human.requested', 'pay', { title: '确认发送:付款' }), E(4, 'human.resolved', 'pay', { approved: false, note: '金额不对', actor: '张工' }), + E(5, 'node.failed', 'pay', { error: '「付款」未确认(被驳回),没有发送:金额不对' })] + const ended = (data) => flatten(mod.decodeRun([...head, E(6, 'run.cancelled', null, data)])) + .find((s) => s.kind === 'lifecycle' && s.id === 'rc-6') + const rejected = ended({ reason: 'rejected', node_id: 'pay', label: '付款', kind: 'action_confirm', decision: 'reject', + actor: '张工', note: '金额不对' }) + check('标题写「被驳回」,不写「已取消」「失败」', rejected?.title === '被驳回' && rejected?.status === 'cancelled', + `${rejected?.title} / ${rejected?.status}`) + check('副标题写谁在哪一步驳回、备注', rejected?.sub === '张工 驳回了「付款」:金额不对', rejected?.sub) + check('指向驳回的那一步(点了画布取景过去)', rejected?.nodeId === 'pay', rejected?.nodeId) + const anonymous = ended({ reason: 'rejected', node_id: 'pay', label: '付款', decision: 'reject', actor: null, note: '' }) + check('没署名、没备注:「「付款」被驳回」,不出 null、undefined', anonymous?.sub === '「付款」被驳回', anonymous?.sub) + const expired = ended({ reason: 'rejected', node_id: 'pay', label: '付款', decision: 'expired_reject', actor: null, + note: '超过时限,按设置驳回' }) + check('超时驳回:写按设置驳回,备注不重复', expired?.sub === '「付款」超过时限,按设置驳回', expired?.sub) + const phase = mod.decodePhase([...head, E(6, 'run.cancelled', null, { reason: 'rejected' })]) + check('相位收成 cancelled(不是 failed、不给继续运行)', phase === 'cancelled', phase) + const stopped = ended({ reason: 'stopped', actor: '李工' }) + check('手动停止照旧「已取消」,写谁停的', stopped?.title === '已取消' && stopped?.sub === '李工 取消了本次运行', + `${stopped?.title} / ${stopped?.sub}`) + const legacy = ended({}) + check('老事件(没有 reason)照旧「已取消」', legacy?.title === '已取消', legacy?.title) +}) + console.log(failed ? `\n✗ ${failed} 项未通过` : '\n✓ 解码器全部通过') process.exit(failed ? 1 : 0) From fbb671a96b3fac1d5df17d1fa3bcacb09724c6c4 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Fri, 16 Oct 2026 10:54:30 +0800 Subject: [PATCH 19/39] =?UTF-8?q?=E4=BD=BF=E7=94=A8=E8=80=85=E9=80=89?= =?UTF-8?q?=E4=BA=BA=EF=BC=88=E5=89=8D=E7=AB=AF=EF=BC=89=EF=BC=9A=E9=80=89?= =?UTF-8?q?=E4=BA=BA=E6=8E=A7=E4=BB=B6=E6=8C=89=E6=B1=A0=E8=BE=B9=E8=BE=93?= =?UTF-8?q?=E5=85=A5=E8=BE=B9=E5=8F=96=E5=80=99=E9=80=89=EF=BC=9B=E4=B8=AA?= =?UTF-8?q?=E4=BA=BA=E6=B8=A0=E9=81=93=E7=9A=84=E5=AE=9E=E4=BE=8B=E4=BA=8B?= =?UTF-8?q?=E4=BB=B6=E6=94=B9=E5=8F=AB=E3=80=8C=E6=88=91=E5=8F=91=E8=B5=B7?= =?UTF-8?q?=E7=9A=84=E5=AE=9E=E4=BE=8B=E5=AE=8C=E6=88=90=20/=20=E5=A4=B1?= =?UTF-8?q?=E8=B4=A5=E3=80=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 为什么:使用者读不到名录,选人控件只能凭记忆手填用户名,填错了服务端以前也照收(05-已知遗留 (b))。服务端已按用途给候选池、 开登录时按同一个池核对(上上个提交),前端要把候选给到人手上。另外个人渠道勾「实例完成 / 失败」以前永远收不到,服务端修好之后 (驳回那个提交里的 notify_run),名字要照实写成只推自己发起的实例(04-对外集成与触达 B1)。 做了什么: - PrincipalPicker 加 source(CandidateSource:form 带 app_id+entry 或 task_id 和 field,review)。搭建者及以上照旧用整份名录; 使用者开了登录、给了 source 时边输入边按开头取候选(250 毫秒防抖),列表里显示显示名、标签悬停给用户名;加的时候按同一个池 问一次服务端,不在池里的当场说「「x」不在可选的人里」(问不到就先收下,提交时由服务端判);池要先输开头才列、列不全时照实提示。 - SchemaForm 按表单用在哪(入口、待办,同上传文件的上下文)给选人控件 source;个人应用提交审核用 purpose=review。 - typesApp:CandidateSource、Candidates、FormField 的 x-principal、FormError 的 principal 码(和服务端 FORM_ERROR_CODES 同序)。 - api.identity.candidates。termsApp 的 PRINCIPAL_TEXT 加三句。 - 设置 → 通知:范围是「仅自己」的渠道,run.finished / run.failed 写「我发起的实例完成 / 失败」(termsApp.NOTIFY_PERSONAL_EVENT_LABEL)。 - 服务端 x-principal 写法不对时的说法改成不带半角冒号接中文的写法(check-copy)。 --- backend/app/auth/directory.py | 4 +- frontend/src/api/auth.ts | 5 +- frontend/src/components/PrincipalPicker.tsx | 81 +++++++++++++++++---- frontend/src/components/SchemaForm.tsx | 14 +++- frontend/src/lib/termsApp.ts | 13 ++++ frontend/src/pages/apps/personal.tsx | 2 +- frontend/src/pages/settings/NotifyTab.tsx | 8 +- frontend/src/typesApp.ts | 29 +++++++- 8 files changed, 132 insertions(+), 24 deletions(-) diff --git a/backend/app/auth/directory.py b/backend/app/auth/directory.py index d981f9bc..fa88a6b7 100644 --- a/backend/app/auth/directory.py +++ b/backend/app/auth/directory.py @@ -119,14 +119,14 @@ POOL_SLOT = "slot:" def pool_problem(value: Any) -> str | None: """x-principal 的写法对不对:只能是 {pool: app | group:<组> | slot:<岗位>}。没问题为 None。""" if not isinstance(value, dict) or set(value) != {"pool"} or not isinstance(value.get("pool"), str): - return "的候选范围只能写 {\"pool\": \"app\"}、{\"pool\": \"group:组名\"} 或 {\"pool\": \"slot:岗位\"}" + return "的候选范围只能写成 {\"pool\": \"app\"}、{\"pool\": \"group:<组名>\"} 或 {\"pool\": \"slot:<岗位>\"}" pool = value["pool"].strip() if pool == POOL_APP: return None for prefix in (POOL_GROUP, POOL_SLOT): if pool.startswith(prefix) and pool[len(prefix):].strip(): return None - return "的候选范围只能是 app(能用这个应用的人)、group:组名 或 slot:岗位" + return "的候选范围只能是 app(能用这个应用的人)、group:<组名> 或 slot:<岗位>" def pool_of(field: Any) -> str: diff --git a/frontend/src/api/auth.ts b/frontend/src/api/auth.ts index 51990394..b20eb648 100644 --- a/frontend/src/api/auth.ts +++ b/frontend/src/api/auth.ts @@ -2,7 +2,7 @@ * 身份的接口(/api/auth、/api/users、/api/groups、/api/audit),挂在 api.auth / api.identity 上。形状见 typesApp.ts。 * 开登录时:没登录的请求回 401(前端跳 /login),角色不够回 403。关登录时 /me 恒为 mode='off'。 */ -import type { AuditRow, AuthMode, Directory, GroupRow, Principal, Role, UserRow } from '../typesApp' +import type { AuditRow, AuthMode, CandidateSource, Candidates, Directory, GroupRow, Principal, Role, UserRow } from '../typesApp' import { del, get, patch, post, put, qs, type RequestOptions } from './http' export const auth = { @@ -29,6 +29,9 @@ export const identity = { groups: () => get<{ items: GroupRow[] }>('/groups'), /** 选人用的名录:搭建者及以上只读(/users、/groups 只给管理员) */ directory: () => get('/directory'), + /** 使用者选人的候选:按用途给的池,只有名字(任何登录的人都能调;用不了那个应用的 403)。q 按开头筛 */ + candidates: (source: CandidateSource, q?: string, limit = 20) => + get(`/directory/candidates${qs({ ...source, q: q || undefined, limit })}`), createGroup: (body: { name: string; label?: string }) => post('/groups', body), setMembers: (id: string, members: string[]) => put(`/groups/${encodeURIComponent(id)}/members`, { members }), deleteGroup: (id: string) => del(`/groups/${encodeURIComponent(id)}`), diff --git a/frontend/src/components/PrincipalPicker.tsx b/frontend/src/components/PrincipalPicker.tsx index 229061d9..c1b68c6e 100644 --- a/frontend/src/components/PrincipalPicker.tsx +++ b/frontend/src/components/PrincipalPicker.tsx @@ -5,15 +5,18 @@ * 名录里没有的、停用了的不让添加,说清是哪一个(账号密码登录时人和组都在本系统里建,名录里没有就是填错了);单点登录的 * 人和组第一次登录后才进名录,名录里没有的照实提醒,点「仍然添加」才加上。已选的里头名录里没有的、停用了的在标签上 * 标出来,不静默放着。 - * 使用者读不到名录:照填,服务端按名字认。关登录时没有用户表,只能手填名字——签批记的也是各自填的署名,这里不假装能核对。 + * 使用者读不到名录(只读与最小可见):给了 source 的(表单里的选人控件、个人应用提交审核)边输入边按开头取候选 + * (GET /api/directory/candidates,05-已知遗留 (b)),只有这件事能选的人、只有名字;加的时候按同一个池问一次服务端,不在池里的 + * 当场说清。提交时服务端照同一个池核对(不在池里 422),这里只是提前说。没给 source 的照填,服务端按名字认。 + * 关登录时没有用户表,只能手填名字——签批记的也是各自填的署名,这里不假装能核对。 */ -import { useEffect, useId, useState } from 'react' +import { useEffect, useId, useRef, useState } from 'react' import { Plus, Users, UserRound, X } from 'lucide-react' import clsx from 'clsx' import { api } from '../api/client' import { PRINCIPAL_TEXT } from '../lib/termsApp' import { useAtLeast, useAuth, useLoginOn } from '../store/auth' -import type { Directory } from '../typesApp' +import type { CandidateSource, Candidates, Directory } from '../typesApp' import { isComposing } from './ui' export interface Principal { @@ -29,8 +32,16 @@ export interface PrincipalPickerProps { /** 只能选一个 */ single?: boolean label?: string + /** + * 使用者取候选的池(服务端按用途算):表单里的选人控件(入口、待办)、个人应用提交审核。搭建者及以上照旧用整份名录, + * 不看它 + */ + source?: CandidateSource } +/** 边输入边取候选:停手这么久再去问,不是每敲一个字问一次 */ +const ASK_DELAY_MS = 250 + /** 名录只取一次,同一页上几个选择器共用;没取到(断线)下次打开再取 */ let directory: Promise | null = null function loadDirectory(): Promise { @@ -46,15 +57,21 @@ function problemOf(p: Principal, known: Directory | null): 'missing' | 'disabled return !user ? 'missing' : user.disabled ? 'disabled' : null } -export function PrincipalPicker({ value, onChange, allowEveryone = false, single = false, label }: PrincipalPickerProps) { +export function PrincipalPicker({ value, onChange, allowEveryone = false, single = false, label, source }: PrincipalPickerProps) { const uid = useId().replace(/[^a-zA-Z0-9_-]/g, '') const loginOn = useLoginOn() // 名录给搭建者及以上(使用者读不到,不去取:取了也是 403) const listable = useAtLeast('builder') && loginOn + // 使用者:有 source 就按池取候选 + const asking = !listable && loginOn && !!source + const sourceKey = source ? JSON.stringify(source) : '' const oidc = useAuth((s) => s.me?.mode === 'oidc') const [kind, setKind] = useState<'user' | 'group'>('user') const [text, setText] = useState('') const [known, setKnown] = useState(null) + const [pool, setPool] = useState(null) + /** 取到过的候选:用户名 → 显示名(标签上写显示名,悬停给用户名) */ + const shown = useRef(new Map()) /** 刚才要加、名录里对不上的那一项:soft 是单点登录下名录里还没有(可以「仍然添加」),其余不让加 */ const [refused, setRefused] = useState<{ item: Principal; message: string; soft: boolean } | null>(null) useEffect(() => { @@ -63,12 +80,38 @@ export function PrincipalPicker({ value, onChange, allowEveryone = false, single void loadDirectory().then((d) => { if (alive) setKnown(d) }) return () => { alive = false } }, [listable]) + useEffect(() => { + if (!asking || !source) return + let alive = true + const timer = window.setTimeout(() => { + api.identity.candidates(source, text.trim()) + .then((c) => { + if (!alive) return + for (const u of c.users) shown.current.set(u.username, u.display_name) + setPool(c) + }) + .catch(() => { if (alive) setPool(null) }) + }, ASK_DELAY_MS) + return () => { alive = false; window.clearTimeout(timer) } + // sourceKey 代替 source:调用方每次渲染都给一个新对象 + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [asking, sourceKey, text]) const has = (p: Principal) => value.some((x) => x.kind === p.kind && x.name === p.name) - const add = (p: Principal, anyway = false) => { + const add = async (p: Principal, anyway = false) => { const item = { ...p, name: p.name.trim() } if (!item.name) return if (has(item)) { setText(''); setRefused(null); return } + if (asking && source && item.kind !== 'everyone') { + // 按同一个池问一次服务端:不在池里的当场说清(提交时服务端照样拒)。问不到(断线)就先收下,提交时由服务端判 + const found = await api.identity.candidates(source, item.name, 50).catch(() => null) + const listed = found && (item.kind === 'group' ? found.groups.some((g) => g.name === item.name) + : found.users.some((u) => u.username === item.name)) + if (found && !listed) { + setRefused({ item, soft: false, message: PRINCIPAL_TEXT.notCandidate(item.name) }) + return + } + } const problem = problemOf(item, known) const soft = problem === 'missing' && oidc if (problem && !(soft && anyway)) { @@ -87,9 +130,16 @@ export function PrincipalPicker({ value, onChange, allowEveryone = false, single const remove = (p: Principal) => onChange(value.filter((x) => !(x.kind === p.kind && x.name === p.name))) const listId = `${uid}-list` const errorId = `${uid}-error` - const options = !known ? [] : kind === 'user' - ? known.users.filter((u) => !u.disabled).map((u) => ({ value: u.username, label: u.display_name })) - : known.groups.map((g) => ({ value: g.name, label: g.label })) + const options = asking + ? (!pool ? [] : kind === 'user' ? pool.users.map((u) => ({ value: u.username, label: u.display_name })) + : pool.groups.map((g) => ({ value: g.name, label: g.label }))) + : !known ? [] : kind === 'user' + ? known.users.filter((u) => !u.disabled).map((u) => ({ value: u.username, label: u.display_name })) + : known.groups.map((g) => ({ value: g.name, label: g.label })) + /** 候选池要先输入开头才列(应用授权给了所有人)、列不全(超过上限) */ + const poolHint = asking && pool + ? (pool.needs_query && !text.trim() ? PRINCIPAL_TEXT.typeToSearch : pool.truncated ? PRINCIPAL_TEXT.truncated : null) + : null return (
    @@ -102,7 +152,9 @@ export function PrincipalPicker({ value, onChange, allowEveryone = false, single data-principal-problem={problem ?? undefined} style={problem ? { color: 'var(--st-waiting)', borderColor: 'var(--st-waiting)' } : undefined}> {p.kind === 'group' ? : p.kind === 'user' ? : null} - {p.kind === 'everyone' ? PRINCIPAL_TEXT.everyone : p.name} + + {p.kind === 'everyone' ? PRINCIPAL_TEXT.everyone : shown.current.get(p.name) ?? p.name} + ({PRINCIPAL_TEXT[p.kind]}) {problem && · {problem === 'disabled' ? PRINCIPAL_TEXT.disabledTag : PRINCIPAL_TEXT.missingTag}} {allowEveryone && ( )} @@ -151,12 +203,13 @@ export function PrincipalPicker({ value, onChange, allowEveryone = false, single style={{ color: refused.soft ? 'var(--st-waiting)' : 'var(--err)' }} data-principal-refused={refused.soft ? 'soft' : 'hard'}> {refused.message} {refused.soft && ( - )}
    )} + {poolHint &&
    {poolHint}
    } {!loginOn &&
    {PRINCIPAL_TEXT.manualHint}
    }
    ) diff --git a/frontend/src/components/SchemaForm.tsx b/frontend/src/components/SchemaForm.tsx index ffa52aaa..b2b0780f 100644 --- a/frontend/src/components/SchemaForm.tsx +++ b/frontend/src/components/SchemaForm.tsx @@ -12,7 +12,7 @@ import { useEffect, useId, useState } from 'react' import type { CSSProperties, KeyboardEvent as ReactKeyboardEvent, ReactNode } from 'react' import clsx from 'clsx' -import type { FormError, FormField, FormSchema } from '../typesApp' +import type { CandidateSource, FormError, FormField, FormSchema } from '../typesApp' import { FORM_TEXT } from '../lib/termsApp' import { fieldOfPath, fieldOrder, fileLimitsOf, localToRfc3339, optionsOf, rfc3339ToLocal, widgetOf, @@ -88,6 +88,16 @@ export function SchemaForm({ schema, value, onChange, errors, readOnly, onSubmit ) } +/** + * 选人控件的候选从哪个池取:表单用在入口(app_id + entry)、待办(task_id)上时,按这个字段问服务端(05-已知遗留 (b)); + * 蓝图试运行之类没有应用的地方不给(搭建者用整份名录) + */ +function principalSource(context: Record | undefined, field: string): CandidateSource | undefined { + if (context?.task_id) return { purpose: 'form', task_id: context.task_id, field } + if (context?.app_id && context.entry) return { purpose: 'form', app_id: context.app_id, entry: context.entry, field } + return undefined +} + function FieldRow({ id, name, field, required, value, error, readOnly, onChange, fileContext }: { id: string; name: string; field: FormField; required: boolean; value: unknown; error?: FormError readOnly: boolean; onChange: (v: unknown) => void; fileContext?: Record @@ -143,7 +153,7 @@ function FieldRow({ id, name, field, required, value, error, readOnly, onChange,
    {label}
    {readOnly ?
    {typeof value === 'string' && value ? value : '—'}
    - : onChange(list[0]?.name || undefined)} />} {foot} diff --git a/frontend/src/lib/termsApp.ts b/frontend/src/lib/termsApp.ts index 13c5b33e..3c24e994 100644 --- a/frontend/src/lib/termsApp.ts +++ b/frontend/src/lib/termsApp.ts @@ -368,6 +368,10 @@ export const PRINCIPAL_TEXT = { /** 已选的那一项不在名录里、或已停用:标在标签上 */ missingTag: '名录里没有', disabledTag: '已停用', + /** 使用者选人:候选按这件事能选的人给(服务端 auth/directory 的候选池,05-已知遗留 (b)) */ + notCandidate: (name: string) => `「${name}」不在可选的人里,请从候选中选择`, + typeToSearch: '能选的人较多:输入名字的开头查找', + truncated: '只列出了一部分:多输入几个字缩小范围', } /** 身份:登录、账户菜单、角色(§4.4 AUTH_TEXT) */ @@ -445,6 +449,15 @@ export const SIGNER_TEXT = { record: '记入留痕', } +/** + * 个人渠道(设置 → 通知,范围是「仅自己」)里实例结束的两个事件:只推自己发起、而且是登录身份发起的实例(服务端 + * notify/service.notify_run,04-对外集成与触达 B1)。以前叫「实例完成 / 失败」,勾了却永远收不到 + */ +export const NOTIFY_PERSONAL_EVENT_LABEL: Record = { + 'run.finished': '我发起的实例完成', + 'run.failed': '我发起的实例失败', +} + /** 发布弹窗按角色收起的档位(Q8:受管级发布只给管理员,服务端 api/workflows 同样拒) */ export const PUBLISH_ROLE_TEXT = { governedAdminOnly: '需要管理员:以受管级发布只给管理员,请联系管理员发布', diff --git a/frontend/src/pages/apps/personal.tsx b/frontend/src/pages/apps/personal.tsx index c5298c15..2e10036d 100644 --- a/frontend/src/pages/apps/personal.tsx +++ b/frontend/src/pages/apps/personal.tsx @@ -171,7 +171,7 @@ function ReviewDialog({ appId, onClose, onSent }: {

    {T.reviewIntro}

    {T.reviewTo}
    - +

    {T.reviewToHint}

    diff --git a/frontend/src/pages/settings/NotifyTab.tsx b/frontend/src/pages/settings/NotifyTab.tsx index d30ffda5..0be8ce32 100644 --- a/frontend/src/pages/settings/NotifyTab.tsx +++ b/frontend/src/pages/settings/NotifyTab.tsx @@ -6,6 +6,7 @@ import { } from '../../components/ui' import { formatDateTime } from '../../lib/format' import { NOTIFY_TEXT as T, SETTINGS_ROLE_TEXT } from '../../lib/terms' +import { NOTIFY_PERSONAL_EVENT_LABEL } from '../../lib/termsApp' import { useAtLeast } from '../../store/auth' import type { NotifyChannelRow } from '../../typesApp' @@ -18,6 +19,9 @@ import type { NotifyChannelRow } from '../../typesApp' const FORMATS = ['generic', 'feishu', 'wecom', 'dingtalk'] as const const EVENTS = Object.keys(T.eventLabel) +/** 事件在这个范围的渠道上怎么叫:个人渠道的实例结束只推自己发起的,叫「我发起的实例完成 / 失败」 */ +const eventLabel = (event: string, scope: string): string => + (scope === 'user' ? NOTIFY_PERSONAL_EVENT_LABEL[event] : undefined) ?? T.eventLabel[event] ?? event export function NotifyTab() { const builder = useAtLeast('builder') @@ -86,7 +90,7 @@ export function NotifyTab() {
    {row.url_host} - {row.events.length > 0 && ` · ${row.events.map((e) => T.eventLabel[e] ?? e).join('、')}`} + {row.events.length > 0 && ` · ${row.events.map((e) => eventLabel(e, row.scope)).join('、')}`}