diff --git a/README.md b/README.md index 6b75eb5f..d890233e 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,21 @@ http://127.0.0.1:8000 。前端产物不存在或比源码旧时才重新构建 MCP 要管理员先在「工具 → 端点」登记。 - `/api/copilot/generate` 改图时只出改动操作,不再是整张新图;每次运行多一条 `run.origin` 事件(在封存范围内)。 +**从旧版本升级(2026-10 下一轮第 1 波)**:有几处行为变化会影响现有部署: + +- 人工驳回(审批、动作确认、填写、代码和工具审批、双盲复算)的运行记为「已取消 · 被驳回」,不再算失败:通知走「实例 + 结束」,定时任务推「未出具(被驳回)」,不能继续运行。配了失败重试或「出错时继续」的关卡,驳回不再重问、也不接着往下走。 + 已封存的老运行不改写。 +- 开了登录时,应用入口和待办表单里的选人控件按候选池核对(缺省是能用这个应用的人),池外的名字回 422(`principal`)。 +- 页面默认只能被本站嵌入(`frame-ancestors 'self'`,另加 `X-Frame-Options`)。已经用 iframe 嵌 AgentLab 的门户, + 在 `AGENTLAB_FRAME_ANCESTORS` 里写上门户的地址。 +- 「对外访问地址」只剩一个设置(「设置 → 通用」;`AGENTLAB_PUBLIC_URL` 优先,写错了启动就报错)。设置里填了地址的部署, + OIDC 回调改按这个地址拼,和身份提供方登记的不一致时要去改登记。开了登录时 `/docs`、`/openapi.json` 要搭建者登录。 +- 探查结构时列类型改存各数据库的完整写法(`VARCHAR2(20 CHAR)`、`ENUM('a','b')`):升级后每个数据源点一次「探查结构」。 + 新运行的表结构快照哈希会变,旧运行不受影响。MySQL / MariaDB 上文字列的剖析说明多一句排序规则,ENUM 列补上数据里没出现的 + 取值,日期列写明排除了几个无效日期,所以升级后第一次重新剖析,这些表的目录各升一次版本。剖析设置的「抽样键值数」改叫 + 「核对键值数」(`DISTINCT … LIMIT` 取的不是随机样本)。 + --- ## 核心能力 @@ -103,10 +118,10 @@ http://127.0.0.1:8000 。前端产物不存在或比源码旧时才重新构建 | **能力登记与端点** | 所有可调用的工具登记在一处,逐项治理读写效果、审批、字段语义、遮罩;外部系统按测试 / 生产两个环境登记成端点,只有登记的端点能连内网,认证头加密;端点读回的表格和查询快照一样能作证([详见](#能力登记与端点)) | | **Agent 护栏** | 同样的调用不重复执行、连续拿不到新信息就收尾、令牌与金额预算、上下文接近窗口先压缩再收尾;步数只留兜底(默认 100) | | **代码沙箱** | 双档隔离:系统沙箱(Seatbelt / bubblewrap,冷启动约 25ms)或 microVM(独立 Linux 内核,内存限额真实生效) | -| **人工介入** | 执行暂停并落盘,人工批准 / 驳回 / 改稿后从断点继续;进程重启不丢状态;可派给岗位、发起人或按表达式算出的人,会签(任一人 / 全体同意,一票否决 / 至少 m 人),设时限,超时提醒、升级或驳回,永远不会自动批准 | +| **人工介入** | 执行暂停并落盘,人工批准 / 驳回 / 改稿后从断点继续;进程重启不丢状态;可派给岗位、发起人或按表达式算出的人,会签(任一人 / 全体同意,一票否决 / 至少 m 人),设时限,超时提醒、升级或驳回,永远不会自动批准;驳回是人的决定,运行记为「被驳回」,不算失败、不重试 | | **断点续跑** | 失败的运行可从失败节点继续,已完成的节点不重复执行;探索运行允许修改节点配置(拓扑必须一致),正式运行固定在发布的版本上、不能改配置后继续;继续运行不改发起人 | | **知识库** | 向量 + BM25 混合检索、倒排索引、可选模型重排;PDF / Word / PowerPoint / HTML / Markdown 解析;向量模型可接本地端点 | -| **表格入库** | Excel / CSV 上传后转为可用 SQL 查询的表,自动生成 `db_query__` 工具——数字由 SQL 计算得出,而非模型读取 | +| **表格入库** | Excel / CSV 上传后转为可用 SQL 查询的表,自动生成 `db_query__` 工具——数字由 SQL 计算得出,而非模型读取;复杂报表按取数配方导入(CSV 同样可以),每月一份的明细能按期累积成一张表,夹在明细中间的小计核对后不进数据 | | **数据目录** | 给数据源的每张表记下粒度、业务主键、业务日期、有效记录条件,每个字段的含义、单位、度量类型和码值,以及表与表的关联;由外键、命名推断、数据剖析和模型起草,逐项确认后生效;表详情里可以和助手对话完善一张表,审阅队列按影响排好先审哪些,目录能导出、导入到别的实例;助手建图、agent 查结构、SQL 检查都读它 | | **SQL 检查** | 按数据目录检查一对多关联后重复计算、存量跨时间相加、比率被相加等七类问题;助手自查时交回修正,受管模板发布时挡住,运行时标在查询结果上、契约指标降档 | | **跨库合并** | 合并查询节点:各库先聚合到同一个键,再在库外按键合并;截断的输入拒收,证据一路追到两边的原始查询 | @@ -115,7 +130,7 @@ http://127.0.0.1:8000 。前端产物不存在或比源码旧时才重新构建 | **自然语言编排** | 助手:一句话生成或改写整张图,自动排版;system 里每个数据源只给一行概览,要看表结构、能力参数、工作流版本时由助手自己「查看」,一次给齐字段、目录和连接条件;交付前按运行时同一套规则自查,查出会运行失败的问题先交回去改,改不好就不自动运行([详见](#助手按需查看)) | | **结构化成果** | JSON Schema 校验,不合格时将错误回传模型自动返工 | | **产出复核** | 运行结束后规则扫描事件流,发现异常才调用模型复核;复核可重组答案但**不得引入原答案中不存在的数字** | -| **追踪与成本** | 全量事件落库可回放;逐节点耗时、token 用量、按目录价折算的成本 | +| **追踪与成本** | 全量事件落库可回放;逐节点耗时、token 用量、按目录价折算的成本;问数据每一轮的花费含助手建图、挑表和复核 | | **三档出具** | 口径卡(受控指标集)+ 出具契约:叙述中每个数字回指指标集,或由报告撰写节点按引用逐段核对,formal / degraded / withheld 三档判定 | | **可点击证据** | 报告撰写节点只写引用标记,数字和表格由系统照快照填值;数字、表名字段名、知识库原话都点得开,一路指到查询结果里的那一格;结论句可要求挂依据或交给另一个模型裁判;证据清单可导出,报告导出为自包含 HTML(数字点得开口径与出处),另有封存哈希核对件「可验证报告」 | | **发布前检查** | 受管级别加证据门禁 G1–G5;问题给出一键修复 / 选项 / 交给 Copilot 三类修法,一律先预览、不许降低要求;老工作流一键升级为可追溯结构 | @@ -738,6 +753,12 @@ Excel / CSV 若传入知识库,只能被切块检索,数字即成为模型 - 内网只对登记的端点开放(管理员开「允许访问内网」),元数据地址和高危端口照样拦;模型给出的地址照旧做完整的 SSRF 检查 - 开了登录:会话 cookie 在库里只存 token 的哈希;非 GET 请求核对来源标记头和 Origin;使用者只能读有授权的运行,看到的查询结果一律遮罩 - 写外部系统必须经「调用工具」节点:先留生效记录再发送,确认卡上的内容哈希在发送前重算比对;结果未知永不自动重发 +- 安全响应头:`nosniff`、`Referrer-Policy`,以及 `frame-ancestors`(缺省 `'self'`,`AGENTLAB_FRAME_ANCESTORS` 列出允许 + 嵌整站的同站门户;只有 `'self'` 时另加 `X-Frame-Options: SAMEORIGIN`)。严格的 `script-src` 目前只发 + `Content-Security-Policy-Report-Only`,观察一个版本没有误报再强制。HSTS 交给反向代理 +- 集成方要用的接口(应用、待办、工件、角色拒绝)的错误一律带机读码(`code`),`detail` 是给人看的原话,可以改 +- 评测、口径对账、口径召回是系统代为发起的运行:证据照常可查,但不进使用统计(探索问题聚类、工具白名单、目录使用 + 次数),记录页默认不列并写明有几次没列出 --- diff --git a/backend/app/api/apps.py b/backend/app/api/apps.py index 68b04481..8d97e99d 100644 --- a/backend/app/api/apps.py +++ b/backend/app/api/apps.py @@ -12,14 +12,18 @@ import dataclasses import json from typing import Any -from fastapi import APIRouter, Depends, Header, HTTPException, Query +from fastapi import APIRouter, Depends, Header, Query from fastapi.responses import JSONResponse from pydantic import BaseModel, Field from sqlalchemy import func, select from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncSession -from app.api.coded import CodedHTTPException +from app.api.coded import ( + APP_BINDINGS_INVALID, APP_DRAFT_INVALID, APP_GRANTS_INVALID, APP_HOOK_NOT_DECLARED, APP_HOOK_NOT_FOUND, + APP_INSTANCE_NOT_FOUND, APP_NOT_FOUND, APP_PARAMS_INVALID, APP_VERSION_NOT_FOUND, FORM_INVALID, RUN_LAUNCH_REFUSED, + CodedHTTPException, +) from app.api.deps import Principal, actor_of, current_principal from app.apps.access import ROLE_TEXT, app_role, at_least, grants_of from app.db.base import get_session @@ -60,11 +64,11 @@ def _principal(x_actor: str | None) -> Principal: async def _app(session: AsyncSession, app_id: str, principal: Principal, need: str) -> tuple[App, str]: app = await session.get(App, app_id) if app is None: - raise HTTPException(404, APP_GONE) + raise CodedHTTPException(404, APP_GONE, APP_NOT_FOUND) role = await app_role(session, principal, app) if role is None: - # 没有任何授权的人不该知道这个应用在不在 - raise HTTPException(404, APP_GONE) + # 没有任何授权的人不该知道这个应用在不在:码和不存在的一样 + raise CodedHTTPException(404, APP_GONE, APP_NOT_FOUND) if not at_least(role, need): raise CodedHTTPException(403, f"需要这个应用的「{ROLE_TEXT[need]}」权限", NO_ACCESS) return app, role @@ -314,7 +318,7 @@ async def derive(payload: DeriveIn, session: AsyncSession = Depends(get_session) try: return {"spec": await derive_app(session, payload.workflow_id, payload.version, name=payload.name)} except DeriveRefused as e: - raise HTTPException(e.status_code, e.message) from e + raise CodedHTTPException(e.status_code, e.message, e.code) from e @router.get("/{app_id}") @@ -359,13 +363,13 @@ async def update_app(app_id: str, payload: AppPatch, session: AsyncSession = Dep app, role = await _app(session, app_id, principal, "manage") _refuse_archived(app, "修改") if payload.draft_bindings is not None and (problem := _bindings_problem(payload.draft_bindings)): - raise HTTPException(422, problem) + raise CodedHTTPException(422, problem, APP_BINDINGS_INVALID) if app.scope == "personal" and payload.draft is not None: # 个人应用的草稿就是在用的那一份:只许改呈现,固定的蓝图、视图来源只由「存成我的应用」改 from app.blueprints.personal import draft_problem if problem := draft_problem(app.draft or {}, payload.draft): - raise HTTPException(422, problem) + raise CodedHTTPException(422, problem, APP_DRAFT_INVALID) for key in ("name", "description", "icon"): if (value := getattr(payload, key)) is not None: setattr(app, key, value) @@ -568,7 +572,7 @@ async def get_version(app_id: str, version: int, session: AsyncSession = Depends row = (await session.execute(select(AppVersion).where( AppVersion.app_id == app.id, AppVersion.version == version))).scalar_one_or_none() if row is None: - raise HTTPException(404, f"「{app.name}」没有 v{version}") + raise CodedHTTPException(404, f"「{app.name}」没有 v{version}", APP_VERSION_NOT_FOUND) # 岗位绑定是这一个部署里的人和组:管理者才看得到 return {**_version_row(row), "spec": row.spec, "bindings": row.bindings if role == "manage" else {"slots": {}}} @@ -611,7 +615,7 @@ async def put_grants(app_id: str, payload: GrantsIn, session: AsyncSession = Dep _refuse_personal(app, "授权给别人") wanted, problem = await grants_problem(session, [(i.principal_kind, i.principal, i.role) for i in payload.items]) if problem: - raise HTTPException(422, problem) + raise CodedHTTPException(422, problem, APP_GRANTS_INVALID) return {"items": await replace_grants(session, app, wanted, actor=actor_of(x_actor))} @@ -713,7 +717,8 @@ async def rotate_hook(app_id: str, name: str, session: AsyncSession = Depends(ge draft, published, rows = await _hook_state(session, app) declared = draft.get(name) or published.get(name) if declared is None: - raise HTTPException(404, f"草稿和当前发布版里都没有声明钩子「{name}」,请先在「钩子」一节里声明它") + raise CodedHTTPException(404, f"草稿和当前发布版里都没有声明钩子「{name}」,请先在「钩子」一节里声明它", + APP_HOOK_NOT_DECLARED) # 从读行锁到提交:两位管理者同时点「生成密钥」,后到的那次读得到前一次插入的行,只轮换它(见 process/hooks.py) async with hooks.hook_lock(app.id, name): row, secret, created = await hooks.rotate_hook(session, app.id, name, kind=declared["kind"], @@ -731,7 +736,7 @@ async def _hook_named(session: AsyncSession, app: App, name: str) -> AppHook: row = await hook_row(session, app.id, name) if row is None: - raise HTTPException(404, f"钩子「{name}」还没有生成密钥") + raise CodedHTTPException(404, f"钩子「{name}」还没有生成密钥", APP_HOOK_NOT_FOUND) return row @@ -778,9 +783,9 @@ def _params(raw: str | None) -> dict[str, Any]: try: value = json.loads(raw) except ValueError as e: - raise HTTPException(422, "页面参数需要是 JSON 对象") from e + raise CodedHTTPException(422, "页面参数需要是 JSON 对象", APP_PARAMS_INVALID) from e if not isinstance(value, dict): - raise HTTPException(422, "页面参数需要是 JSON 对象") + raise CodedHTTPException(422, "页面参数需要是 JSON 对象", APP_PARAMS_INVALID) return value @@ -788,14 +793,16 @@ async def _instance(session: AsyncSession, app: App, run_id: str, principal: Pri """实例要属于这个应用;只有 use 的人只能看自己发起的(关登录时都能看)。""" run = await session.get(Run, run_id) if run is None or run.app_id != app.id or run.origin_kind not in ("app_entry", "hook"): - raise HTTPException(404, "实例不存在") + raise CodedHTTPException(404, "实例不存在", APP_INSTANCE_NOT_FOUND) if principal.mode != "off" and not at_least(role, "observe") and run.started_by != principal.username: raise CodedHTTPException(403, "只能查看自己发起的实例", NO_ACCESS) return run -def _view_refused(e: Any) -> HTTPException: - return CodedHTTPException(e.status_code, e.message, e.code) if e.code else HTTPException(e.status_code, e.message) +def _view_refused(e: Any) -> CodedHTTPException: + """视图、入口、补算、发起运行的拒绝 → 带 code 的 HTTP 错误。只有引擎不发起运行(LaunchRefused)有不带码的几种(工作流 + 没发布、版本不存在、图没通过校验……),补上同一类的 run_launch_refused,detail 照旧说清是哪一种。""" + return CodedHTTPException(e.status_code, e.message, e.code or RUN_LAUNCH_REFUSED) @router.get("/{app_id}/pages/{page_id}") @@ -834,7 +841,7 @@ async def start(app_id: str, entry_id: str, payload: StartIn, session: AsyncSess try: run = await start_entry(app.id, entry_id, payload.values, principal) except FormRejected as e: - return JSONResponse(status_code=422, content={"detail": "表单没有通过校验", "errors": e.errors}) + return JSONResponse(status_code=422, content={"detail": "表单没有通过校验", "code": FORM_INVALID, "errors": e.errors}) except (EntryRefused, LaunchRefused) as e: raise _view_refused(e) from e return {"run_id": run.id, "title": instance_title(run)} @@ -905,7 +912,7 @@ async def refresh_view(app_id: str, view_id: str, payload: RefreshIn, session: A run, cached = await run_view(session, app=app, version=version, view_id=view_id, params=payload.params, principal=principal, actor=actor_of(x_actor)) except FormRejected as e: - return JSONResponse(status_code=422, content={"detail": "参数没有通过校验", "errors": e.errors}) + return JSONResponse(status_code=422, content={"detail": "参数没有通过校验", "code": FORM_INVALID, "errors": e.errors}) except (ViewRefused, LaunchRefused) as e: raise _view_refused(e) from e return {"run_id": run.id, "cached": cached} diff --git a/backend/app/api/artifacts.py b/backend/app/api/artifacts.py index e8099dbe..e210ffba 100644 --- a/backend/app/api/artifacts.py +++ b/backend/app/api/artifacts.py @@ -2,9 +2,12 @@ from __future__ import annotations from typing import Any -from fastapi import APIRouter, HTTPException +from fastapi import APIRouter from sqlalchemy import select +from app.api.coded import ( + ARTIFACT_FORBIDDEN, ARTIFACT_KIND_FORBIDDEN, ARTIFACT_NOT_FOUND, ARTIFACT_TAMPERED, CodedHTTPException, +) from app.api.deps import can_read_artifact, current_principal from app.core import artifact_store @@ -23,14 +26,14 @@ async def get_artifact(artifact_id: str) -> dict[str, Any]: principal = current_principal() restricted = principal.mode != "off" and not principal.at_least("builder") if principal.mode != "off" and not await can_read_artifact(principal, artifact_id): - raise HTTPException(403, "没有权限查看这件工件") + raise CodedHTTPException(403, "没有权限查看这件工件", ARTIFACT_FORBIDDEN) try: content = artifact_store.load(artifact_id) except ValueError as e: # 哈希对不上:明确报给调用方,这正是工件库存在的意义 - raise HTTPException(409, str(e)) from e + raise CodedHTTPException(409, str(e), ARTIFACT_TAMPERED) from e if content is None: - raise HTTPException(404, "工件不存在,可能已被删除") + raise CodedHTTPException(404, "工件不存在,可能已被删除", ARTIFACT_NOT_FOUND) if restricted: content = await _masked_for_user(principal, artifact_id, content) return {"id": artifact_id, "content": content} @@ -68,7 +71,7 @@ async def _masked_for_user(principal: Any, artifact_id: str, content: Any) -> An return await _masked_metrics(principal, artifact_id, content, rows) if _RECEIPT in kinds and isinstance(content, dict): return await _masked_receipt(content, rows) - raise HTTPException(403, "这类工件的原文只对搭建者及以上的角色开放") + raise CodedHTTPException(403, "这类工件的原文只对搭建者及以上的角色开放", ARTIFACT_KIND_FORBIDDEN) async def _masked_snapshot(content: dict[str, Any], *, endpoint: bool) -> dict[str, Any]: @@ -106,7 +109,7 @@ async def _masked_metrics(principal: Any, artifact_id: str, card: dict[str, Any] owner = (run_id, node_id or "") break if owner is None: - raise HTTPException(403, "没有权限查看这件工件") + raise CodedHTTPException(403, "没有权限查看这件工件", ARTIFACT_FORBIDDEN) sealed = await _sealed(owner[0]) masks = _Masks(sealed) metrics: list[Any] = [] diff --git a/backend/app/api/auth.py b/backend/app/api/auth.py index 6a4c87d9..19fc9cf6 100644 --- a/backend/app/api/auth.py +++ b/backend/app/api/auth.py @@ -17,6 +17,7 @@ from pydantic import BaseModel, Field from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession +from app.api.coded import AUTH_REQUIRED from app.api.deps import Principal, current_principal, decode_actor, ensure_role from app.auth import ( MODES, SETTING_KEY, current_mode, env_mode, load_settings, remember_settings, stored_settings, user_can_chat, @@ -25,7 +26,6 @@ from app.auth import oidc, sessions from app.auth.audit import record from app.auth.passwords import hash_password, password_problem, verify_password from app.core import crypto -from app.core.config import settings from app.db.base import get_session from app.db.models import Setting, User @@ -60,7 +60,7 @@ async def me(x_actor: str | None = Header(default=None), session: AsyncSession = return await principal_full(session, Principal(username=actor, display_name=actor, role="admin", mode="off")) if not principal.username: can_bootstrap = principal.mode == "local" and not await _local_admin_exists(session) - return JSONResponse(status_code=401, content={"detail": "请先登录", "code": "auth_required", + return JSONResponse(status_code=401, content={"detail": "请先登录", "code": AUTH_REQUIRED, "mode": principal.mode, "can_bootstrap": can_bootstrap}) return await principal_full(session, principal) @@ -113,7 +113,10 @@ async def logout(request: Request, session: AsyncSession = Depends(get_session)) def _redirect_uri(request: Request) -> str: - base = settings.public_url.rstrip("/") or str(request.base_url).rstrip("/") + """回调地址:对外访问地址(环境变量优先,其次设置,core/public_url.py),都没有按请求自己的地址拼。""" + from app.core.public_url import public_base_url + + base = public_base_url() or str(request.base_url).rstrip("/") return f"{base}/api/auth/oidc/callback" diff --git a/backend/app/api/bundle.py b/backend/app/api/bundle.py index 00e973b0..f0256724 100644 --- a/backend/app/api/bundle.py +++ b/backend/app/api/bundle.py @@ -53,7 +53,8 @@ async def export_bundle(run_id: str) -> Response: raise CodedHTTPException(404, "运行记录不存在", RUN_NOT_FOUND) info = {"id": run.id, "workflow_id": run.workflow_id, "workflow_name": run.workflow_name, "version": run.version, "version_hash": run.version_hash, "run_class": run.run_class, - "status": run.status, "started_at": run.started_at, "finished_at": run.finished_at} + "status": run.status, "end_reason": run.end_reason, "started_at": run.started_at, + "finished_at": run.finished_at} # 封存之后追加的事件(按需裁判、审批质询):_sealed 只留了封存范围内的,这几条另取,单列在页面上 later = [tuple(r) for r in await session.execute( select(RunEvent.seq, RunEvent.type, RunEvent.node_id, RunEvent.data) diff --git a/backend/app/api/coded.py b/backend/app/api/coded.py index d8fdba23..4227b648 100644 --- a/backend/app/api/coded.py +++ b/backend/app/api/coded.py @@ -29,6 +29,55 @@ CATALOG_DRAFT_BUSY = "catalog_draft_busy" CATALOG_DRAFT_MODEL_UNAVAILABLE = "catalog_draft_model_unavailable" CATALOG_DRAFT_FINISHED = "catalog_draft_finished" +# ---------------------------------------------------------------- 集成接口(04 细案 A7) +# 外部系统用访问密钥调的应用、待办、工件接口,以及角色、运行读权限的拒绝:按 code 认是哪种错,detail 照旧是给人看的那句 + +#: 身份:没登录(401,中间件、auth.require 都回它)、角色不够(403,路由级依赖、接口内部的 ensure_role、开登录时的 /docs) +AUTH_REQUIRED = "auth_required" +ROLE_REQUIRED = "role_required" +#: 运行:这次运行读不了(403,api/deps 的证据、运行接口);别人的个人应用刷新、个人蓝图试运行只给拥有者(403) +RUN_FORBIDDEN = "run_forbidden" +RUN_PRIVATE = "run_private" +#: 问数据和助手:使用者没开问数据,或者这个接口不在问数据的范围里(403) +CHAT_FORBIDDEN = "chat_forbidden" + +#: 应用(api/apps.py)。app_not_found 同入口发起时的 apps/entries.APP_GONE;没有任何授权的人也回它(不该知道应用在不在) +APP_NOT_FOUND = "app_not_found" +APP_VERSION_NOT_FOUND = "app_version_not_found" +APP_INSTANCE_NOT_FOUND = "app_instance_not_found" +#: 草稿里、当前发布版里都没有声明这个钩子(404);声明了、还没生成密钥(404) +APP_HOOK_NOT_DECLARED = "app_hook_not_declared" +APP_HOOK_NOT_FOUND = "app_hook_not_found" +#: 改草稿、授权时写进来的不合规(422):岗位绑定、个人应用的草稿(只许改呈现)、授权的人和组 +APP_BINDINGS_INVALID = "app_bindings_invalid" +APP_DRAFT_INVALID = "app_draft_invalid" +APP_GRANTS_INVALID = "app_grants_invalid" +#: 页面参数不是 JSON 对象(422) +APP_PARAMS_INVALID = "app_params_invalid" +#: 入口表单、按需视图的参数没通过校验(422,另带 errors:[{path, code, message}])。同待办回复、入站钩子的表单错误 +FORM_INVALID = "form_invalid" +#: 发起实例、刷新视图、补算时引擎不发起运行,又没有更细的码(engine/launch.LaunchRefused 没带 code 的那几种) +RUN_LAUNCH_REFUSED = "run_launch_refused" +# 派生应用草稿时工作流、版本不存在(404)的码在 apps/derive.py(WORKFLOW_GONE、VERSION_GONE);入口、视图、补算的码 +# 在 apps/entries.py、apps/resolve.py、apps/backfill.py,同这里一样都是 app_ 开头 + +#: 待办(api/tasks.py)。服务层(process/tasks.py 的 TaskRefused)没带码的拒绝按状态补上同一类的码:404 不存在;403 +#: 不能这样做(不是管理员也不是应用的管理者,不能改派、催办);409 待办已经处理过、关闭了、刚被别人回复,或者所属的 +#: 运行不再等这条回复——刷新看最新状态;422 回复的内容不对。带了码的照用自己的(task_not_assigned、form_invalid……) +TASK_NOT_FOUND = "task_not_found" +TASK_FORBIDDEN = "task_forbidden" +TASK_CLOSED = "task_closed" +TASK_INVALID = "task_invalid" + +#: 工件(api/artifacts.py):没有权限(403)、这类工件的原文只给搭建者(403)、不存在(404)、内容和哈希对不上(409) +ARTIFACT_FORBIDDEN = "artifact_forbidden" +ARTIFACT_KIND_FORBIDDEN = "artifact_kind_forbidden" +ARTIFACT_NOT_FOUND = "artifact_not_found" +ARTIFACT_TAMPERED = "artifact_tampered" + +#: 设置(api/settings.py):对外访问地址由环境变量 AGENTLAB_PUBLIC_URL 指定时,设置里那一项不能改成别的值(409) +PUBLIC_URL_ENV_LOCKED = "public_url_env_locked" + class CodedHTTPException(HTTPException): """field:出错的是哪一项(表单据此把报错落到那一格)。可选,没有就不出现在返回里。""" diff --git a/backend/app/api/conversations.py b/backend/app/api/conversations.py index 75e73ebc..b54d7410 100644 --- a/backend/app/api/conversations.py +++ b/backend/app/api/conversations.py @@ -118,7 +118,7 @@ class ConversationOut(BaseModel): turn_count: int = 0 #: 列表里给个副标题,让人一眼认出是哪次聊天 last_question: str = "" - #: 最后一轮停在哪:running | waiting | error | cancelled | suspended | done, + #: 最后一轮停在哪:running | waiting | error | cancelled | rejected | suspended | done, #: 没有轮次时为 None。见 last_status() last_status: str | None = None #: 最后一轮的运行。建图阶段就断了的没有 @@ -173,18 +173,24 @@ BUILD_STALE = timedelta(minutes=15) def last_status( - turn: Any, run_status: str | None, pending: bool, now: datetime | None = None, + turn: Any, run_status: str | None, pending: bool, now: datetime | None = None, end_reason: str | None = None, ) -> str: """一轮停在哪。轮次状态只有 running / done / error 三种,细分要看运行和元数据。 轮次状态是前端写的:页面关掉、服务重启时它停在 running,这时要以运行的状态为准; 运行也没有的,是断在了建图阶段。已经收尾的轮次以轮次自己为准——它写下的就是 用户看到的结局(取消、服务重启挂起在 error 里分开记)。 + + 被审批人驳回结束的(运行 cancelled + end_reason=rejected,05-已知遗留 (a))单独记 rejected,不和手动停止混成 + cancelled:前端落库时 meta 里记着 rejected(outcome 照旧是 cancelled),页面关掉没落库的看运行的 end_reason。 """ - outcome = (turn_meta(turn) or {}).get("outcome") + meta = turn_meta(turn) or {} + outcome = meta.get("outcome") if turn.status == "done": return "done" if turn.status == "error": + if outcome == "cancelled" and meta.get("rejected"): + return "rejected" if outcome == "cancelled" or (turn.error or "").strip() == "已取消": return "cancelled" if outcome == "suspended": @@ -192,7 +198,7 @@ def last_status( live = _from_run(run_status, pending) return live if live in ("running", "waiting") else "suspended" return "error" - live = _from_run(run_status, pending) + live = _from_run(run_status, pending, end_reason) if live: return live updated = turn.updated_at @@ -202,7 +208,9 @@ def last_status( return "error" if stale else "running" -def _from_run(run_status: str | None, pending: bool) -> str | None: +def _from_run(run_status: str | None, pending: bool, end_reason: str | None = None) -> str | None: + if run_status == "cancelled" and end_reason == "rejected": + return "rejected" return { "queued": "running", "running": "running", "interrupted": "waiting" if pending else "suspended", @@ -236,11 +244,14 @@ async def _statuses( run_of = {cid: turn_run_id(t) for cid, t in turns.items()} run_ids = {r for r in run_of.values() if r} runs: dict[str, str] = {} + ended: dict[str, str | None] = {} waiting: set[str] = set() if run_ids: - runs = dict((await session.execute( - select(Run.id, Run.status).where(Run.id.in_(run_ids)) - )).all()) + rows = (await session.execute( + select(Run.id, Run.status, Run.end_reason).where(Run.id.in_(run_ids)) + )).all() + runs = {r: st for r, st, _ in rows} + ended = {r: reason for r, _, reason in rows} stopped = [r for r, st in runs.items() if st == "interrupted"] if stopped: waiting = set((await session.execute( @@ -251,7 +262,8 @@ async def _statuses( out: dict[str, tuple[str, str | None]] = {} for cid, turn in turns.items(): run_id = run_of[cid] if run_of[cid] in runs else None - out[cid] = (last_status(turn, runs.get(run_id or ""), run_id in waiting, now), + out[cid] = (last_status(turn, runs.get(run_id or ""), run_id in waiting, now, + end_reason=ended.get(run_id or "")), run_id) return out diff --git a/backend/app/api/copilot.py b/backend/app/api/copilot.py index 2f8766a5..f8b9e200 100644 --- a/backend/app/api/copilot.py +++ b/backend/app/api/copilot.py @@ -110,11 +110,12 @@ def _tool_catalog(rows: list[Any], *, builtins: frozenset[str] | None = None) -> async def _look_service(session: AsyncSession, payload: GenerateIn, sources: list[Any], *, graph: dict[str, Any] | None, kinds: tuple[str, ...] = ("table", "capability"), - ) -> Any: - """这一轮的查看服务(copilot_context.look_service)。挑选模型用同一个助手模型,第一次要挑时才拿。""" + catalog_view: Any = None) -> Any: + """这一轮的查看服务(copilot_context.look_service)。挑选模型用同一个助手模型,第一次要挑时才拿。catalog_view:表详情 + 里给哪一份数据目录(data/catalog_view.py),不给是当前目录。""" spec = await copilot_model_spec(session, payload, max_tokens=copilot_context.PICK_MAX_TOKENS) return await copilot_context.look_service(sources, spec=spec, graph=graph, kinds=kinds, - answer=payload.intent == "answer") + answer=payload.intent == "answer", catalog_view=catalog_view) NODE_REFERENCE = """\ @@ -1756,45 +1757,81 @@ def _repair_request( @router.post("/generate-stream") -async def generate_stream(payload: GenerateIn, session: AsyncSession = Depends(get_session)): +async def generate_stream(payload: GenerateIn): """流式版生成:SSE 逐操作推送,前端边收边把节点摆上画布。 - 生成过程(copilot_gen)读到 look 就停下、查看、接着写,查看记录作为 look 事件发给前端(前端存进这一轮的 - meta.looks)。结束时(done 或流断)发一条 final:服务端把累积的图排版、校验后整体给出——过程可见性来自操作流, - 最终质量仍由和手工编排相同的 layout + validate 保证。模型问了用户(ask)的,这一轮到此为止,不自查修正、不自动运行。 + 生成过程整个在 answer_ops 里(评测直接迭代它,和问数据页走同一条路),这里只包一层 SSE。开流前的拒绝(模型没配好、 + 对话读不了、限定的数据源都不在)发生在 answer_ops 交出第一条操作之前:先取第一条,拒绝照旧是 HTTP 错误,不进流。 """ from fastapi.responses import StreamingResponse - from app.data.sqlcheck import load_checkers + ops = answer_ops(payload) + first = await ops.__anext__() + async def event_stream(): + yield _sse(first) + async for op in ops: + yield _sse(op) + + return StreamingResponse( + event_stream(), + media_type="text/event-stream", + headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, + ) + + +async def answer_ops(payload: GenerateIn, *, catalog_view: Any = None): + """生成过程:按需求起草、按需查看、自查修正、排版校验,逐条交出操作(op dict)。问数据页(经 SSE)和评测(直接迭代) + 走的是这同一条路(细案 01 做法 2)。 + + 读到 look 就停下、查看、接着写,查看记录作为 look 操作交出(前端存进这一轮的 meta.looks)。结束时(done 或流断)交一条 + final:累积的图排版、校验后整体给出——过程可见性来自操作流,最终质量仍由和手工编排相同的 layout + validate 保证。模型 + 问了用户(ask)的,这一轮到此为止,不自查修正、不自动运行。 + + 开流前的准备(取数据源、读目录检查器、取助手模型、拼历史、组查看服务)用自己的短会话,在交出第一条操作之前做完: + 拒绝在这里抛 HTTPException(调用方没迭代到第一条就知道)。catalog_view 是这一轮读哪一份数据目录(data/catalog_view.py: + 查看的表详情、自查的 SQL 检查器都按它),不给是当前目录。 + + final 带 next(chat/turn.next_step:run 运行、ready 只建不跑、fail 判失败),问数据页照它走。最后多交一条 usage + (providers/usage.py):{op: usage, by_model, input_tokens, output_tokens, calls, cost_usd, + unpriced_calls}——这一轮助手建图(含挑表、接着写、自查修正)花了多少。前面每一条和抽出来之前逐条一样。运行不在这里, + 不计。 + """ from app.api.deps import restricted + from app.chat import turn + from app.data.catalog_view import CatalogView + from app.data.sqlcheck import load_checkers + from app.db.base import SessionLocal + from app.providers import usage - # 带了对话的,先核对这个人能不能用它(D-d):历史、上一轮的图都从它来 - await _ensure_conversation_access(session, payload.conversation_id) - # 开登录后的 user 角色(D-c):图只能用 chat_policy 放行的节点和工具,提示词先说、自查按同一个判断挡 - chat_user = restricted() - sources = await _sources(session, payload.datasource_ids) - scope = {r.name for r in sources} if payload.datasource_ids else None - # 基于数据目录的 SQL 检查器:在开流之前读好目录(流里不再碰这个会话),自查和收尾共用 - checkers = await load_checkers(session, sources) - nodes, edges = _start_graph(payload) - # 用户摆好的坐标,收尾排版时原样留着 - pinned = _pinned_positions(payload.base_graph) + view = CatalogView.parse(catalog_view) + async with SessionLocal() as session: + # 带了对话的,先核对这个人能不能用它(D-d):历史、上一轮的图都从它来 + await _ensure_conversation_access(session, payload.conversation_id) + # 开登录后的 user 角色(D-c):图只能用 chat_policy 放行的节点和工具,提示词先说、自查按同一个判断挡 + chat_user = restricted() + sources = await _sources(session, payload.datasource_ids) + scope = {r.name for r in sources} if payload.datasource_ids else None + # 基于数据目录的 SQL 检查器:在开流之前读好目录(流里不再碰这个会话),自查和收尾共用 + checkers = await load_checkers(session, sources, catalog_view=view) + nodes, edges = _start_graph(payload) + # 用户摆好的坐标,收尾排版时原样留着 + pinned = _pinned_positions(payload.base_graph) - try: - model, model_id = await _op_model(session, payload) - except ProviderNotConfigured as e: - raise HTTPException(400, _unconfigured(e)) from e + try: + model, model_id = await _op_model(session, payload) + except ProviderNotConfigured as e: + raise HTTPException(400, _unconfigured(e)) from e - human = _with_history(_graph_request(payload), await _history_section(session, payload.conversation_id)) - # 兜底用:模型只发了改动操作时,把它们重放到这张图上 - prev_graph = await _previous_graph(session, payload.conversation_id) - # 比对工具绑定的基准:画布上是改之前的图;问数据页是上一轮的图(沿用它的节点 id) - baseline = copy.deepcopy( - (payload.base_graph or {}).get("nodes") or (prev_graph or {}).get("nodes") or []) - # 查看服务:读目录、读能力登记在开流之前做完。现有工作流(画布上的原图、问数据追问时上一轮的图)查过的表列在概览里 - looks = await _look_service(session, payload, sources, graph=payload.base_graph if ( - payload.base_graph or {}).get("nodes") else prev_graph) + human = _with_history(_graph_request(payload), await _history_section(session, payload.conversation_id)) + # 兜底用:模型只发了改动操作时,把它们重放到这张图上 + prev_graph = await _previous_graph(session, payload.conversation_id) + # 比对工具绑定的基准:画布上是改之前的图;问数据页是上一轮的图(沿用它的节点 id) + baseline = copy.deepcopy( + (payload.base_graph or {}).get("nodes") or (prev_graph or {}).get("nodes") or []) + # 查看服务:读目录、读能力登记在开流之前做完。现有工作流(画布上的原图、问数据追问时上一轮的图)查过的表列在概览里 + looks = await _look_service(session, payload, sources, graph=payload.base_graph if ( + payload.base_graph or {}).get("nodes") else prev_graph, catalog_view=view) # 这一份 system 一直用到最后:接着写、自查修正都沿用,不换 system = _graph_system(sources, scope, await copilot_context.section(looks), looks, chat_user=chat_user) applier = GraphApplier(nodes, edges) @@ -1803,8 +1840,8 @@ async def generate_stream(payload: GenerateIn, session: AsyncSession = Depends(g checker=GraphChecker(applier, scope=scope, sources=sources, baseline=baseline, checkers=checkers, chat_user=chat_user)) - async def event_stream(): - yield _sse({"op": "model", "model": model_id}) + async def steps(): + yield {"op": "model", "model": model_id} patches = 0 try: async for op in _with_heartbeat(gen.draft(human), started=started): @@ -1813,23 +1850,24 @@ async def generate_stream(payload: GenerateIn, session: AsyncSession = Depends(g # reply 先压着(applier.replied):前端收到 reply 就收尾,提案得排在它前面转出 if (checked := await catalog_patch_op(op, sources)) is not None: patches += 1 - yield _sse(checked) + yield checked continue - yield _sse(op) + yield op if applier.replied is not None: # 这句话不需要工作流:后面那套排版校验对着一张空图跑,只会得到"图是空的,先拖一个节点进来" - yield _sse(applier.replied) + yield applier.replied return if gen.asked is not None and not applier.effective: # 先问清楚才能动手,什么都还没改:这一轮到此为止。照样发一条 final 收尾,带上问题(asked)和没动过的图: # 前端据此把这一轮记成「助手反问」、问题进历史(以前流就这么断了,问数据页一直停在「进行中」), # 不运行、不重排。图给一份对象(画布上原来的图、问数据上一轮的图,都没有就是空图),老前端照样认得 - yield _sse(_asked_final(gen.asked, payload.base_graph, prev_graph)) + asked_final = _asked_final(gen.asked, payload.base_graph, prev_graph) + yield {**asked_final, "next": turn.next_step(asked_final)} return # 只发了目录修改提案,既没回答也没改图(用户只是在说一条数据事实):补一句回答收尾, # 不能按下面的「模型没有给出任何修改」报错——提案已经摆在用户面前了 if patches and all(o.get("op") == "plan" for o in applier.seen): - yield _sse({"op": "reply", "text": PATCH_ONLY_REPLY}) + yield {"op": "reply", "text": PATCH_ONLY_REPLY} return # 流正常结束,却没有 done、也没有一个操作能认(格式跑偏,每一行都被跳过): # 往下走就是拿原图去自查、交付,界面上写「无需修改」——而模型什么都没说 @@ -1837,12 +1875,11 @@ async def generate_stream(payload: GenerateIn, session: AsyncSession = Depends(g raise CopilotIncomplete("模型没有给出任何修改,也没有说明原因", hint="画布未修改。可以重试,或把需求描述得更具体") except CopilotIncomplete as e: - yield _sse({"op": "error", "message": f"助手本轮未完成:{e}", "hint": e.hint, "detail": e.detail}) + yield {"op": "error", "message": f"助手本轮未完成:{e}", "hint": e.hint, "detail": e.detail} return except Exception as e: # noqa: BLE001 reason, hint = explain_error(e) - yield _sse({"op": "error", "message": f"助手本轮未完成:{reason}", - "hint": hint, "detail": raw_error(e)}) + yield {"op": "error", "message": f"助手本轮未完成:{reason}", "hint": hint, "detail": raw_error(e)} return # 兜底:一个操作都没落地,而上一轮有图。 @@ -1858,7 +1895,7 @@ async def generate_stream(payload: GenerateIn, session: AsyncSession = Depends(g # 以前这些问题照样交付、照样自动开跑——跑到那一步才炸(开发库里四次运行死在 # 循环 / 分支条件里套了 {{ }}),前面的步骤白跑,报错还停在半路 async for op in _with_heartbeat(gen.repair(), phase="repairing"): - yield _sse(op) + yield op remaining = _blocking_issues(nodes, edges, scope, sources=sources, baseline=baseline, checkers=checkers, chat_user=chat_user) # 工具绑定变化不挡运行,但要在自查这一步就说出来:工具被改没了的图照样 @@ -1871,11 +1908,9 @@ async def generate_stream(payload: GenerateIn, session: AsyncSession = Depends(g # 改不好也照实交付、列出问题,但不自动运行:明知跑不起来的图,开跑只会 # 在半路报一个用户看不懂的错 autorun = False - yield _sse({"op": "check", "status": "failed", "issues": remaining, - "warnings": warnings}) + yield {"op": "check", "status": "failed", "issues": remaining, "warnings": warnings} elif remaining is not None: - yield _sse({"op": "check", "status": "passed", "repaired": gen.repaired, - "warnings": warnings}) + yield {"op": "check", "status": "passed", "repaired": gen.repaired, "warnings": warnings} # 收尾:排版 + 校验,把最终图整体交付。改图时旧节点留在原处,只排新节点 try: @@ -1913,16 +1948,17 @@ async def generate_stream(payload: GenerateIn, session: AsyncSession = Depends(g # 改了图又问了用户(这一轮照样结束、不自动运行):问题跟着 final 交付,前端记进这一轮 **({"asked": _asked_of(gen.asked)} if gen.asked is not None else {}), } + # 建完之后怎么走(chat/turn.next_step):run 运行、ready 只建不跑、fail 判失败。问数据页照它走,评测用同一个函数 + final["next"] = turn.next_step(final) except Exception as e: # noqa: BLE001 final = {"op": "error", "message": f"模型返回的工作流结构有误:{graph_error(e)}", "hint": "请重试,或把需求描述得更具体", "detail": raw_error(e)} - yield _sse(final) + yield final - return StreamingResponse( - event_stream(), - media_type="text/event-stream", - headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, - ) + spent = usage.meter() + async for op in usage.metered(steps(), spent): + yield op + yield {"op": "usage", **spent.summary()} # -------------------------------------------------------------------------- @@ -2774,13 +2810,31 @@ class ReviewIn(BaseModel): #: 默认只在扫出异常信号时才叫模型。打开它就每次都复核——排查时有用, #: 平时不开:干净的运行没什么可复核的,代价却是每次问数都多一次调用 always: bool = False + #: 这一轮已经自动试了几次(从 0 数,同 chat/turn.retry_after_review):结果里的 retry_instruction 据此给不给。 + #: 不传按第一次算 + attempt: int = Field(default=0, ge=0) @router.post("/review") async def review_run( payload: ReviewIn, session: AsyncSession = Depends(get_session) ) -> dict[str, Any]: - """接住一次编排的产出,判断它能不能就这么交给用户。 + """接住一次编排的产出,判断它能不能就这么交给用户(规则见 review_of_run)。这里只核对能不能读这次运行。""" + from app.api.deps import PRIVATE_RUN, can_read_run, current_principal + from app.db.models import Run + + run = await session.get(Run, payload.run_id) + if not run: + raise HTTPException(404, "运行记录不存在,可能已被删除") + if not await can_read_run(current_principal(), run): + raise HTTPException(403, PRIVATE_RUN) + return await review_of_run(session, run, question=payload.question, provider=payload.provider, + model=payload.model, always=payload.always, attempt=payload.attempt) + + +async def review_of_run(session: AsyncSession, run: Any, *, question: str, provider: str | None = None, + model: str | None = None, always: bool = False, attempt: int = 0) -> dict[str, Any]: + """复核一次运行(问数据页经 /copilot/review、评测直接调,同一个函数)。 「跑完了」和「答得对」是两回事。运行状态只说明流程走完了,而过程中检索 降级、工具报错、agent 步数用满这些事,原先一件都不会进到答案里——用户拿到 @@ -2788,47 +2842,57 @@ async def review_run( 规则层先扫一遍(engine/review.py:scan),没扫出东西就到此为止、直接返回, 不花一次模型调用;扫出来了才叫模型重组答案并把异常说破。 - """ - from app.api.deps import PRIVATE_RUN, can_read_run, current_principal - from app.db.models import Run, RunEvent - from app.engine import review as rv - run = await session.get(Run, payload.run_id) - if not run: - raise HTTPException(404, "运行记录不存在,可能已被删除") - if not await can_read_run(current_principal(), run): - raise HTTPException(403, PRIVATE_RUN) + 结果是 ReviewResult.as_dict(),另带两项: + - retry_instruction:要不要带着原图重来一次(chat/turn.retry_after_review,按 attempt 算这一轮还能不能再试)——要的话 + 是交给助手的那句话(chat/turn.retry_instruction),不要是 null。问数据页照它走; + - usage:复核这一次模型调用花了多少(providers/usage.py 的汇总,没调模型时是全 0),这一轮的花费 = 助手 + 运行 + 复核。 + """ + from app.chat import turn + from app.db.models import RunEvent + from app.engine import review as rv + from app.providers import usage rows = list((await session.execute( - select(RunEvent).where(RunEvent.run_id == payload.run_id).order_by(RunEvent.seq) + select(RunEvent).where(RunEvent.run_id == run.id).order_by(RunEvent.seq) )).scalars()) output = run.output if isinstance(run.output, dict) else None signals = rv.scan(rows, output) answer = rv.answer_text(output) + spent = usage.meter() - if not signals and not payload.always: - return rv.ReviewResult(verdict="ok", signals=[]).as_dict() + def done(result: Any) -> dict[str, Any]: + out = result.as_dict() + out["retry_instruction"] = ( + turn.retry_instruction(question, turn.review_reason(out), turn.has_nodes(run.graph)) + if turn.retry_after_review(out, attempt) else None) + out["usage"] = spent.summary() + return out + + if not signals and not always: + return done(rv.ReviewResult(verdict="ok", signals=[])) try: spec = await copilot_model_spec(session, GenerateIn( - instruction="review", provider=payload.provider, model=payload.model, + instruction="review", provider=provider, model=model, ), max_tokens=4096) - model, _ = await get_chat_model(session, spec) + chat_model, _ = await get_chat_model(session, spec) except (ProviderNotConfigured, HTTPException) as e: # 没配模型不该让这一轮挂掉:规则层已经知道出了什么事,照样说得出话 - return rv.ReviewResult( + return done(rv.ReviewResult( verdict="annotated", note=";".join(s.detail for s in signals) or ( _unconfigured(e) if isinstance(e, ProviderNotConfigured) else str(e.detail) ), retry=rv.should_retry(signals), signals=signals, severity=rv.worst(signals), - ).as_dict() + )) - result = await rv.review( - model, question=payload.question, answer=answer, signals=signals, - # 报告撰写节点产出的答案逐段对应着证据文档:只能加说明,不能改写 - locked=bool(output and output.get("_evidence")), - ) - return result.as_dict() + with usage.metering(spent): + result = await rv.review( + chat_model, question=question, answer=answer, signals=signals, + # 报告撰写节点产出的答案逐段对应着证据文档:只能加说明,不能改写 + locked=bool(output and output.get("_evidence")), + ) + return done(result) diff --git a/backend/app/api/copilot_blueprint.py b/backend/app/api/copilot_blueprint.py index 8068edc0..32cc2064 100644 --- a/backend/app/api/copilot_blueprint.py +++ b/backend/app/api/copilot_blueprint.py @@ -293,7 +293,7 @@ async def blueprint_stream(payload: BlueprintStreamIn, session: AsyncSession = D system = "\n\n".join(part for part in ( BLUEPRINT_SYSTEM_PROMPT, _templates_text(personal=personal), await looks.overview(), BLUEPRINT_PROTOCOL_PROMPT, looks.guide(), BLUEPRINT_LOOK_WHEN_PROMPT) if part) - missing = [i for i in blueprint_issues(spec, for_="trial") if i["level"] == "error"] + missing = [i for i in blueprint_issues(spec, for_="trial", graph=row.graph) if i["level"] == "error"] human = copilot._with_history(BLUEPRINT_REQUEST_PROMPT.format( spec=json.dumps(spec, ensure_ascii=False), instruction=instruction, issues=BLUEPRINT_ISSUES_PROMPT.format(lines="\n".join(f"- {i['path']} {i['message']}" for i in missing)) diff --git a/backend/app/api/copilot_context.py b/backend/app/api/copilot_context.py index 6c8e807e..7375d5f0 100644 --- a/backend/app/api/copilot_context.py +++ b/backend/app/api/copilot_context.py @@ -65,16 +65,17 @@ SYSTEM_LOOK_UPGRADE_NEED = "要升级的工作流里,查库节点的 SQL 用 async def look_service(sources: list[Any], *, spec: ModelSpec | None, graph: Mapping[str, Any] | None = None, - kinds: tuple[str, ...] = ("table", "capability"), answer: bool = False) -> LookService: + kinds: tuple[str, ...] = ("table", "capability"), answer: bool = False, + catalog_view: Any = None) -> LookService: """按入口组装这一轮的查看服务。读目录、读能力登记都用各自的短会话,开流前后都能调。 spec 是助手的模型规格:挑选模型第一次要用时才按它拿(关思考、额度按 look/pick.PICK_MAX_TOKENS)。graph 是现有工作流,概览里列出它查过的表。 - answer:问数据(意图为回答),能力只列只读的。 + answer:问数据(意图为回答),能力只列只读的。catalog_view:表详情里给哪一份数据目录(data/catalog_view.py),不给是当前目录。 """ picker = Picker(spec) made: list[Any] = [] if "table" in kinds: - made.append(await TableKind.load(sources, graph=graph, picker=picker)) + made.append(await TableKind.load(sources, graph=graph, picker=picker, catalog_view=catalog_view)) if "capability" in kinds: made.append(await CapabilityKind.load(picker=picker, read_only=answer)) return LookService(made) diff --git a/backend/app/api/datasources.py b/backend/app/api/datasources.py index 38bebb9b..3f8a972f 100644 --- a/backend/app/api/datasources.py +++ b/backend/app/api/datasources.py @@ -18,7 +18,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.api import health from app.api.coded import PROFILE_SETTINGS_INVALID, CodedHTTPException -from app.api.deps import signer_of +from app.api.deps import signer_of, signer_verified from app.core.errors import AUTH, NETWORK, TIMEOUT, classify, explain, raw from app.core.config import settings from app.core.crypto import encrypt, mask @@ -1104,7 +1104,8 @@ async def purge_raw( if imp.raw_state == "purged": raise CodedHTTPException(409, "这次导入的原件已经清除", "raw_already_purged") try: - purged = await table_versions.purge_import_raw(session, imp, reason=reason, signed_by=signed_by) + purged = await table_versions.purge_import_raw(session, imp, reason=reason, signed_by=signed_by, + signer_verified=signer_verified()) except table_versions.StoreUnavailable as e: raise CodedHTTPException(503, str(e), "store_unavailable") from e deleted, others = purged.deleted, purged.others diff --git a/backend/app/api/deps.py b/backend/app/api/deps.py index 6fa03e2d..9b65b17e 100644 --- a/backend/app/api/deps.py +++ b/backend/app/api/deps.py @@ -26,9 +26,11 @@ from dataclasses import dataclass, field from typing import Any from urllib.parse import unquote -from fastapi import Depends, HTTPException +from fastapi import Depends from starlette.requests import HTTPConnection +from app.api.coded import CHAT_FORBIDDEN, ROLE_REQUIRED, RUN_FORBIDDEN, RUN_PRIVATE, CodedHTTPException + #: 登录方式 AUTH_MODES = ("off", "local", "oidc") #: 角色由高到低 @@ -105,6 +107,15 @@ def signer_of(given: Any, header: str | None, limit: int = 100) -> str | None: return decode_actor(header) +def signer_verified() -> bool: + """signer_of 记下的署名是不是认证过的身份:开了登录、这次请求的身份认证过时为真;关登录时署名是自报的,为假。 + + 导入清单、累积快照清单、作废记录、清除原件记录的「是否认证」记这个(docs/design/next-round/05-已知遗留.md (c)), + 和审计表、待办派单、run.origin 的 verified 同一个口径。证据要在生成那一刻定下:调用方和 signed_by 一起往下传。""" + principal = _principal.get() + return principal is not None and principal.mode != "off" and bool(principal.verified) + + def current_principal() -> Principal: """这次请求的完整身份。关登录时:等同 admin、未认证,username 是中间件按 X-Actor 解出的自报署名;中间件没写 (不在请求里,比如测试直接调函数、启动时的整理、调度循环)时同样等同 admin、未认证,username 为空。""" @@ -140,7 +151,7 @@ async def can_read_artifact(principal: Principal, artifact_id: str) -> bool: def ensure_role(role: str) -> None: - """接口内部用:至少是 role,否则 401(没登录)/ 403(角色不够)。关登录时放行。 + """接口内部用:至少是 role,否则 401(没登录,auth_required)/ 403(角色不够,role_required)。关登录时放行。 用在要看请求内容才知道要什么角色的地方(比如 MCP 的 stdio 只有管理员能建)。""" from app.auth import require @@ -169,8 +180,8 @@ async def ensure_conversation_access(session: Any, conversation_id: str | None) return await check(session, conversation_id) -def _forbidden() -> HTTPException: - return HTTPException(403, "没有权限查看这次运行") +def _forbidden() -> CodedHTTPException: + return CodedHTTPException(403, "没有权限查看这次运行", RUN_FORBIDDEN) #: user 角色在自己的问数据运行上能读的(GET /api/runs/{id} 之后的部分):运行记录、事件、运行时的图,加上实时流 @@ -235,13 +246,13 @@ async def _run_scope(conn: HTTPConnection) -> None: return ensure_role("user") if not user_can_chat(): - raise HTTPException(403, _RUNS_FOR_BUILDERS) + raise CodedHTTPException(403, _RUNS_FOR_BUILDERS, ROLE_REQUIRED) method = conn.scope.get("method", "GET") run_id = conn.path_params.get("run_id") if run_id is None: if method == "POST" and conn.url.path.rstrip("/") == "/api/runs": return - raise HTTPException(403, _RUNS_FOR_BUILDERS) + raise CodedHTTPException(403, _RUNS_FOR_BUILDERS, ROLE_REQUIRED) run = await _load_run(run_id) rest = conn.url.path.rstrip("/").removeprefix(f"/api/runs/{run_id}") if method == "GET" and rest in _CHAT_RUN_READS: @@ -252,7 +263,7 @@ async def _run_scope(conn: HTTPConnection) -> None: raise _forbidden() if method == "POST" and rest == "/cancel": return - raise HTTPException(403, "没有权限:这项操作需要搭建者及以上的角色") + raise CodedHTTPException(403, "没有权限:这项操作需要搭建者及以上的角色", ROLE_REQUIRED) async def _load_run(run_id: str | None) -> Any: @@ -282,7 +293,7 @@ async def _refuse_private(conn: HTTPConnection, principal: Principal) -> None: from fastapi import WebSocketException, status raise WebSocketException(code=status.WS_1008_POLICY_VIOLATION) - raise HTTPException(403, PRIVATE_RUN) + raise CodedHTTPException(403, PRIVATE_RUN, RUN_PRIVATE) async def _private_run_scope(conn: HTTPConnection) -> None: @@ -310,7 +321,7 @@ async def _evidence_scope(conn: HTTPConnection) -> None: return ensure_role("user") if conn.scope.get("method", "GET") != "GET": - raise HTTPException(403, "没有权限:这项操作需要搭建者及以上的角色") + raise CodedHTTPException(403, "没有权限:这项操作需要搭建者及以上的角色", ROLE_REQUIRED) run_id = conn.path_params.get("run_id") if not run_id or not await can_read_run(principal, run_id): raise _forbidden() @@ -327,7 +338,8 @@ async def _chat_scope(conn: HTTPConnection) -> None: method = conn.scope.get("method", "GET") if user_can_chat() and (path.startswith("/api/conversations") or (method, path) in _CHAT_ROUTES): return - raise HTTPException(403, "没有权限:问数据和助手需要搭建者及以上的角色,或由管理员为使用者开启问数据") + raise CodedHTTPException(403, "没有权限:问数据和助手需要搭建者及以上的角色,或由管理员为使用者开启问数据", + CHAT_FORBIDDEN) #: user 角色开了问数据时能用的其余接口:问数据的生成流、读当前助手模型、问数据页选数据源用的数据源名单 diff --git a/backend/app/api/evidence.py b/backend/app/api/evidence.py index 75a77730..9ce10855 100644 --- a/backend/app/api/evidence.py +++ b/backend/app/api/evidence.py @@ -2851,7 +2851,7 @@ async def export_report_html(run_id: str, report: str | None = None) -> Response if run is None: # 两次读之间被删掉了 raise CodedHTTPException(404, "运行记录不存在", RUN_NOT_FOUND) info = {"id": run.id, "workflow_id": run.workflow_id, "workflow_name": run.workflow_name, - "run_class": run.run_class, "status": run.status, + "run_class": run.run_class, "status": run.status, "end_reason": run.end_reason, "started_at": run.started_at, "finished_at": run.finished_at} data = build_report(info, report_node=chosen.node_id, doc_artifact=chosen.doc_artifact, doc_hash_ok=chosen.hash_ok, doc=doc, seal=sealed.seal, chains=chains, refs=refs, diff --git a/backend/app/api/governance.py b/backend/app/api/governance.py index 710e1707..a5a73074 100644 --- a/backend/app/api/governance.py +++ b/backend/app/api/governance.py @@ -1,6 +1,5 @@ from __future__ import annotations -import json from collections import defaultdict from typing import Any @@ -45,11 +44,14 @@ async def tool_usage( merged.extend(agent.get("tools") or []) granted[node.id] = sorted(set(merged)) + from app.engine.origin import counted_runs + + # 只算计入统计的运行(engine/origin.counted_runs):评测、口径对账、召回替人跑的运行用到了哪些工具不说明白名单该不该收 run_ids = [ rid for (rid,) in await session.execute( select(Run.id) - .where(Run.workflow_id == workflow_id) + .where(Run.workflow_id == workflow_id, counted_runs()) .order_by(Run.created_at.desc()) .limit(limit_runs) ) @@ -95,62 +97,39 @@ async def exploratory_clusters( ) -> dict[str, Any]: """探索性问题聚类:同一个问题被反复问,就该长成模板里的固定一节。 - 这就是模板生长的触发器。用本地 embedding 做贪心聚类, - 出现 ≥3 次的簇标记为晋升候选。 + 这就是模板生长的触发器。取问题、聚类全站只有一份(app/chat/questions.py):只取问题原文(不含 history),问数据一轮 + 一条(自动重来不多算),评测这类系统派生的运行、别人的个人应用刷新和个人蓝图试运行都不进。出现 ≥3 次的簇标记为晋升 + 候选。embedder 是聚类用的向量模型:本地哈希的语义能力弱,结果只是下限。 """ - from sqlalchemy import not_ - from app.api.deps import current_principal - from app.auth.access import hidden_runs - from app.memory.embeddings import cosine, embed_texts + from app.chat.questions import cluster_questions, embedder, recent_questions - rows = list( - await session.execute( - select(Run.id, Run.input, Run.workflow_name, Run.created_at) - # 别人的个人应用刷新、个人蓝图试运行只给拥有者(同运行列表),不进聚类 - .where(Run.run_class != "formal", not_(hidden_runs(current_principal()))) - .order_by(Run.created_at.desc()) - .limit(limit) - ) - ) - items = [] - for rid, input_payload, wf_name, created in rows: - text = json.dumps(input_payload or {}, ensure_ascii=False, sort_keys=True) - if len(text) <= 4: # 空输入没有聚类意义 - continue - items.append({"run_id": rid, "text": text, "workflow": wf_name, - "at": created.isoformat() if created else None}) - if not items: - return {"clusters": [], "considered": 0} + questions = await recent_questions(session, current_principal(), limit=limit) + if not questions: + return {"clusters": [], "considered": 0, "embedder": embedder()} + clusters = await cluster_questions(questions, threshold=threshold) - vectors = await embed_texts([i["text"] for i in items]) - clusters: list[dict[str, Any]] = [] - for i, item in enumerate(items): - placed = False - for cluster in clusters: - if cosine(vectors[i], vectors[cluster["seed"]]) >= threshold: - cluster["members"].append(item) - placed = True - break - if not placed: - clusters.append({"seed": i, "members": [item]}) + def _at(value: Any) -> str | None: + return value.isoformat() if value else None out = [ { - "count": len(c["members"]), - "representative": c["members"][0]["text"][:200], - "workflow": c["members"][0]["workflow"], - "run_ids": [m["run_id"] for m in c["members"]][:20], - "first_at": c["members"][-1]["at"], - "last_at": c["members"][0]["at"], - "promote_candidate": len(c["members"]) >= 3, + "count": c.count, + "representative": c.members[0].text[:200], + "workflow": c.members[0].workflow, + "run_ids": [m.run_id for m in c.members if m.run_id][:20], + "turn_ids": [m.turn_id for m in c.members if m.turn_id][:20], + "first_at": _at(c.members[-1].at), + "last_at": _at(c.members[0].at), + "promote_candidate": c.count >= 3, } for c in clusters - if len(c["members"]) >= 2 + if c.count >= 2 ] out.sort(key=lambda c: c["count"], reverse=True) return { "clusters": out, - "considered": len(items), + "considered": len(questions), + "embedder": embedder(), "hint": "被标记为可固化的问题反复出现,建议用「提取模板」把它固定为工作流中的一步", } diff --git a/backend/app/api/identity.py b/backend/app/api/identity.py index 01759cd5..845f9b64 100644 --- a/backend/app/api/identity.py +++ b/backend/app/api/identity.py @@ -27,6 +27,9 @@ router = APIRouter(prefix="/api", tags=["identity"]) #: 人和组的名录(GET /api/directory):只读、给搭建者及以上(main.py 挂 require_role("builder")),岗位绑定、应用授权的 #: 选人控件据此给候选、核对填的名字在不在。只有用户名、显示名、是否停用和组名,不含邮箱、角色、登录时间 directory_router = APIRouter(prefix="/api", tags=["identity"]) +#: 使用者选人的候选(GET /api/directory/candidates,05-已知遗留 (b)):任何登录的人都能调(main.py 挂 require_role("user")), +#: 按用途给候选池,只有用户名和显示名 +candidates_router = APIRouter(prefix="/api", tags=["identity"]) def _iso(value: Any) -> str | None: @@ -224,6 +227,56 @@ async def directory(session: AsyncSession = Depends(get_session)) -> dict[str, A "groups": [{"name": g.name, "label": g.label} for g in known.groups.values()]} +@candidates_router.get("/directory/candidates") +async def candidates(purpose: str, app_id: str | None = None, entry: str | None = None, task_id: str | None = None, + field: str | None = None, q: str | None = None, limit: int = 20, + session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """使用者选人时的候选:{users: [{username, display_name}], groups: [{name, label}], truncated, needs_query?}。 + + 不给是否停用(停用的直接不列)、角色、邮箱、组成员。purpose=form:表单里的选人控件,带 app_id 和 entry(入口表单) + 或 task_id(待办表单)、field(哪个控件),池按字段的 x-principal 算、缺省是能用这个应用的人,要求调用的人能用这个 + 应用(否则 403);应用授权给了所有人时不带前缀 q 不列(needs_query)。purpose=review:个人应用提交审核,启用的搭建者 + 和管理员。q 按用户名或显示名的开头筛,limit 最多 50。开了登录时提交的值按同一个池核对(auth/directory.principal_errors)。""" + from app.api.coded import CodedHTTPException + from app.api.deps import current_principal + from app.apps.access import app_role, at_least + from app.apps.tree import entry_of, resolve_form + from app.auth.directory import PURPOSES, field_pool, principal_fields, review_pool + from app.db.models_apps import App, AppVersion + from app.db.models_process import Task + + if purpose not in PURPOSES: + raise CodedHTTPException(422, f"候选的用途只能是 {' / '.join(PURPOSES)}", "candidates_purpose") + if purpose == "review": + return (await review_pool(session)).pick(q, limit) | {"groups": []} + principal = current_principal() + bindings: dict[str, Any] | None = None + form: Any = None + task = await session.get(Task, task_id) if task_id else None + app = await session.get(App, task.app_id if task is not None else app_id) if (task or app_id) else None + if app is None or (task_id and task is None): + raise CodedHTTPException(404, "应用或待办不存在", "candidates_not_found") + if not at_least(await app_role(session, principal, app), "use"): + raise CodedHTTPException(403, "你没有这个应用的使用权限,看不到它的候选人", "candidates_forbidden") + if task is not None: + form = task.form + if task.app_version is not None: + bindings = await session.scalar(select(AppVersion.bindings).where( + AppVersion.app_id == app.id, AppVersion.version == task.app_version)) + else: + version = (await session.execute(select(AppVersion).where( + AppVersion.app_id == app.id, AppVersion.version == app.published_version))).scalar_one_or_none() \ + if app.published_version else None + spec = version.spec if version is not None else {} + found = entry_of(spec or {}, entry or "") if entry else None + form = resolve_form(spec or {}, found.get("form")) if found else None + bindings = version.bindings if version is not None else None + target = principal_fields(form).get(field or "") + if target is None: + raise CodedHTTPException(404, "表单里没有这个选人控件", "candidates_not_found") + return (await field_pool(session, app, bindings, target)).pick(q, limit) + + # ---------------------------------------------------------------- 审计 diff --git a/backend/app/api/notify.py b/backend/app/api/notify.py index f5e4ddbb..8702528a 100644 --- a/backend/app/api/notify.py +++ b/backend/app/api/notify.py @@ -21,7 +21,7 @@ from app.core.crypto import encrypt from app.db.base import SessionLocal from app.db.models import NotifyChannel, NotifyDelivery from app.notify import core -from app.notify.service import EVENTS, EVENT_LABEL, channel_out +from app.notify.service import EVENTS, channel_out, subscribe_problem router = APIRouter(prefix="/api/notify", tags=["notify"]) @@ -61,12 +61,14 @@ async def _safe_personal(scope: str, url: str) -> None: raise HTTPException(422, problem) -def _validated(payload: ChannelIn, *, url: str | None) -> None: +def _validated(payload: ChannelIn, *, url: str | None, saved: list[str] | None = None) -> None: + """saved 是渠道已经订阅着的事件:只核对这次新勾的(notify/service.subscribe_problem)。以前存下的照旧——当时界面对 + 哪种渠道都列全部事件,现在才按事件推给哪几条路核对,不能让这些渠道连开关、改名都 422。""" if payload.format not in core.FORMATS: raise HTTPException(422, f"消息格式只能是{'、'.join(core.FORMAT_LABEL.values())}之一") - unknown = [e for e in payload.events if e not in EVENTS] - if unknown: - raise HTTPException(422, f"不认识的通知事件:{'、'.join(unknown)}。可选:{'、'.join(EVENT_LABEL.values())}") + for event in dict.fromkeys(payload.events): + if event not in (saved or ()) and (problem := subscribe_problem(event, payload.scope)): + raise HTTPException(422, problem) if url is not None and (problem := core.format_problem(payload.format, url)): raise HTTPException(422, problem) @@ -84,8 +86,10 @@ async def list_channels() -> dict[str, Any]: @router.get("/events") -async def list_events() -> list[dict[str, str]]: - return [{"event": e, "label": EVENT_LABEL[e]} for e in EVENTS] +async def list_events() -> list[dict[str, Any]]: + """登记的通知事件和推给哪几条路(typesApp.ts 的 NotifyEventRow):设置页、应用的通知按 scopes 列可订阅的事件。 + scopes 是空的,是登记了、还没有推送的,照实给出来,由界面不列。""" + return [{"event": name, "label": event.label, "scopes": list(event.scopes)} for name, event in EVENTS.items()] @router.post("/channels", status_code=201) @@ -150,9 +154,9 @@ async def update_channel(channel_id: str, payload: ChannelIn) -> dict[str, Any]: from app.core.crypto import decrypt current = decrypt(row.url) or "" - _validated(payload, url=current or None) + _validated(payload, url=current or None, saved=list(row.events or [])) else: - _validated(payload, url=url) + _validated(payload, url=url, saved=list(row.events or [])) await _safe_personal(row.scope, url) row.url = encrypt(url) row.name, row.format = payload.name.strip(), payload.format diff --git a/backend/app/api/runs.py b/backend/app/api/runs.py index 3a35364e..bc3ce553 100644 --- a/backend/app/api/runs.py +++ b/backend/app/api/runs.py @@ -56,6 +56,9 @@ class RunOut(BaseModel): input: dict[str, Any] output: dict[str, Any] error: str | None + #: 结束在 cancelled 的原因(docs/design/next-round/00-总方案.md X9):rejected 人工驳回、stopped 手动停止。 + #: 其余终态、加这一列之前的运行为空。界面、通知、定时推送都读这一个字段,不再解析报错原文 + end_reason: str | None = None usage: dict[str, Any] run_class: str | None = "exploratory" version: int | None = None @@ -171,13 +174,54 @@ async def list_runs( 以前只有 limit:待审批的运行排在第 145 位,列表只取 100 条,点了徽标也找不到它。 筛选都走服务端,翻页用 created_at 做游标(before 传上一页最后一条的 created_at)。 别人的个人应用刷新、个人蓝图试运行不列(只给拥有者,auth/access.hidden_runs),管理员也一样。 + 没按来源筛时,系统派生的运行(评测、口径对账、口径召回:engine/origin.SYSTEM_ORIGINS)默认不列——几十道评测题的运行 + 会把真实用户的运行挤出第一页;按来源点名就列。没列出几次由 GET /api/runs/unlisted 照实说,界面写在列表底下。 """ + stmt = _filtered(select(Run), workflow_id=workflow_id, status=status, run_class=run_class, q=q, + schedule_id=schedule_id, origin_kind=origin_kind) + stmt = stmt.order_by(Run.created_at.desc()).limit(limit) + if before is not None: + stmt = stmt.where(Run.created_at < before) + return list((await session.execute(stmt)).scalars()) + + +@router.get("/unlisted") +async def unlisted_runs( + workflow_id: str | None = Query(default=None), + status: str | None = Query(default=None), + run_class: str | None = None, + q: str | None = Query(default=None), + schedule_id: str | None = Query(default=None), + origin_kind: str | None = Query(default=None), + session: AsyncSession = Depends(get_session), +) -> dict[str, Any]: + """同样的筛选下,运行列表因为「系统派生的运行默认不列」而没列出的有几次:{total, by_origin: {来源: 次数}}。 + + 按来源筛了的(origin_kind 给了)什么都不藏,total 为 0。别人的个人应用刷新、个人蓝图试运行不算在内:它们不是「没列出」, + 是读不了。""" + from sqlalchemy import func + + from app.engine.origin import SYSTEM_ORIGINS + + if _csv(origin_kind): + return {"total": 0, "by_origin": {}} + stmt = _filtered(select(Run.origin_kind, func.count()), workflow_id=workflow_id, status=status, run_class=run_class, + q=q, schedule_id=schedule_id, origin_kind=",".join(SYSTEM_ORIGINS)) + rows = (await session.execute(stmt.group_by(Run.origin_kind))).all() + by_origin = {kind: int(n) for kind, n in rows if kind and n} + return {"total": sum(by_origin.values()), "by_origin": by_origin} + + +def _filtered(stmt: Any, *, workflow_id: str | None, status: str | None, run_class: str | None, q: str | None, + schedule_id: str | None, origin_kind: str | None) -> Any: + """运行列表的筛选条件(列表和「没列出几次」共用一份)。""" from sqlalchemy import not_ from app.api.deps import current_principal from app.auth.access import hidden_runs + from app.engine.origin import counted_runs - stmt = select(Run).where(not_(hidden_runs(current_principal()))).order_by(Run.created_at.desc()).limit(limit) + stmt = stmt.where(not_(hidden_runs(current_principal()))) if workflow_id == UNSAVED: stmt = stmt.where(Run.workflow_id.is_(None)) elif workflow_id: @@ -193,9 +237,10 @@ async def list_runs( if origins := _csv(origin_kind): kinds = [o for o in origins if o != NO_ORIGIN] stmt = stmt.where(or_(Run.origin_kind.in_(kinds), *([Run.origin_kind.is_(None)] if NO_ORIGIN in origins else []))) - if before is not None: - stmt = stmt.where(Run.created_at < before) - return list((await session.execute(stmt)).scalars()) + else: + # 没点名来源:只列计入统计的运行,系统派生的(评测、口径对账、召回)不列;来源为空的老运行照常列出 + stmt = stmt.where(counted_runs()) + return stmt def _csv(value: str | None) -> list[str]: @@ -289,11 +334,13 @@ async def cancel_run(run_id: str, x_actor: str | None = Header(default=None)) -> run = await session.get(Run, run_id) if run is None: raise HTTPException(404, "运行记录不存在") - raise HTTPException(409, _cannot_cancel(run.status)) + raise HTTPException(409, _cannot_cancel(run.status, run.end_reason)) return {"ok": True, "status": outcome} -def _cannot_cancel(status: str) -> str: +def _cannot_cancel(status: str, end_reason: str | None = None) -> str: + if status == "cancelled" and end_reason == "rejected": + return "这次运行已被驳回结束,没有要停止的。" return { "succeeded": "这次运行已完成,没有要停止的。", "failed": "这次运行已失败结束,没有要停止的;如需继续,请点「继续运行」。", @@ -396,7 +443,7 @@ async def continue_run( except KeyError as e: raise _missing(e) from e except ContinueRefused as e: - # 正式运行带图(formal_graph_locked)、写入结果未知还没核实(action_unresolved):带机器码 + # 正式运行带图(formal_graph_locked)、写入结果未知还没核实(action_unresolved)、被驳回了的(run_rejected):带机器码 raise CodedHTTPException(409, str(e), e.code) from e except ValueError as e: raise HTTPException(409, str(e)) from e diff --git a/backend/app/api/schedules.py b/backend/app/api/schedules.py index 35e8cca1..d7f5b0c3 100644 --- a/backend/app/api/schedules.py +++ b/backend/app/api/schedules.py @@ -491,6 +491,8 @@ async def create_schedule_row(session: AsyncSession, body: dict[str, Any], *, ac values = schedule_values(body) hook = values.pop("webhook") workflow = await _check_workflow(session, values["workflow_id"], values["run_class"], values["input_template"]) + # 目标的几列只经 scheduler/service 的一处封装写(工作流目标:target_id 同 workflow_id) + values |= service.target_columns("workflow", values["workflow_id"]) row = Schedule(**values, workflow_name=workflow.name, next_run_at=service.next_run(values["cron"], values["timezone"], values["enabled"]), created_by=actor, **_webhook_columns(hook)) @@ -528,7 +530,8 @@ async def update_schedule(schedule_id: str, request: Request, if "name" in body: values["name"] = _name(body["name"]) if "workflow_id" in body: - values["workflow_id"] = _workflow_id(body["workflow_id"]) + # 目标的几列一起改(scheduler/service.target_columns):只改 workflow_id 的话 target_id 还指着原来的工作流 + values |= service.target_columns("workflow", _workflow_id(body["workflow_id"])) if "cron" in body: values["cron"] = _cron(body["cron"]) if "timezone" in body: diff --git a/backend/app/api/settings.py b/backend/app/api/settings.py index 83fcaaf7..c45a685a 100644 --- a/backend/app/api/settings.py +++ b/backend/app/api/settings.py @@ -10,7 +10,8 @@ from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from app.api import health -from app.api.coded import CodedHTTPException +from app.api.coded import PUBLIC_URL_ENV_LOCKED, CodedHTTPException +from app.core import public_url from app.core.errors import explain, not_configured, raw from app.core.config import settings as app_settings from app.core.crypto import encrypt, mask @@ -604,35 +605,32 @@ def catalog_draft_values(stored: Any) -> dict[str, Any]: # -------------------------------------------------------------------------- -# 通知:定时任务推送 Webhook 时拼运行链接用的对外访问地址 +# 对外访问地址:通知、Webhook 里的链接,单点登录的回调,来源核对(core/public_url.py) # -------------------------------------------------------------------------- #: 设置里 notify 这一组。public_base_url 是别人打开 AgentLab 用的地址(https://agentlab.example.com),通知里的 -#: 运行链接 = 它 + /runs/{运行 id}。服务端不知道自己在反向代理后面叫什么,只能由人填;留空的通知里不附链接 -NOTIFY_GROUP = "notify" -NOTIFY_DEFAULTS: dict[str, Any] = {"public_base_url": None} -NOTIFY_SETTING_LABEL = {"public_base_url": "对外访问地址"} +#: 运行链接 = 它 + /runs/{运行 id}。服务端不知道自己在反向代理后面叫什么,只能由人填;留空的通知里不附链接。 +#: 全站只认一个对外地址(core/public_url.py):环境变量 AGENTLAB_PUBLIC_URL 设了的话这一项不生效、不能改(PUT 回 409)。 +#: 键名沿用 notify 一组,界面上在「设置 → 通用」 +NOTIFY_GROUP = public_url.SETTING_GROUP +NOTIFY_DEFAULTS: dict[str, Any] = {public_url.SETTING_KEY: None} +NOTIFY_SETTING_LABEL = {public_url.SETTING_KEY: "对外访问地址"} +#: GET /api/settings 另给的只读一组:{value: 生效的对外访问地址, env_locked: 是不是环境变量指定的}。设置页据此显示 +#: 「由环境变量指定」;PUT 带上它也不落库(同 limits) +PUBLIC_URL_GROUP = "public_url" def _base_url_problem(value: Any) -> str | None: if value is None or (isinstance(value, str) and not value.strip()): return None - label = NOTIFY_SETTING_LABEL["public_base_url"] + label = NOTIFY_SETTING_LABEL[public_url.SETTING_KEY] hint = (f"「{label}」需要是 http:// 或 https:// 开头的完整地址(例如 https://agentlab.example.com)," - f"留空表示通知里不附运行链接,当前为「{value}」") + f"留空表示通知里不附链接,当前为「{value}」") if not isinstance(value, str): return hint - from urllib.parse import urlsplit - - try: - parts = urlsplit(value.strip()) - except ValueError: - return hint - if parts.scheme not in ("http", "https") or not parts.hostname or " " in value.strip(): - return hint - if parts.query or parts.fragment: - return f"「{label}」不能带 ? 或 # 之后的部分,运行链接会接在地址后面,当前为「{value}」" - return None + if (problem := public_url.url_problem(value)) == "suffix": + return f"「{label}」不能带 ? 或 # 之后的部分,链接会接在地址后面,当前为「{value}」" + return hint if problem else None def notify_problem(value: Any) -> str | None: @@ -651,18 +649,23 @@ def notify_problem(value: Any) -> str | None: def notify_values(stored: Any) -> dict[str, Any]: """库里存的(或刚写进来的)notify 一组 → 补齐默认、规范化:去掉首尾空白和末尾的 /,空串当没填;存坏了的按没填。""" stored = stored if isinstance(stored, dict) else {} - url = stored.get("public_base_url") - if not isinstance(url, str) or not url.strip() or _base_url_problem(url): - return {"public_base_url": None} - return {"public_base_url": url.strip().rstrip("/")} + return {public_url.SETTING_KEY: public_url.normalize(stored.get(public_url.SETTING_KEY))} async def notify_settings(session: AsyncSession) -> dict[str, Any]: - """设置里的 notify 一组,没存过的补默认。定时任务推送 Webhook 时从这里读。""" + """设置里存的 notify 一组,没存过的补默认。要拼链接的地方不读它:一律经 core/public_url(环境变量优先)。""" row = await session.get(Setting, NOTIFY_GROUP) return notify_values(row.value if row else None) +def public_url_values(stored_notify: Any) -> dict[str, Any]: + """GET /api/settings 里只读的 public_url 一组:生效的那个地址(环境变量优先,其次库里存的这一项),是不是环境变量 + 指定的。按库里存的算,不读进程缓存:设置页看到的就是落库的样子。""" + locked = public_url.env_locked() + value = public_url.env_url() if locked else notify_values(stored_notify)[public_url.SETTING_KEY] + return {"value": value, "env_locked": locked} + + # -------------------------------------------------------------------------- # 应用:按需视图的每日上限(D-j)、使用者能不能建个人应用(docs/design/normal-user-apps.md N7) # -------------------------------------------------------------------------- @@ -754,7 +757,7 @@ DEFAULT_SETTINGS: dict[str, Any] = { JUDGE_GROUP: dict(JUDGE_DEFAULTS), # 核查(双盲复算、反方审稿、审批质询):核查模型、三项的开关和上限、每日金额上限。和裁判一组一样由用户定 VERIFY_GROUP: verify_values(None), - # 通知:定时任务的 Webhook 里拼运行链接用的对外访问地址 + # 对外访问地址(存在 notify 一组):通知、Webhook 里的链接,单点登录的回调,来源核对。环境变量优先(core/public_url.py) NOTIFY_GROUP: notify_values(None), # 起草模型:数据目录起草、按配方导入的 AI 起草用的模型,留空沿用助手的模型 CATALOG_DRAFT_GROUP: catalog_draft_values(None), @@ -808,6 +811,7 @@ async def get_settings(session: AsyncSession = Depends(get_session)) -> dict[str # limits 是服务端配置(环境变量),只读:库里可能存着旧版设置页写进去的一份 #(见过 max_agent_steps: 25),它从来不生效,却会盖住真实的上限,界面照着错的数许诺 stored.pop("limits", None) + stored.pop(PUBLIC_URL_GROUP, None) # 裁判、核查的每日计数是引擎记的账,不是用户设置:设置页整组回写时带着一份旧的,会把今天的花费清零 stored.pop(JUDGE_SPEND, None) stored.pop(VERIFY_SPEND, None) @@ -818,6 +822,7 @@ async def get_settings(session: AsyncSession = Depends(get_session)) -> dict[str merged[JUDGE_GROUP] = judge_values(stored.get(JUDGE_GROUP)) merged[VERIFY_GROUP] = verify_values(stored.get(VERIFY_GROUP)) merged[NOTIFY_GROUP] = notify_values(stored.get(NOTIFY_GROUP)) + merged[PUBLIC_URL_GROUP] = public_url_values(stored.get(NOTIFY_GROUP)) merged[CATALOG_DRAFT_GROUP] = catalog_draft_values(stored.get(CATALOG_DRAFT_GROUP)) merged[APPS_GROUP] = apps_values(stored.get(APPS_GROUP)) for key, value in stored.items(): @@ -910,6 +915,11 @@ async def put_settings( raise HTTPException(422, problem) # 存规范化之后的:末尾带 / 的地址拼链接会多出一个斜杠 payload.values[NOTIFY_GROUP] = notify_values(payload.values[NOTIFY_GROUP]) + if public_url.env_locked() and payload.values[NOTIFY_GROUP] != await notify_settings(session): + # 环境变量指定了对外访问地址:改这一项不会生效,照实拒绝,不悄悄存一个用不上的值。原样写回(整组回写)不算改 + raise CodedHTTPException( + 409, f"对外访问地址由环境变量 {public_url.ENV_NAME} 指定({public_url.env_url()}),不能在设置里修改。" + "要改请改部署配置后重启服务", PUBLIC_URL_ENV_LOCKED, field=f"{NOTIFY_GROUP}.{public_url.SETTING_KEY}") if APPS_GROUP in payload.values: if problem := apps_problem(payload.values[APPS_GROUP]): raise HTTPException(422, problem) @@ -917,9 +927,9 @@ async def put_settings( if found := await _provider_problem(session, payload.values): raise CodedHTTPException(422, found[1], PROVIDER_INVALID, field=found[0]) for key, value in payload.values.items(): - if key in (TRUST_KEY, "limits", JUDGE_SPEND, VERIFY_SPEND, *_hidden_groups()): + if key in (TRUST_KEY, "limits", PUBLIC_URL_GROUP, JUDGE_SPEND, VERIFY_SPEND, *_hidden_groups()): # 信任三档只走 PUT /api/tools/trust:设置页整组回写时带着一份旧的, - # 会把刚在工具页改的档位冲掉。limits 是环境变量定的,写进库也不生效。 + # 会把刚在工具页改的档位冲掉。limits 是环境变量定的、public_url 是算出来的,写进库也不生效。 # 裁判、核查的每日计数是引擎记的账,只有引擎自己写。登录设置只经 /api/auth/settings 改(_hidden_groups) continue row = await session.get(Setting, key) @@ -928,6 +938,9 @@ async def put_settings( else: session.add(Setting(key=key, value=value)) await session.commit() + if NOTIFY_GROUP in payload.values: + # 中间件、单点登录的回调按进程缓存取对外访问地址:写完库就刷新,不用重启 + public_url.remember(payload.values[NOTIFY_GROUP][public_url.SETTING_KEY]) return await get_settings(session) diff --git a/backend/app/api/source_versions.py b/backend/app/api/source_versions.py index 7d8a07ba..bd77947f 100644 --- a/backend/app/api/source_versions.py +++ b/backend/app/api/source_versions.py @@ -21,7 +21,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.api.coded import CodedHTTPException from app.api.datasources import _out -from app.api.deps import signer_of +from app.api.deps import signer_of, signer_verified from app.data import source_versions, table_versions from app.data.recipe_imports import ImportRefused from app.db.base import get_session @@ -170,7 +170,8 @@ async def remove_period(source_id: str, import_id: str, request: Request, x_acto source = await _source(session, source_id) try: out = await source_versions.remove_period(session, source, import_id, expected_current=expected, - reason=reason or "", signed_by=_signed(body, x_actor)) + reason=reason or "", signed_by=_signed(body, x_actor), + signer_verified=signer_verified()) except _REFUSALS as e: raise _refused(e) from e return {"source": await _source_out(session, source_id), **out} @@ -193,7 +194,7 @@ async def revoke_acceptance(source_id: str, import_id: str, request: Request, try: out = await source_versions.revoke_acceptance( session, source, import_id, expected_current=expected, expected_target=target, ack_mask_lost=ack, - reason=reason or "", signed_by=_signed(body, x_actor)) + reason=reason or "", signed_by=_signed(body, x_actor), signer_verified=signer_verified()) except _REFUSALS as e: raise _refused(e) from e return {"source": await _source_out(session, source_id), **out} diff --git a/backend/app/api/table_imports.py b/backend/app/api/table_imports.py index 24e13370..1fdf408e 100644 --- a/backend/app/api/table_imports.py +++ b/backend/app/api/table_imports.py @@ -28,7 +28,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.api.coded import CodedHTTPException from app.api.datasources import _NAME_PATTERN, _out, read_upload -from app.api.deps import current_principal, signer_of +from app.api.deps import current_principal, signer_of, signer_verified from app.data import recipe_imports, table_versions from app.data.recipe_imports import EditRequest, ImportRefused from app.data.recipe_types import Acceptance, AiAvailability, PeriodInput, Recipe, Selection @@ -422,7 +422,8 @@ async def commit(staging_id: str, request: Request, x_actor: str | None = Header try: result = await recipe_imports.commit_staging( session, staging, trial_id=trial_id if isinstance(trial_id, str) else "", - confirmations=confirmations, acceptances=_acceptances(body.get("acceptances"), who), signed_by=who) + confirmations=confirmations, acceptances=_acceptances(body.get("acceptances"), who), signed_by=who, + signer_verified=signer_verified()) except _REFUSALS as e: raise _refused(e) from e source = await session.get(DataSource, result.source_id, populate_existing=True) diff --git a/backend/app/api/tasks.py b/backend/app/api/tasks.py index 8418558d..8089e350 100644 --- a/backend/app/api/tasks.py +++ b/backend/app/api/tasks.py @@ -1,17 +1,20 @@ """待办(BE-2):列表、计数、详情、回复、改派、催办。设计文档 §3.7、§3.10。 身份经 api/deps.current_principal 取;关登录时它没有名字,用自报的署名(X-Actor)补上,回复人一栏才写得出人。 -回复被拒回 403 / 404 / 409 / 422,{detail, code};表单不通过时另带 errors(同应用入口的 422)。 +回复被拒回 403 / 404 / 409 / 422,{detail, code},一律带 code(外部系统按它认);表单不通过时另带 errors(同应用入口的 +422)。服务层没带码的拒绝按状态补上同一类的码(api/coded.py 的 TASK_*)。 """ from __future__ import annotations from typing import Any, Literal -from fastapi import APIRouter, Header, HTTPException, Query +from fastapi import APIRouter, Header, Query from fastapi.responses import JSONResponse from pydantic import BaseModel, Field -from app.api.coded import CodedHTTPException +from app.api.coded import ( + ROLE_REQUIRED, TASK_CLOSED, TASK_FORBIDDEN, TASK_INVALID, TASK_NOT_FOUND, CodedHTTPException, +) from app.api.deps import current_principal, decode_actor from app.process import tasks as task_service from app.process.tasks import FormInvalid, TaskRefused @@ -23,12 +26,14 @@ def principal_of(x_actor: str | None) -> Any: return task_service.principal_with_actor(current_principal(), decode_actor(x_actor)) -def refused(e: TaskRefused) -> HTTPException | JSONResponse: +#: 服务层(process/tasks.py、process/actions.py、blueprints/personal.py)没带码的拒绝按状态归到哪一类 +_CODE_BY_STATUS = {404: TASK_NOT_FOUND, 403: TASK_FORBIDDEN, 409: TASK_CLOSED, 422: TASK_INVALID} + + +def refused(e: TaskRefused) -> CodedHTTPException | JSONResponse: if isinstance(e, FormInvalid): return JSONResponse({"detail": str(e), "code": e.code, "errors": e.errors}, status_code=422) - if e.code: - return CodedHTTPException(e.status_code, str(e), e.code) - return HTTPException(e.status_code, str(e)) + return CodedHTTPException(e.status_code, str(e), e.code or _CODE_BY_STATUS.get(e.status_code, TASK_CLOSED)) def _raise(e: TaskRefused) -> Any: @@ -53,7 +58,7 @@ async def list_tasks( ) -> dict[str, Any]: principal = principal_of(x_actor) if assigned == "all" and principal.mode != "off" and not principal.at_least("builder"): - raise HTTPException(403, "只能查看派给自己的待办") + raise CodedHTTPException(403, "只能查看派给自己的待办", ROLE_REQUIRED) statuses = None if status == "all" else _csv(status) or ["pending"] items = await task_service.list_tasks(principal, assigned=assigned, statuses=statuses, app_id=app_id, run_id=run_id, limit=limit) @@ -71,7 +76,7 @@ async def get_task(task_id: str, x_actor: str | None = Header(default=None)) -> principal = principal_of(x_actor) out = await task_service.detail(task_id, principal) if out is None or not task_service.visible(out, principal): - raise HTTPException(404, "待办不存在,所属的运行可能已被删除") + raise CodedHTTPException(404, "待办不存在,所属的运行可能已被删除", TASK_NOT_FOUND) return out diff --git a/backend/app/apps/derive.py b/backend/app/apps/derive.py index 069b9672..7fbb84d4 100644 --- a/backend/app/apps/derive.py +++ b/backend/app/apps/derive.py @@ -24,13 +24,19 @@ VIEW = "latest" INSTANCE_VIEW = "inst" ENTRY = "start" RECONCILER = "reconciler" +#: 派生不了时的机器码:工作流、版本不存在(404) +WORKFLOW_GONE = "workflow_not_found" +VERSION_GONE = "workflow_version_not_found" class DeriveRefused(Exception): - def __init__(self, status_code: int, message: str) -> None: + """派生不了。code 是给程序认的机器码,接口照它回。""" + + def __init__(self, status_code: int, message: str, code: str) -> None: super().__init__(message) self.status_code = status_code self.message = message + self.code = code def _safe(text: str) -> str: @@ -45,10 +51,10 @@ async def derive_app(session: AsyncSession, workflow_id: str, version: int, *, n workflow = await session.get(Workflow, workflow_id) if workflow is None: - raise DeriveRefused(404, "工作流不存在,可能已被删除") + raise DeriveRefused(404, "工作流不存在,可能已被删除", WORKFLOW_GONE) row = await version_row(session, workflow_id, version) if row is None: - raise DeriveRefused(404, f"「{workflow.name}」不存在 v{version} 版本") + raise DeriveRefused(404, f"「{workflow.name}」不存在 v{version} 版本", VERSION_GONE) state = PinState(alias=PIN, pin={"workflow_id": workflow_id, "version": version}, workflow=workflow, row=row) schedule = (await session.execute(select(Schedule).where( Schedule.workflow_id == workflow_id, Schedule.run_class == "formal", Schedule.enabled.is_(True)) diff --git a/backend/app/apps/entries.py b/backend/app/apps/entries.py index 56bd7c69..76b6980a 100644 --- a/backend/app/apps/entries.py +++ b/backend/app/apps/entries.py @@ -113,6 +113,13 @@ async def start_entry(app_id: str, entry_id: str, values: dict[str, Any], princi form = resolve_form(spec, entry.get("form")) or {"type": "object", "properties": {}} if errors := validate_form(form, values if isinstance(values, dict) else None): raise FormRejected(errors) + # 选人控件:开了登录时按候选池核对(auth/directory.py,05-已知遗留 (b))。以前原样传下去,填错的名字会把待办 + # 派给一个不存在的人——没人能处理,也不报错 + from app.auth.directory import principal_errors + + if errors := await principal_errors(session, app, version.bindings, form, values, + getattr(principal, "mode", "off")): + raise FormRejected(errors) # 表单里的文件(docs/design/normal-user-apps.md N9):核对是提交人在这个入口传的、还没用过,先占住;表单值换成 # 登记的那一份引用(连同 sha256 进运行输入、随封存固定)。发起成功绑到这次运行,发起不成放回去 values, claimed = await _claim_files(form, values, principal, app.id, entry_id) diff --git a/backend/app/apps/forms.py b/backend/app/apps/forms.py index fc4f5ff1..d0f63b67 100644 --- a/backend/app/apps/forms.py +++ b/backend/app/apps/forms.py @@ -28,6 +28,8 @@ _FIELD_KEYS = frozenset({ "minLength", "maxLength", "maxItems", "format", "items", "x-widget", # 文件控件的限制:accept(类型或扩展名的列表)、max_mb(docs/design/normal-user-apps.md §3.8) "x-file", + # 选人控件的候选池:{pool: app | group:<组> | slot:<岗位>}(auth/directory.py,05-已知遗留 (b)) + "x-principal", }) #: 文件控件只认这几项写法 _FILE_FIELD_KEYS = frozenset({"type", "title", "description", "readOnly", "x-widget", "x-file"}) @@ -88,6 +90,13 @@ def _field_problems(field: Any) -> list[str]: return out + _file_field_problems(field) if "x-file" in field: out.append("不是文件控件,不能写文件的限制") + if "x-principal" in field: + from app.auth.directory import pool_problem + + if field.get("x-widget") != "principal": + out.append("不是选人控件,不能写候选范围") + elif problem := pool_problem(field["x-principal"]): + out.append(problem) if kind not in FORM_TYPES: out.append(f"的类型只能是 {' / '.join(FORM_TYPES)}") if "format" in field and field["format"] not in FORM_FORMATS: @@ -153,7 +162,7 @@ def _clean_schema(form: dict[str, Any]) -> dict[str, Any]: continue props[name] = {k: v for k, v in field.items() if k in _FIELD_KEYS and k not in ("format", "x-widget", "x-enum-labels", "title", - "description", "readOnly", "default")} + "description", "readOnly", "default", "x-principal")} return {"type": "object", "properties": props, "required": [r for r in form.get("required") or [] if r in props]} diff --git a/backend/app/apps/resolve.py b/backend/app/apps/resolve.py index 97564ea6..0a23a758 100644 --- a/backend/app/apps/resolve.py +++ b/backend/app/apps/resolve.py @@ -51,6 +51,8 @@ NEEDS_UPGRADE = "app_needs_upgrade" STATUS_TEXT = {"queued": "排队中", "running": "运行中", "interrupted": "等待处理", "succeeded": "已完成", "failed": "失败", "cancelled": "已取消"} +#: 被驳回、又说不出是哪一步时(没有节点可指)的说法 +REJECTED_TEXT = "被驳回" TIER_TEXT = {"formal": "完整出具", "degraded": "降档出具", "withheld": "不予出具"} NO_TIER = "未声明出具契约" #: 蓝图试运行(docs/design/normal-user-apps.md N4):印章的来源、说明,趋势只有一个点时的说明,预览里入口按钮为什么点不了 @@ -1277,8 +1279,9 @@ async def _trails(session: AsyncSession, runs: list[Run]) -> dict[str, _Trail]: for row in (await session.execute(select(ActionRecord).where(ActionRecord.run_id.in_(ids)) .order_by(ActionRecord.created_at, ActionRecord.attempt))).scalars(): out[row.run_id].actions.append(row) - # 节点事件只有结束了的运行要看(走到了哪个成果节点、停在哪个节点上) - wanted = [r.id for r in runs if r.status in ("succeeded", "failed")] + # 节点事件只有结束了的运行要看(走到了哪个成果节点、停在哪个节点上)。取消的只有被驳回的要看(停在哪一步) + wanted = [r.id for r in runs if r.status in ("succeeded", "failed") + or (r.status == "cancelled" and r.end_reason == "rejected")] if wanted: for run_id, etype, node_id in (await session.execute( select(RunEvent.run_id, RunEvent.type, RunEvent.node_id).where( @@ -1325,6 +1328,26 @@ def rejected_outputs(graph: Any) -> dict[str, set[str]]: return out +def _rejection_text(trail: _Trail, node_id: str, title: str) -> str | None: + """停在被人驳回的待办上时怎么说:动作确认写「「付款」未确认(被驳回)」,别的写「「x」被驳回」;不是驳回的为 None。""" + kind, decision = _decision(trail, node_id) + if kind == "action_confirm" and decision == "reject": + return f"「{title}」未确认(被驳回)" + if decision in _REJECTIONS: + return f"「{title}」被驳回" + return None + + +async def legacy_rejected(session: AsyncSession, run: Run) -> bool: + """加 end_reason 之前被人驳回的运行记的是 failed(封存不改写):失败的那一步上最近一条处理完的待办是驳回。和实例页 + 认存量驳回(instance_state 的 failed 分支)是同一份判断;runner 据此不许它接着跑。""" + if run.status != "failed": + return False + trail = (await _trails(session, [run]))[run.id] + failed = [nid for etype, nid in trail.events if etype == "node.failed"] + return bool(failed) and _decision(trail, failed[-1])[1] in _REJECTIONS + + def instance_state(run: Run, trail: _Trail) -> dict[str, str]: """实例给人看的状态 {code, text}:code 是界面画状态标记用的状态(运行的那几种,加 held),text 说这件事办到哪了、 结论是什么,不只是运行状态(P1)。 @@ -1334,10 +1357,12 @@ def instance_state(run: Run, trail: _Trail) -> dict[str, str]: 否定的那种结局——这次运行里人工节点驳回了、从它的「驳回」出口一路走到的成果节点(rejected_outputs)——code 记 cancelled(同「未确认(被驳回)」,是人的决定,不画成成功),文字照旧写成果节点的标题; - 停着:有已发出、结果未知的动作,写「等待核实写入结果」;否则按在等的待办说(等待审批、等待确认发送……); - - 失败:停在一个被人驳回的待办上的——动作确认被驳回(「付款」未确认(被驳回))、审批设了驳回即终止、补充输入被 - 驳回——是人的决定,不是故障,code 记 cancelled;核实为未生效、选了放弃的同理。外部系统拒绝之类照写失败。 + - 被驳回(cancelled + end_reason=rejected,engine/context.NodeRejected):动作确认被驳回写「「付款」未确认(被驳回)」, + 审批设了驳回即终止、补充输入被驳回等写「「x」被驳回」,code 记 cancelled;手动停止的写「已取消」; + - 失败:加 end_reason 之前被人驳回的运行记的是 failed(封存不改写),停在被驳回的待办上的照上一条说,和新形状说法 + 一样;核实为未生效、选了放弃的同理,code 记 cancelled。外部系统拒绝之类照写失败。 - 运行本身的状态不改(驳回动作确认在引擎里就是节点失败、运行结束):这里只是把它说清楚。""" + 通知(notify/service.notify_run)的文字也取自这里:实例页和推送说同一句话。""" from app.apps.tree import node_map nodes = node_map(run.graph) @@ -1360,15 +1385,18 @@ def instance_state(run: Run, trail: _Trail) -> dict[str, str]: negative = any(_decision(trail, gate)[1] in _REJECTIONS for gate in gates) return {"code": "cancelled" if negative else "succeeded", "text": title(reached[0])} return {"code": "succeeded", "text": STATUS_TEXT["succeeded"]} + if run.status == "cancelled" and run.end_reason == "rejected": + failed = [nid for etype, nid in trail.events if etype == "node.failed"] + node_id = run.error_node_id or (failed[-1] if failed else None) + said = _rejection_text(trail, node_id, title(node_id)) if node_id else None + # 待办没留下(老的审批、没开出待办)时按驳回的那一步说,不退回「已取消」:这是人的决定,不是有人停了它 + return {"code": "cancelled", "text": said or (f"「{title(node_id)}」被驳回" if node_id else REJECTED_TEXT)} if run.status == "failed": failed = [nid for etype, nid in trail.events if etype == "node.failed"] if failed: node_id = failed[-1] - kind, decision = _decision(trail, node_id) - if kind == "action_confirm" and decision == "reject": - return {"code": "cancelled", "text": f"「{title(node_id)}」未确认(被驳回)"} - if decision in _REJECTIONS: - return {"code": "cancelled", "text": f"「{title(node_id)}」被驳回"} + if said := _rejection_text(trail, node_id, title(node_id)): + return {"code": "cancelled", "text": said} record = next((a for a in reversed(trail.actions) if a.node_id == node_id), None) if record is not None and record.state == "failed": if (record.resolution or {}).get("outcome") == "abandon": diff --git a/backend/app/apps/spec.py b/backend/app/apps/spec.py index a64b7df0..e8f3edfd 100644 --- a/backend/app/apps/spec.py +++ b/backend/app/apps/spec.py @@ -47,9 +47,10 @@ FILE_FIELD_TYPE = "object" FILE_REF_KEYS = ("file_id", "name", "size", "sha256", "type") #: 表单字段能用的 format。不支持 pattern:JS 和 Python 的正则语义不同,会出现前端放行、后端拒收 FORM_FORMATS = ("date", "date-time", "email", "uri") -#: 表单校验错误的 code。file:文件控件的值不对(没上传、不是本人传的、已经用过、类型或大小不符,app/files/store.py) +#: 表单校验错误的 code。file:文件控件的值不对(没上传、不是本人传的、已经用过、类型或大小不符,app/files/store.py); +#: principal:开了登录时选人控件的值不在候选池里(auth/directory.principal_errors) FORM_ERROR_CODES = ("required", "type", "enum", "minimum", "maximum", "min_length", "max_length", "max_items", "format", - "file") + "file", "principal") class _Strict(BaseModel): diff --git a/backend/app/auth/__init__.py b/backend/app/auth/__init__.py index def0941d..361b5153 100644 --- a/backend/app/auth/__init__.py +++ b/backend/app/auth/__init__.py @@ -15,7 +15,9 @@ import logging import os from typing import Any -from fastapi import Depends, HTTPException +from fastapi import Depends + +from app.api.coded import AUTH_REQUIRED, ROLE_REQUIRED, CodedHTTPException logger = logging.getLogger(__name__) @@ -77,17 +79,24 @@ def remember_settings(value: dict[str, Any]) -> None: _loaded = True +def refusal(principal: Any, role: str) -> CodedHTTPException | None: + """principal 够不够 role:够(或关登录)是 None;不够交回该回的错——没登录 401(auth_required),角色不够 403 + (role_required)。路由级依赖、接口内部的 ensure_role、中间件拦 /docs 都用它,同一个判断、同一句话。""" + if principal.mode == "off": + return None + if not principal.username: + return CodedHTTPException(401, "请先登录", AUTH_REQUIRED) + if not principal.at_least(role): + return CodedHTTPException(403, f"没有权限:这项操作需要{ROLE_LABEL.get(role, role)}及以上的角色", ROLE_REQUIRED) + return None + + def require(role: str) -> None: - """当前请求至少是 role,否则 401(没登录)/ 403(角色不够)。关登录时放行。""" + """当前请求至少是 role,否则 401(没登录)/ 403(角色不够),都带 code。关登录时放行。""" from app.api.deps import current_principal - principal = current_principal() - if principal.mode == "off": - return - if not principal.username: - raise HTTPException(401, "请先登录") - if not principal.at_least(role): - raise HTTPException(403, f"没有权限:这项操作需要{ROLE_LABEL.get(role, role)}及以上的角色") + if (refused := refusal(current_principal(), role)) is not None: + raise refused def role_dependency(role: str) -> Any: diff --git a/backend/app/auth/directory.py b/backend/app/auth/directory.py index 6f689ec1..fa88a6b7 100644 --- a/backend/app/auth/directory.py +++ b/backend/app/auth/directory.py @@ -1,6 +1,10 @@ """人和组的名录:选人控件的候选(GET /api/directory)、应用检查核对岗位绑定和授权、改授权时核对填的名字,同一处取、 同一个口径(停用的人也在名录里,标 disabled:已经绑着他的岗位、授权要能认出「这个人停用了」,不是「没有这个人」)。 +使用者选人(表单里的选人控件、个人应用提交审核)不拿整份名录,拿按用途算的候选池(candidates,GET +/api/directory/candidates):只有这件事能选的人,只给用户名和显示名,停用的不列;开了登录时提交的值按同一个池核对 +(docs/design/next-round/05-已知遗留.md (b))。名录整份仍只给搭建者。 + 名录里对不上怎么算看登录方式(同前端 PrincipalPicker): - 账号密码登录:人和组都在本系统里建,名录里没有就是填错了(或人被删了),停用的人处理不了待办——都算错; - 单点登录:人和组第一次登录后才进名录,名录里没有的照实提醒、不拒收;停用的人名录里认得出,照样算错; @@ -10,7 +14,7 @@ from __future__ import annotations from collections.abc import Iterable from dataclasses import dataclass, field -from typing import Literal +from typing import Any, Literal from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession @@ -95,3 +99,175 @@ def mismatch(directory: Directory, mode: str, kind: str, name: str) -> Mismatch return Mismatch(kind, name, problem, False, f"{what}名录里还没有:单点登录的人和组第一次登录后才进名录,名字要和身份提供方里的一致才生效") return Mismatch(kind, name, problem, True, f"{what}名录里没有") + + +# -------------------------------------------------------------------------- +# 候选池:使用者选人时能选谁(05-已知遗留 (b)) +# -------------------------------------------------------------------------- + +#: 候选池的用途:form 表单里的选人控件(入口表单、待办表单);review 个人应用提交审核选审核的人 +PURPOSES = ("form", "review") +#: 一次最多给几位候选 +CANDIDATES_MAX = 50 +#: 表单字段声明候选池(x-principal.pool):app 能用这个应用的人(缺省);group:<组> 这个组的成员;slot:<岗位> 这个岗位 +#: 绑定的人和组(按应用版本的 bindings) +POOL_APP = "app" +POOL_GROUP = "group:" +POOL_SLOT = "slot:" + + +def pool_problem(value: Any) -> str | None: + """x-principal 的写法对不对:只能是 {pool: app | group:<组> | slot:<岗位>}。没问题为 None。""" + if not isinstance(value, dict) or set(value) != {"pool"} or not isinstance(value.get("pool"), str): + return "的候选范围只能写成 {\"pool\": \"app\"}、{\"pool\": \"group:<组名>\"} 或 {\"pool\": \"slot:<岗位>\"}" + pool = value["pool"].strip() + if pool == POOL_APP: + return None + for prefix in (POOL_GROUP, POOL_SLOT): + if pool.startswith(prefix) and pool[len(prefix):].strip(): + return None + return "的候选范围只能是 app(能用这个应用的人)、group:<组名> 或 slot:<岗位>" + + +def pool_of(field: Any) -> str: + """表单字段声明的候选池;没声明、写法不对的按缺省(能用这个应用的人)。""" + setting = field.get("x-principal") if isinstance(field, dict) else None + if isinstance(setting, dict) and pool_problem(setting) is None: + return setting["pool"].strip() + return POOL_APP + + +@dataclass +class Pool: + """一个候选池。open:池就是全部启用的人(应用授权给了所有人)——不带前缀不列,免得等于发通讯录。""" + users: dict[str, DirectoryUser] = field(default_factory=dict) + groups: dict[str, DirectoryGroup] = field(default_factory=dict) + open: bool = False + + def has(self, name: str) -> bool: + """提交的这个名字在不在池里(用户名或组名)。""" + return name in self.users or name in self.groups + + def pick(self, q: str | None, limit: int = CANDIDATES_MAX) -> dict[str, Any]: + """接口给的那一份:{users: [{username, display_name}], groups: [{name, label}], truncated};按前缀筛 + (用户名或显示名,不分大小写)。池是 open 的、又没带前缀:什么都不给,needs_query 为真。""" + prefix = (q or "").strip().lower() + if self.open and not prefix: + return {"users": [], "groups": [], "truncated": False, "needs_query": True} + limit = max(1, min(int(limit or CANDIDATES_MAX), CANDIDATES_MAX)) + + def hit(*names: str) -> bool: + return not prefix or any(n.lower().startswith(prefix) for n in names if n) + + users = [u for u in sorted(self.users.values(), key=lambda u: u.username) if hit(u.username, u.display_name)] + groups = [g for g in sorted(self.groups.values(), key=lambda g: g.name) if hit(g.name, g.label)] + return {"users": [{"username": u.username, "display_name": u.display_name} for u in users[:limit]], + "groups": [{"name": g.name, "label": g.label} for g in groups[:limit]], + "truncated": len(users) > limit or len(groups) > limit} + + +async def _members(session: AsyncSession, group: str) -> dict[str, DirectoryUser]: + from app.db.models import Group, GroupMember, User + + rows = (await session.execute( + select(User).join(GroupMember, GroupMember.user_id == User.id).join(Group, Group.id == GroupMember.group_id) + .where(Group.name == group, User.disabled.is_(False)))).scalars() + return {u.username: DirectoryUser(u.username, u.display_name or u.username, False) for u in rows} + + +async def _people(session: AsyncSession, items: Iterable[tuple[str, str]]) -> Pool: + """一组(种类, 名字)展开成池:人按名录认(停用的、名录里没有的不列),组列出组本身、再展开成员。""" + from app.db.models import Group, User + + pool = Pool() + items = list(items) + names = sorted({n for k, n in items if k == "user" and n}) + if names: + for u in (await session.execute(select(User).where(User.username.in_(names), User.disabled.is_(False))))\ + .scalars(): + pool.users[u.username] = DirectoryUser(u.username, u.display_name or u.username, False) + groups = sorted({n for k, n in items if k == "group" and n}) + if groups: + for g in (await session.execute(select(Group).where(Group.name.in_(groups)))).scalars(): + pool.groups[g.name] = DirectoryGroup(g.name, g.label or g.name) + pool.users.update(await _members(session, g.name)) + return pool + + +async def _everyone(session: AsyncSession) -> Pool: + from app.db.models import Group, User + + pool = Pool(open=True) + for u in (await session.execute(select(User).where(User.disabled.is_(False)))).scalars(): + pool.users[u.username] = DirectoryUser(u.username, u.display_name or u.username, False) + for g in (await session.execute(select(Group))).scalars(): + pool.groups[g.name] = DirectoryGroup(g.name, g.label or g.name) + return pool + + +async def app_pool(session: AsyncSession, app: Any) -> Pool: + """能用这个应用的人:使用及以上的授权(人、组展开成员),加上管理它的人(管理员、建它的人)。授权给了所有人的,池就是 + 全部启用的人(open)。按 apps/access 的口径:搭建者没有授权时只能查看全部实例,不算「能用」的候选。""" + from app.apps.access import RANK, grants_of + from app.db.models import User + + grants = [g for g in await grants_of(session, app.id) if RANK.get(g.role, 0) >= RANK["use"]] + if any(g.principal_kind == "everyone" for g in grants): + return await _everyone(session) + admins = (await session.execute(select(User.username).where(User.role == "admin"))).scalars() + items = [(g.principal_kind, g.principal) for g in grants] + [("user", a) for a in admins] + if getattr(app, "created_by", None): + items.append(("user", app.created_by)) + return await _people(session, items) + + +async def field_pool(session: AsyncSession, app: Any, bindings: dict[str, Any] | None, field: Any) -> Pool: + """表单里一个选人控件的候选池(按字段的 x-principal,缺省能用这个应用的人)。bindings 是这次用到的应用版本的绑定。""" + pool = pool_of(field) + if pool.startswith(POOL_GROUP): + name = pool[len(POOL_GROUP):].strip() + return Pool(users=await _members(session, name)) + if pool.startswith(POOL_SLOT): + bound = ((bindings or {}).get("slots") or {}).get(pool[len(POOL_SLOT):].strip()) + items = [(str(b.get("kind") or "user"), str(b.get("name") or "")) for b in bound or [] if isinstance(b, dict)] + if any(k == "everyone" for k, _ in items): + return await _everyone(session) + return await _people(session, items) + return await app_pool(session, app) + + +async def review_pool(session: AsyncSession) -> Pool: + """个人应用提交审核能交给谁:启用的搭建者和管理员(同 blueprints/personal._review_targets 的规则)。""" + from app.db.models import User + + rows = (await session.execute(select(User).where(User.role.in_(("builder", "admin")), + User.disabled.is_(False)))).scalars() + return Pool(users={u.username: DirectoryUser(u.username, u.display_name or u.username, False) for u in rows}) + + +def principal_fields(form: Any) -> dict[str, dict[str, Any]]: + """表单里的选人控件:{字段名: 字段}。""" + props = form.get("properties") if isinstance(form, dict) else None + return {name: f for name, f in (props or {}).items() if isinstance(f, dict) and f.get("x-widget") == "principal"} + + +#: 选人控件的值不在候选池里时表单错误的 code(apps/spec.FORM_ERROR_CODES 的 principal) +PRINCIPAL_ERROR_CODE = "principal" + + +async def principal_errors(session: AsyncSession, app: Any, bindings: dict[str, Any] | None, form: Any, + values: Any, mode: str) -> list[dict[str, str]]: + """开了登录时按候选池核对表单里选人控件的值:不在池里的按表单错误交回(code=principal),话写成「「x」不在可选的人里」。 + 关登录不核对:没有用户表,签批记的是各自填的署名。没填的不在这里报(必填由表单校验报)。""" + if mode == "off" or not isinstance(values, dict): + return [] + out: list[dict[str, str]] = [] + for name, f in principal_fields(form).items(): + value = values.get(name) + if value in (None, "") or not isinstance(value, str): + continue + pool = await field_pool(session, app, bindings, f) + if not pool.has(value.strip()): + hint = ":单点登录的人要先登录一次才进名录" if mode == "oidc" else ",请从候选中选择" + out.append({"path": name, "code": PRINCIPAL_ERROR_CODE, "message": f"「{value.strip()}」不在可选的人里{hint}"}) + return out diff --git a/backend/app/auth/middleware.py b/backend/app/auth/middleware.py index 05da0def..3a9cfaa8 100644 --- a/backend/app/auth/middleware.py +++ b/backend/app/auth/middleware.py @@ -9,6 +9,8 @@ - 会话 cookie → Principal(认证过的用户名、角色、组);没有会话是匿名(username=None),X-Actor 一律不看; - /api 下除了登录本身、健康检查、入站钩子,匿名一律 401。各路由的角色要求另由 main.py 的路由级依赖管,这里兜底: 哪个路由忘了挂依赖,也不会对没登录的人敞开; +- 接口说明(FastAPI 自带的 /docs、/redoc、/openapi.json)和 /api 一样拦:匿名 401,不到搭建者 403。它们是 FastAPI 自己 + 挂的路由,挂不上路由级依赖,所以角色也在这里判。以前只拦 /api/ 开头的,开了登录接口全貌也对匿名的人敞开; - 非 GET 的请求必须带 X-AgentLab-Request 头,带了 Origin 的必须是本站或登记的前端地址(CSRF)。自定义请求头跨站发不出来 (要过 CORS 预检,只放行配置里的前端地址),SameSite=Lax 的 cookie 也不随跨站的表单提交带过去; - WebSocket:没登录的握手前就关掉;登录了的交给路由,按运行判(api/deps.py 的 run_scope:user 角色只能连自己能读、 @@ -19,8 +21,8 @@ from __future__ import annotations import json from http.cookies import SimpleCookie from typing import Any -from urllib.parse import urlsplit +from app.api.coded import AUTH_REQUIRED from app.core.config import settings #: 开登录时匿名也能访问的接口 @@ -28,6 +30,8 @@ _PUBLIC = frozenset({"/api/health", "/api/auth/me", "/api/auth/login", "/api/aut "/api/auth/oidc/callback", "/api/auth/jwks", "/api/auth/bootstrap"}) #: 只认签名、不走会话的入站钩子(也不查 CSRF:调用方是外部系统,没有浏览器的 Origin) _HOOKS = "/api/hooks/" +#: FastAPI 自带的接口说明(main.py 里 FastAPI() 的缺省地址):开登录时要搭建者 +_DOCS = ("/docs", "/redoc", "/openapi.json") _SAFE_METHODS = frozenset({"GET", "HEAD", "OPTIONS"}) CSRF_HEADER = "x-agentlab-request" @@ -49,11 +53,17 @@ def _cookie(headers: dict[str, str], name: str) -> str | None: return morsel.value if morsel else None +def _is_docs(path: str) -> bool: + return any(path == p or path.startswith(p + "/") for p in _DOCS) + + def _origin_ok(origin: str, headers: dict[str, str], scope: dict[str, Any]) -> bool: + from app.core.public_url import origin_of, public_base_url + allowed = {o.rstrip("/") for o in settings.cors_origins} - if settings.public_url: - parts = urlsplit(settings.public_url) - allowed.add(f"{parts.scheme}://{parts.netloc}") + # 对外访问地址(环境变量或设置里的那一项,core/public_url.py):反向代理后面请求的 Host 不一定是它 + if address := public_base_url(): + allowed.add(origin_of(address)) host = headers.get("x-forwarded-host") or headers.get("host") if host: scheme = headers.get("x-forwarded-proto") or scope.get("scheme") or "http" @@ -92,7 +102,7 @@ class AuthMiddleware: await self.app(scope, receive, send) return from app.api.deps import reset_principal, set_principal - from app.auth import current_mode + from app.auth import current_mode, refusal # 登录设置在启动时读进缓存(main.py 的 _platform_startup),改设置的接口改完就刷新;这里不查库 mode = current_mode() @@ -130,7 +140,11 @@ class AuthMiddleware: return if (scope["type"] == "http" and is_api and not principal.username and path not in _PUBLIC and not path.startswith(_HOOKS)): - await _reply(send, 401, "请先登录", "auth_required") + await _reply(send, 401, "请先登录", AUTH_REQUIRED) + return + if scope["type"] == "http" and _is_docs(path) and (refused := refusal(principal, "builder")) is not None: + # 和路由级依赖(auth.require)同一个判断、同一句话:没登录 401,角色不够 403 + await _reply(send, refused.status_code, refused.detail, refused.code) return token = set_principal(principal) diff --git a/backend/app/blueprints/instantiate.py b/backend/app/blueprints/instantiate.py index d7e966ab..1cb96967 100644 --- a/backend/app/blueprints/instantiate.py +++ b/backend/app/blueprints/instantiate.py @@ -198,7 +198,7 @@ def instantiate(blueprint: dict[str, Any], *, filled: dict[str, Any] | None = No spec = _spec_of(blueprint) ref = spec.get("template") or {} template = get_template(str(ref.get("id") or "")) - problems = [i for i in blueprint_issues(spec, for_="trial") if i["level"] == "error"] + problems = [i for i in blueprint_issues(spec, for_="trial", graph=graph) if i["level"] == "error"] if template is None: from app.apps.spec import empty_spec diff --git a/backend/app/blueprints/ops.py b/backend/app/blueprints/ops.py index a626fed5..59d1aaa4 100644 --- a/backend/app/blueprints/ops.py +++ b/backend/app/blueprints/ops.py @@ -520,8 +520,47 @@ def _container(spec: dict[str, Any], tokens: list[str]) -> tuple[dict[str, Any], raise BpRefused(f"不能改这个位置:/{'/'.join(tokens)}") -def _set(spec: dict[str, Any], tokens: list[str], value: Any, source: str) -> None: +def is_freeform(spec: dict[str, Any]) -> bool: + from app.blueprints.templates._common import is_freeform as imports_graph + + return imports_graph(spec) + + +#: 自由形的「谁来做」只定由谁做(05-已知遗留 (d)、遗留-Q5) +FREEFORM_PEOPLE_ONLY_WHO = "自由形蓝图的「谁来做」只定每一步由谁做;会签、时限、超时、加减步骤在画布上改" +FREEFORM_PEOPLE_USER = "「谁来做」要由你来定:派活给谁是人的决定,助手不替你选" + + +def _freeform_who(spec: dict[str, Any], tokens: list[str], value: Any, source: str, graph: Any) -> None: + """自由形的 /people//who:i 是按导入的图列出的第几个岗位(freeform.people_rows 的岗位行,界面照那一份的 path 写), + 绑定存进 spec.people[](id 就是岗位名、只有 who)。只收人和组:岗位、发起人在图上已经写定了。""" + from app.blueprints.templates import freeform + from app.blueprints.templates._common import with_graph + + if len(tokens) != 3 or tokens[2] != "who": + raise BpRefused(FREEFORM_PEOPLE_ONLY_WHO, code="bp.template_mismatch") + if source != "user": + raise BpRefused(FREEFORM_PEOPLE_USER) + if graph is None: + raise BpRefused("这张蓝图还没有导入的图,没法按图定谁来做") + rows = [r for r in freeform.people_rows(with_graph(spec, graph)) if r["kind"] == "slot"] + slot = rows[_index(tokens[1], len(rows))]["slot"] + checked = _checked("people", "who", value, spec) + if checked and any(p["kind"] not in ("user", "group") for p in checked): + raise BpRefused("这里只选人或组:岗位、发起人在图上已经定了") + entries = spec.setdefault("people", []) + entry = next((e for e in entries if e.get("id") == slot), None) + if entry is None: + entry = {"id": slot} + entries.append(entry) + entry["who"] = _item(checked, source) + + +def _set(spec: dict[str, Any], tokens: list[str], value: Any, source: str, graph: Any = None) -> None: card = tokens[0] + if card == "people" and is_freeform(spec): + _freeform_who(spec, tokens, value, source, graph) + return if card == "access": if len(tokens) != 2 or tokens[1] not in ACCESS_KINDS: raise BpRefused("「谁能看」只有使用、查看全部、管理三档") @@ -577,6 +616,8 @@ def _set(spec: dict[str, Any], tokens: list[str], value: Any, source: str) -> No def _add(spec: dict[str, Any], tokens: list[str], value: Any, source: str) -> None: card = tokens[0] + if card == "people" and is_freeform(spec): + raise BpRefused(FREEFORM_PEOPLE_ONLY_WHO, code="bp.template_mismatch") if card == "access": if len(tokens) not in (2, 3) or tokens[1] not in ACCESS_KINDS: raise BpRefused("「谁能看」只有使用、查看全部、管理三档") @@ -598,8 +639,13 @@ def _add(spec: dict[str, Any], tokens: list[str], value: Any, source: str) -> No entries.insert(at, _new_entry(card, value, source, spec)) -def _remove(spec: dict[str, Any], tokens: list[str], source: str) -> None: +def _remove(spec: dict[str, Any], tokens: list[str], source: str, graph: Any = None) -> None: card = tokens[0] + if card == "people" and is_freeform(spec): + if len(tokens) != 3: + raise BpRefused(FREEFORM_PEOPLE_ONLY_WHO, code="bp.template_mismatch") + _freeform_who(spec, tokens, None, source, graph) + return if card == "access": if len(tokens) != 3 or tokens[1] not in ACCESS_KINDS: raise BpRefused("「谁能看」要写明删哪一档的第几个") @@ -655,8 +701,9 @@ def _template(spec: dict[str, Any], op: BpOp, source: str) -> None: prefill(template.id, spec) -def apply_bp_op(spec: dict[str, Any], op: dict[str, Any], *, source: str) -> dict[str, Any]: - """把一个 bp_* 操作落到蓝图上,交回新的蓝图(不改入参)。source 是 user / assistant。不合规时抛 BpRefused。""" +def apply_bp_op(spec: dict[str, Any], op: dict[str, Any], *, source: str, graph: Any = None) -> dict[str, Any]: + """把一个 bp_* 操作落到蓝图上,交回新的蓝图(不改入参)。source 是 user / assistant。不合规时抛 BpRefused。 + graph 是蓝图存的图:自由形的「谁来做」按它列岗位(没给时那张卡片改不了)。""" if source not in BP_SOURCES: raise BpRefused(f"操作的来源只能是用户或助手,当前为「{source}」") if not isinstance(op, dict) or op.get("op") not in BP_OPS: @@ -677,11 +724,11 @@ def apply_bp_op(spec: dict[str, Any], op: dict[str, Any], *, source: str) -> dic if source != "user" and (only := next((k for k in USER_ONLY if tuple(tokens[:len(k)]) == k), None)): raise BpRefused(_USER_ONLY_TEXT[only[-1]]) if parsed.op == "bp_set": - _set(out, tokens, parsed.value, source) + _set(out, tokens, parsed.value, source, graph) elif parsed.op == "bp_add": _add(out, tokens, parsed.value, source) else: - _remove(out, tokens, source) + _remove(out, tokens, source, graph) try: return parse_blueprint(out).model_dump(mode="json", by_alias=True) except ValidationError as e: @@ -708,9 +755,9 @@ def _value(item: Any) -> Any: return item.get("value") if isinstance(item, dict) else None -def blueprint_issues(spec: dict[str, Any], *, for_: str) -> list[dict[str, Any]]: +def blueprint_issues(spec: dict[str, Any], *, for_: str, graph: Any = None) -> list[dict[str, Any]]: """蓝图检查(§3.13 bp.*):for_ = trial(缺项挡试运行)/ publish(待确认项挡发布)/ personal。 - 每条 {code, level, path, message}。""" + 每条 {code, level, path, message}。graph 是蓝图存的图:自由形的「谁来做」按它查(没给时不查这一张)。""" from app.blueprints.templates import get_template if for_ not in ISSUE_PURPOSES: @@ -743,7 +790,10 @@ def blueprint_issues(spec: dict[str, Any], *, for_: str) -> list[dict[str, Any]] _data_issues(spec, "data" in required, add) if "cadence" in required and _value(spec.get("cadence")) is None: add("bp.missing", "/cadence", "「多久更新」还没定:每天、每周还是每月,几点,统计哪一期") - if "people" in used: + if "people" in used and template.imports_graph: + if graph is not None and for_ != "personal": + out += _freeform_people_issues(spec, graph, publish) + elif "people" in used: _people_issues(spec, "people" in required, publish, add) if "fields" in used: _field_issues(spec, "fields" in required, add) @@ -836,6 +886,32 @@ def _people_issues(spec: dict[str, Any], required: bool, publish: bool, add: Cal f"「{label}」的条件拿「{threshold['field']}」比大小,它要是数字字段") +def _freeform_people_issues(spec: dict[str, Any], graph: Any, publish: bool) -> list[dict[str, Any]]: + """自由形的「谁来做」(freeform.people_rows):岗位、核实人没绑的报 bp.people_unbound(试运行 warning:待办派给试运行的 + 人自己;发布 error),路径就是那一行的 path,带上 slot(发布单据此去掉同一件事的 app.slot_unbound / app.reconciler_unbound)。 + 图里已经没有的岗位、卡片上还留着绑定的,照实提醒(warning),不静默删掉。""" + from app.blueprints.templates import freeform + from app.blueprints.templates._common import with_graph + + out: list[dict[str, Any]] = [] + level = "error" if publish else "warning" + rows = freeform.people_rows(with_graph(spec, graph)) + for row in rows: + if row["kind"] not in ("slot", "reconciler") or row["who"]: + continue + message = (f"{row['label']}还没定:{row['detail']},写入外部系统的结果不明时要有人去目标系统里核实" + if row["kind"] == "reconciler" else f"{row['label']}还没定谁来处理({row['detail']})") + out.append({"code": "bp.people_unbound", "level": level, "path": row["path"], "message": message, + "slot": row["slot"]}) + known = {r["slot"] for r in rows if r["kind"] == "slot"} + for step in spec.get("people") or []: + slot = str(step.get("id") or "") + if slot and slot not in known and _value(step.get("who")): + out.append({"code": "bp.template_mismatch", "level": "warning", "path": "/people", + "message": f"岗位「{slot}」在导入的图里已经没有了:卡片上留着的这条绑定不会生效"}) + return out + + def _field_issues(spec: dict[str, Any], required: bool, add: Callable[..., None]) -> None: fields = spec.get("fields") or [] if required and not fields: diff --git a/backend/app/blueprints/publish_plan.py b/backend/app/blueprints/publish_plan.py index 3bbcecf1..b3c7a289 100644 --- a/backend/app/blueprints/publish_plan.py +++ b/backend/app/blueprints/publish_plan.py @@ -70,9 +70,13 @@ NOT_INSTANTIATED = "plan.not_instantiated" GRAPH_CHANGED = "plan.graph_changed" UNTRIALED = "plan.untrialed" TRIAL_FAILED = "plan.trial_failed" +#: 最近一次试运行在某一步被人驳回、没有走完(cancelled + end_reason=rejected):人的决定,不是故障,不挡发布、照实说 +TRIAL_REJECTED = "plan.trial_rejected" TRIAL_DIFFERS = "plan.trial_differs" #: 蓝图检查:写回的结果不明时没人核实(blueprints/ops.blueprint_issues) RECONCILER_UNBOUND = "bp.reconciler_unbound" +#: 蓝图检查:「谁来做」有一步还没定由谁处理;自由形的带着 slot(哪个岗位,核实人是 apps/derive.RECONCILER) +PEOPLE_UNBOUND = "bp.people_unbound" ADMIN_REQUIRED = "plan.admin_required" DOWNGRADE_ADMIN = "plan.downgrade_admin" PREFLIGHT = "plan.preflight" @@ -237,8 +241,10 @@ async def _simulated_pins(session: AsyncSession, spec: dict[str, Any], target: _ def bindings_of(bp: Blueprint) -> dict[str, Any]: """岗位绑定:「谁来做」卡片上写了具体的人或组的,按模板定的岗位名绑上(templates/_common.bindings_of)。""" from app.blueprints.templates._common import bindings_of as from_cards + from app.blueprints.templates._common import with_graph - return from_cards(bp.spec or {}) + # 带上蓝图存的图:自由形的核实人要不要绑按导入的图算(05-已知遗留 (d)) + return from_cards(with_graph(bp.spec or {}, bp.graph)) def grant_items(bp: Blueprint) -> list[tuple[str, str, str]]: @@ -352,6 +358,14 @@ async def _preflight(session: AsyncSession, bp: Blueprint, target: _Target, *, l issues = [i for i in issues if i.get("code") != "app.reconciler_unbound" and not (i.get("code") == "app.slot_unbound" and i.get("path") == f"slots.{RECONCILER}")] + if unbound := {str(i["slot"]) for i in out if i.get("code") == PEOPLE_UNBOUND and i.get("slot")}: + # 自由形的「谁来做」:岗位、核实人没绑,蓝图检查已经指到卡片上那一行(bp.people_unbound 带着 slot),应用检查说的是 + # 同一件事,不再重复列(同上面核实岗位的办法) + from app.apps.derive import RECONCILER + + issues = [i for i in issues if not (i.get("code") == "app.slot_unbound" + and str(i.get("path") or "").removeprefix("slots.") in unbound) + and not (i.get("code") == "app.reconciler_unbound" and RECONCILER in unbound)] out += [{**i, "source": "app"} for i in issues] # 授权:给的人和组要在名录里(同授权接口) _, problem = await grants_problem(session, grant_items(bp)) @@ -363,6 +377,12 @@ async def _preflight(session: AsyncSession, bp: Blueprint, target: _Target, *, l if is_trial_of(trial, bp) and trial.status == "failed": why = (trial.error or "").strip().splitlines()[0][:200] if (trial.error or "").strip() else "" out.append(_problem(TRIAL_FAILED, f"最近一次试运行失败了{f':{why}' if why else ''}", source="plan")) + elif is_trial_of(trial, bp) and trial.status == "cancelled" and trial.end_reason == "rejected": + # 搭建者在试运行里驳回了写入、审批:那是他的决定,不是图出了错,不说成「试运行失败了」、也不挡发布;只照实说 + # 没走完——驳回之后的步骤没用真数看过 + where = _node_title(trial.graph, trial.error_node_id) + out.append(_problem(TRIAL_REJECTED, f"最近一次试运行{f'在「{where}」' if where else ''}被驳回、没有走完", + "warning", source="plan")) elif not is_trial_of(trial, bp) or trial.status != "succeeded": out.append(_problem(UNTRIALED, "还没有成功的试运行:发布之前没用真数看过结果", "warning", source="plan")) elif graph_hash(trial.graph or {}) != target.digest: @@ -370,6 +390,17 @@ async def _preflight(session: AsyncSession, bp: Blueprint, target: _Target, *, l return [_with_who(p) for p in out] +def _node_title(graph: Any, node_id: str | None) -> str | None: + """节点在图上的名字(没有名字的给 id);没有节点时为 None。""" + if not node_id: + return None + for node in (graph or {}).get("nodes") or []: + if isinstance(node, dict) and node.get("id") == node_id: + data = node.get("data") if isinstance(node.get("data"), dict) else {} + return str(data.get("label") or node_id) + return node_id + + def _with_who(issue: dict[str, Any]) -> dict[str, Any]: """问题带上白话(没登记白话的照旧只有技术说法)和归谁处理(you / assistant / admin,摘要按它分三栏)。""" from app.plain import who_of, with_plain @@ -428,6 +459,7 @@ def _summary(bp: Blueprint, target: _Target, *, level: str, first_run: bool, opt if current and current > target.version else "" codes = {p.get("code") for p in problems} tried = ";发布的内容和最近一次试运行不同(试运行之后又改过),建议再试一次" if TRIAL_DIFFERS in codes else \ + ";最近一次试运行被驳回、没有走完" if TRIAL_REJECTED in codes else \ ";还没有用真数试运行过" if UNTRIALED in codes else "" if target.published_as == level: say("publish_workflow", f"「{name}」v{target.version} 已经按{LEVEL_TEXT[level]}级别发布过,不重复发布{tried}") diff --git a/backend/app/blueprints/store.py b/backend/app/blueprints/store.py index a3ef2fc4..377a0b3f 100644 --- a/backend/app/blueprints/store.py +++ b/backend/app/blueprints/store.py @@ -183,7 +183,7 @@ async def apply_ops_and_save(session: AsyncSession, blueprint: Any, ops: list[di try: op = await _verified_reuse(session, raw) before = _at(spec, op.get("path")) - spec = apply_bp_op(spec, op, source=source) + spec = apply_bp_op(spec, op, source=source, graph=blueprint.graph) except BpRefused as e: if not partial: raise @@ -313,7 +313,7 @@ def blueprint_issues_of(blueprint: Any) -> list[dict[str, Any]]: from app.plain import with_plain spec = blueprint.spec or {} - issues = blueprint_issues(spec, for_="personal" if blueprint.scope == "personal" else "publish") + issues = blueprint_issues(spec, for_="personal" if blueprint.scope == "personal" else "publish", graph=blueprint.graph) for joint_id, state in (blueprint.joints or {}).items(): if isinstance(state, dict) and state.get("status") in ("pending", "refused"): issues.append({"code": "tpl.joint_unfilled" if state["status"] == "pending" else "tpl.joint_refused", @@ -375,11 +375,19 @@ async def blueprint_view(session: AsyncSession, blueprint: Any) -> dict[str, Any return blueprint_out(blueprint, extra=await data_source_issues(session, blueprint)) +def _derived_people(blueprint: Any) -> list[dict[str, Any]]: + from app.blueprints.templates import freeform + from app.blueprints.templates._common import with_graph + + return freeform.people_rows(with_graph(blueprint.spec or {}, getattr(blueprint, "graph", None))) + + def blueprint_out(blueprint: Any, *, issues: bool = True, extra: list[dict[str, Any]] | None = None) -> dict[str, Any]: """一张蓝图交给前端的样子(frontend/src/typesBlueprint.ts 的 Blueprint)。关节只给状态和模板声明的说法、类型、分组 (JointState:status、by、note、label、kind、group),不给填的值(值在图上看)。reconciler_needed:写回卡片上要不要 - 摆「结果不明时谁去核实」(目标能力没登记查询,templates/_common.needs_reconciler),界面不自己判。""" - from app.blueprints.templates._common import needs_reconciler + 摆「结果不明时谁去核实」(目标能力没登记查询,templates/_common.needs_reconciler),界面不自己判。derived_people:自由形 + 按导入的图列出的「谁来做」(templates/freeform.people_rows,05-已知遗留 (d)),别的模板为空。""" + from app.blueprints.templates._common import is_freeform, needs_reconciler, with_graph joints = {jid: {k: v for k, v in state.items() if k in ("status", "by", "note", "label", "kind", "group")} for jid, state in (blueprint.joints or {}).items() if isinstance(state, dict)} @@ -389,7 +397,9 @@ def blueprint_out(blueprint: Any, *, issues: bool = True, extra: list[dict[str, "template_version": blueprint.template_version, "source": blueprint.source or {}, "conversation_id": blueprint.conversation_id, "workflow_id": blueprint.workflow_id, "app_id": blueprint.app_id, "last_trial_run_id": blueprint.last_trial_run_id, "graph_hash": blueprint.graph_hash, "joints": joints, - "copied_from": blueprint.copied_from, "reconciler_needed": needs_reconciler(blueprint.spec or {}), + "copied_from": blueprint.copied_from, + "reconciler_needed": needs_reconciler(with_graph(blueprint.spec or {}, getattr(blueprint, "graph", None))), + "derived_people": _derived_people(blueprint) if is_freeform(blueprint.spec or {}) else None, "created_at": blueprint.created_at.isoformat() if blueprint.created_at else None, "updated_at": blueprint.updated_at.isoformat() if blueprint.updated_at else None, } diff --git a/backend/app/blueprints/templates/_common.py b/backend/app/blueprints/templates/_common.py index 589c221a..20ee6e0c 100644 --- a/backend/app/blueprints/templates/_common.py +++ b/backend/app/blueprints/templates/_common.py @@ -78,6 +78,19 @@ def context(bp: dict[str, Any]) -> dict[str, Any]: return ctx if isinstance(ctx, dict) else {} +def with_graph(spec: dict[str, Any], graph: Any) -> dict[str, Any]: + """只有卡片(Blueprint.spec)、没有实例化上下文的地方,带上蓝图存的图再交给模板的函数:自由形的「谁来做」、要不要 + 核实人都按导入的图算(freeform.people_rows / needs_reconciler)。""" + return {**spec, CONTEXT_KEY: {**context(spec), "graph": graph}} + + +def is_freeform(spec: dict[str, Any]) -> bool: + """卡片上选的是导入图的模板(自由形)。""" + ref = spec.get("template") + template = get_template(str(ref.get("id") or "")) if isinstance(ref, dict) else None + return template is not None and template.imports_graph + + def app_name(bp: dict[str, Any], fallback: str) -> str: """应用和工作流的名字:实例化时给的名字,没给就用目的的前一段。""" name = str(context(bp).get("name") or "").strip() @@ -434,6 +447,12 @@ def needs_reconciler(bp: dict[str, Any]) -> bool: 蓝图(reconciler_needed)共用这一份。""" ref = bp.get("template") template = get_template(str(ref.get("id") or "")) if isinstance(ref, dict) else None + if template is not None and template.imports_graph: + # 自由形没有「写回哪里」:按导入的图里的写入算(bp 要带着图,with_graph),核实人存在 /writeback/reconciler + # (05-已知遗留 (d)) + from app.blueprints.templates import freeform + + return freeform.needs_reconciler(bp) if template is None or "writeback" not in template.cards: return False capability = value((bp.get("writeback") or {}).get("capability")) diff --git a/backend/app/blueprints/templates/freeform.py b/backend/app/blueprints/templates/freeform.py index 42cc6c3a..db886bb4 100644 --- a/backend/app/blueprints/templates/freeform.py +++ b/backend/app/blueprints/templates/freeform.py @@ -1,5 +1,11 @@ """自由形 freeform(§3.3):从已有的运行(问数据的一轮回答,经 from-run 提取)或已发布的工作流版本导入,图原样用, -没有关节。蓝图只有目的、谁能看、多久更新三张卡片;要改图只能回画布(搭建者)。 +没有关节。蓝图有目的、谁能看、多久更新、谁来做四张卡片;要改图只能回画布(搭建者)。 + +「谁来做」(docs/design/next-round/05-已知遗留.md (d)):骨架就是导入的图,卡片只填「谁」这个关节——按图列出用到的岗位 +(人工节点派给谁、超时升级给谁、写入的发送确认派给谁,people_rows),写入能力没登记查询时再加一行「写入结果不明时谁去核实」。 +只定由谁做:会签、时限、超时在画布上改(改了会改图,图哈希就和发布的那一版对不上)。绑定存在 spec.people[](Step.id 就是 +岗位名、只写 who,_common.slot_of / bindings_of 原样生效),核实人存在 /writeback/reconciler。以前这张卡片不存在,第一次 +发布挡在 app.slot_unbound / app.reconciler_unbound 上,白话却叫人去「谁来做」里选人。 图由实例化的上下文给(graph,导入时存在蓝图上)。应用骨架按 apps/derive.py 的派生规则:有人工节点或写入的按流程, 否则按看板;写入节点按能力登记的进程内快照认(和运行时同一个判断,capabilities.registry.writes_externally)。 @@ -38,6 +44,65 @@ def _writes(graph: dict[str, Any]) -> dict[str, str]: return out +#: 「谁来做」一行里每处用途怎么说(apps/tree.SLOT_ROLES,加上核实) +USE_TEXT = {"assignee": "处理", "escalation": "超时升级", "confirm": "发送前确认", "reconcile": "写入结果不明时去核实"} +#: 不按岗位派的步骤(只读行):发起人自己、按表达式算出来的人、等外部系统回调 +FIXED_TEXT = {"initiator": "由发起人自己处理", "expression": "按图里的表达式算出由谁处理(在画布上改)", + "hook": "等外部系统回调,不派给人"} +#: 核实人那一行的名字 +RECONCILER_LABEL = "写入结果不明时谁去核实" + + +def people_rows(bp: dict[str, Any]) -> list[dict[str, Any]]: + """按导入的图列出「谁来做」:[{kind, slot, label, detail, path, who, uses}]。 + + kind=slot:图里引用的一个岗位(同一个岗位用在几处合成一行,按节点出现的顺序),path 是 /people//who(i 是岗位行的 + 序号),who 是卡片上绑的人或组;kind=reconciler:写入能力没登记查询时的核实人,path 是 /writeback/reconciler; + kind=initiator / expression / hook:不按岗位派的步骤,只读(path 为空)。label 用节点在图上的名字拼(岗位名是画布上 + 随手写的 id),detail 写全每一处用途。界面照这一份画,不自己从图里推。""" + from app.apps.derive import RECONCILER + from app.apps.tree import node_map, slot_refs + + graph = imported(bp) + writes = _writes(graph) + nodes = node_map(graph) + bound = {str(step.get("id")): c.value(step.get("who")) for step in bp.get("people") or [] if isinstance(step, dict)} + rows: list[dict[str, Any]] = [] + by_slot: dict[str, dict[str, Any]] = {} + for ref in slot_refs(graph, writes): + row = by_slot.get(ref["slot"]) + if row is None: + row = by_slot[ref["slot"]] = {"kind": "slot", "slot": ref["slot"], "path": f"/people/{len(by_slot)}/who", + "who": bound.get(ref["slot"]), "uses": []} + rows.append(row) + row["uses"].append({"node_id": ref["node_id"], "title": ref["title"], "role": ref["role"]}) + unchecked = [(nid, cap) for nid, cap in writes.items() if not c.write_lookup(cap)] + if unchecked: + rows.append({"kind": "reconciler", "slot": RECONCILER, "path": "/writeback/reconciler", + "who": c.value((bp.get("writeback") or {}).get("reconciler")), + "uses": [{"node_id": nid, "title": nodes[nid]["title"], "role": "reconcile"} for nid, _ in unchecked]}) + for nid, n in nodes.items(): + if n["type"] != "human" and nid not in writes: + continue + assignee = n["config"].get("assignee") + kind = assignee.get("kind") if isinstance(assignee, dict) else None + if kind in FIXED_TEXT: + rows.append({"kind": kind, "slot": None, "path": None, "who": None, + "uses": [{"node_id": nid, "title": n["title"], + "role": "assignee" if n["type"] == "human" else "confirm"}]}) + for row in rows: + titles = list(dict.fromkeys(u["title"] for u in row["uses"])) + row["label"] = RECONCILER_LABEL if row["kind"] == "reconciler" else "、".join(f"「{t}」" for t in titles) + row["detail"] = FIXED_TEXT[row["kind"]] if row["kind"] in FIXED_TEXT else ";".join( + f"「{u['title']}」{USE_TEXT[u['role']]}" for u in row["uses"]) + return rows + + +def needs_reconciler(bp: dict[str, Any]) -> bool: + """导入的图里有写入能力没登记查询(同 app_skeleton 加核实岗位的判断)。""" + return any(not c.write_lookup(cap) for cap in _writes(imported(bp)).values()) + + def app_skeleton(bp: dict[str, Any], graph: dict[str, Any]) -> dict[str, Any]: from app.apps.derive import ENTRY, INSTANCE_VIEW, PIN, VIEW, _entries, _results from app.apps.forms import input_schema @@ -103,6 +168,7 @@ TEMPLATE = register(TemplateDef( applies_when="用户已经有一次满意的问数据回答,或者一个已发布的工作流,想直接做成应用;需求在五个场景之外时也用它。", cards={"purpose": "必填:一句话说清这个应用做什么", "cadence": "可选:要不要定期自动出数", + "people": "可选:只定每一步由谁做;会签、时限、超时在画布上改", "access": "可选:谁能用、谁能看全部、谁能管理"}, personal="导入的图满足问数据的规则(只读、没有人工节点和写入)时可以", skeleton=skeleton, app_skeleton=app_skeleton, schedule=schedule, tags=["导入"], diff --git a/backend/app/blueprints/trial.py b/backend/app/blueprints/trial.py index 1142ece8..96578058 100644 --- a/backend/app/blueprints/trial.py +++ b/backend/app/blueprints/trial.py @@ -147,7 +147,7 @@ async def materialize(session: AsyncSession, bp: Blueprint, *, principal: Princi digest = graph_hash(graph or {}) app_spec = _draft_pins(result.app_spec, bp, digest) # 实例化交回的是挡路的 error;蓝图检查里的提醒(模板版本旧了之类)另外带上 - warnings = [i for i in blueprint_issues(bp.spec or {}, for_="trial") if i.get("level") != "error"] + warnings = [i for i in blueprint_issues(bp.spec or {}, for_="trial", graph=bp.graph) if i.get("level") != "error"] problems = [with_plain(p) for p in [*result.problems, *warnings]] bp.graph, bp.graph_hash, bp.joints = graph, digest, result.joints bp.updated_by = principal.username if principal else bp.updated_by diff --git a/backend/app/bundle/build.py b/backend/app/bundle/build.py index 806ecff7..a7ca0679 100644 --- a/backend/app/bundle/build.py +++ b/backend/app/bundle/build.py @@ -1237,8 +1237,8 @@ def build_bundle( ) -> dict[str, Any]: """一次已封存运行 → 页面里的数据(bundle-data)。 - - run:运行记录的几样(id、workflow_id、workflow_name、version、version_hash、run_class、status、 - started_at、finished_at) + - run:运行记录的几样(id、workflow_id、workflow_name、version、version_hash、run_class、status、end_reason、 + started_at、finished_at)。end_reason 分开同是 cancelled 的被驳回和手动停止(05-已知遗留 (a)),不在封存范围内 - events:这次运行的全部事件 (seq, type, node_id, data),按数据库里 seq 的顺序。和封存核对 (runner.verify_manifest)读的是同一个顺序:seq 相同的几条,浏览器按这里的先后排,清单才算得一样 - manifest_seq / manifest_hash:封存范围(老运行是按结束事件推算的那个,legacy 为真)和封存哈希 @@ -1306,7 +1306,7 @@ def build_bundle( "meta": { "run_id": run.get("id"), "workflow_id": run.get("workflow_id"), "workflow_name": display_name(run), "version": run.get("version"), "version_hash": run.get("version_hash"), - "run_class": run.get("run_class"), "status": run.get("status"), + "run_class": run.get("run_class"), "status": run.get("status"), "end_reason": run.get("end_reason"), "started_at": _iso(run.get("started_at")), "finished_at": _iso(run.get("finished_at")), "manifest_seq": manifest_seq, "manifest_hash": manifest_hash, "legacy": bool(legacy), "tier": _tier(sealed), "exported_at": exported_at, diff --git a/backend/app/bundle/verifier.js b/backend/app/bundle/verifier.js index 0d253e43..f8cebed3 100644 --- a/backend/app/bundle/verifier.js +++ b/backend/app/bundle/verifier.js @@ -49,7 +49,9 @@ const KIND_LABEL = { report_doc: '报告文档', metric_set: '口径卡', query: '查询快照' } const TIER_LABEL = { formal: '完整出具', degraded: '降档出具', withheld: '不予出具' } const CLASS_LABEL = { formal: '正式运行', exploratory: '探索运行' } - const STATUS_LABEL = { succeeded: '已完成', failed: '已失败', cancelled: '已取消' } + // rejected:被审批人驳回结束(运行记录 cancelled + end_reason=rejected)。封存里它和手动停止是同一种结束事件(运行取消), + // 分开二者的原因在那条事件的内容里,内容不随文件附带:见 checkMeta + const STATUS_LABEL = { succeeded: '已完成', failed: '已失败', cancelled: '已取消', rejected: '被驳回' } /** 结束类事件 → 它说明的运行结果(runner:成功、失败、取消各发一条,落库之后才封存) */ const TERMINAL = { 'run.finished': 'succeeded', 'run.failed': 'failed', 'run.cancelled': 'cancelled' } const EVENT_LABEL = { @@ -84,6 +86,7 @@ range: (seq, count) => `第 1–${seq} 条事件(范围内 ${count} 条)`, rangeLegacy: '(早期运行,范围按结束事件推算)', markPass: '已核对', + markPassRejected: '已核对运行取消,被驳回取自运行记录', markFail: (claim) => `文件里写的是「${claim}」,见核对明细 ⑥`, markNa: '无法核对', noReports: '这次运行没有报告文档,下方只核对运行记录。', @@ -208,6 +211,8 @@ inputFrom: (query, row, column) => (query ? `查询 ${query}` : '') + (Number.isInteger(row) ? ` · 第 ${row + 1} 行` : '') + (column ? ` · ${column}` : ''), inputNotAttached: '快照未随文件附带(见核对明细 ③),输入的值按口径卡的记录展示', statusOk: (seq, what) => `与第 ${seq} 条事件(${what})一致`, + statusOkRejected: (seq, what) => `与第 ${seq} 条事件(${what})一致。被驳回和手动停止在封存里是同一种结束事件,` + + '「被驳回」取自运行记录(这条事件的内容不随文件附带),需要在系统中对照', statusMismatch: (claim, seq, what) => `文件里写的是「${claim}」,与封存范围内最后一条结束类事件(第 ${seq} 条,${what})不符`, statusLegacy: '早期运行在结束事件落库之前封存,结束事件不在封存范围内,运行结果无法核对', statusNoEvent: '封存范围内没有运行结束、失败或取消的事件,运行结果无法核对', @@ -1509,7 +1514,8 @@ */ function checkMeta(meta, out) { const items = [] - const add = (field, shown, claim, result, reason) => items.push({ field, shown, claim, result, reason }) + // mark:显示值旁边那一句的特例(默认按结果写已核对 / 不通过 / 无法核对) + const add = (field, shown, claim, result, reason, mark) => items.push({ field, shown, claim, result, reason, mark }) const order = out.order const sealOk = out.manifest[0].result === PASS const label = (map, value) => map[value] || (value ? String(value) : '—') @@ -1519,7 +1525,10 @@ // 它就是范围内的最后一条;早期运行在 run.finished 落库之前封存,成功结束的那条在范围外,失败、取消的那条在范围的最后。 // 先失败、经继续运行接着跑的,范围里那条运行失败之后运行又继续了,它不是运行的结束:拿它去对,元信息改成「已失败」 // 也会被标成已核对。真正截止的那条不在范围里时,只有早期运行成功结束说得通(无法核对),别的说法都没有事件印证 - const statusClaim = label(STATUS_LABEL, meta.status) + // 被驳回(cancelled + end_reason=rejected)照运行记录写「被驳回」,不写成手动停止的「已取消」;能核对的只有结束事件 + // 是运行取消这一半,另一半写明取自运行记录 + const rejected = meta.status === 'cancelled' && meta.end_reason === 'rejected' + const statusClaim = rejected ? STATUS_LABEL.rejected : label(STATUS_LABEL, meta.status) const lastSeq = order.length ? order[order.length - 1].seq : null const final = [...order].reverse().find((ev) => TERMINAL[ev.type]) // 同一序号的几条(老数据里有)都算最后:结束事件和别的事件撞了序号,不能因此说它之后运行又继续了 @@ -1527,8 +1536,12 @@ if (!sealOk) add('status', statusClaim, statusClaim, NA, T.metaNeedsManifest) else if (end) { const ok = TERMINAL[end.type] === meta.status - add('status', label(STATUS_LABEL, TERMINAL[end.type]), statusClaim, ok ? PASS : FAIL, - ok ? T.statusOk(end.seq, what(end)) : T.statusMismatch(statusClaim, end.seq, what(end))) + if (ok && rejected) { + add('status', statusClaim, statusClaim, PASS, T.statusOkRejected(end.seq, what(end)), T.markPassRejected) + } else { + add('status', label(STATUS_LABEL, TERMINAL[end.type]), statusClaim, ok ? PASS : FAIL, + ok ? T.statusOk(end.seq, what(end)) : T.statusMismatch(statusClaim, end.seq, what(end))) + } } else if (meta.legacy && meta.status === 'succeeded') add('status', statusClaim, statusClaim, NA, T.statusLegacy) // 显示的值写「—」:范围里没有说明运行怎样结束的事件,不能把文件里写的当成核对出来的 else if (final) add('status', '—', statusClaim, FAIL, T.statusContinued(statusClaim, final.seq, what(final))) @@ -1899,7 +1912,7 @@ const checked = (field, fallback) => { const item = out.meta.find((m) => m.field === field) if (!item) return fallback - const note = item.result === PASS ? T.markPass : item.result === FAIL ? T.markFail(item.claim) : T.markNa + const note = item.mark ?? (item.result === PASS ? T.markPass : item.result === FAIL ? T.markFail(item.claim) : T.markNa) return h('span', { 'data-meta': field, 'data-result': item.result }, item.shown, h('span', { class: item.result === FAIL ? 'bad' : 'muted' }, `(${note})`)) } diff --git a/backend/app/chat/__init__.py b/backend/app/chat/__init__.py new file mode 100644 index 00000000..508c2ca1 --- /dev/null +++ b/backend/app/chat/__init__.py @@ -0,0 +1,5 @@ +"""问数据(一轮:提问 → 建图 → 运行 → 复核)在后端的部分。 + +- turn.py:一轮里的几个「决定」(要不要运行、输入怎么填、要不要带原图重来、重来时说什么),问数据页和评测走同一套; +- questions.py:问过的问题原文和它们的聚类,全站只有这一份(治理接口、以后的口径账本、评测都调它)。 +""" diff --git a/backend/app/chat/questions.py b/backend/app/chat/questions.py new file mode 100644 index 00000000..6e7e9201 --- /dev/null +++ b/backend/app/chat/questions.py @@ -0,0 +1,115 @@ +"""问过的问题和它们的聚类:全站只有这一份(docs/design/next-round/00-总方案.md X2)。 + +同一个问题被反复问,就该长成模板里的固定一节、沉淀成口径——探索问题聚类(/governance/exploratory-clusters)、以后口径 +账本的「可沉淀的问题」、评测的问题重复度(E3)量的都是这件事,各算一份的话几个数对不上,所以取问题和聚类只写在这里。 + +**取什么。** 只取问题原文,不取整份输入:以前把运行的整份 input(含 history)拿去聚,同一段对话里前几轮的问答会把不同的 +问题聚成一类,重复度偏高。 + +- 问数据:一轮一条(conversation_turns.question,用户原本问的那句话)。同一轮里的自动重来、用户点的「重试」只是换了这一轮 + 挂着的运行,不多出一条——按运行数的话,答坏了自动重来一次就被算成问了两遍。没挂运行的轮次(只回了一句、助手反问) + 也算:问题照样被问过。 +- 别的探索运行(画布试跑、蓝图试运行、接口……):输入里有 question 这个字段的取它;没有的不收(输入里别的字段不是问题)。 + 问数据的运行由上面一轮一条代表,这里不再数。 + +**不收什么。** 系统派生的运行(评测、口径对账、口径召回:engine/origin.counted_runs)不收,评测题不能把聚类刷高; +这个人读不了的只给拥有者的运行(auth/access.hidden_runs)不收;正式运行不收(探索才谈得上「沉淀」)。 + +聚类用 memory/embeddings 的向量贪心聚:按时间从新到旧,和某一类的种子足够像就并进去,否则自立一类。本地向量是词频 +哈希、语义能力弱,结果是下限——调用方据 embedder 照实写明用的是哪个向量模型。 +""" +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any + +from sqlalchemy import func, not_, select +from sqlalchemy.ext.asyncio import AsyncSession + +#: 缺省的相似度阈值(同治理接口一直以来的 0.62) +DEFAULT_THRESHOLD = 0.62 + + +@dataclass(frozen=True) +class Question: + """问过的一个问题。turn_id 是问数据的那一轮(别的探索运行没有);run_id 是它挂着的运行(只回了一句的轮次没有)。""" + + text: str + at: datetime | None + run_id: str | None = None + turn_id: str | None = None + workflow: str | None = None + origin: str | None = None + + +@dataclass +class QuestionCluster: + """一类问题:members 按时间从新到旧,第一个是种子。""" + + members: list[Question] = field(default_factory=list) + + @property + def count(self) -> int: + return len(self.members) + + +async def recent_questions(session: AsyncSession, principal: Any, *, limit: int) -> list[Question]: + """最近问过的问题,从新到旧,最多 limit 条(取法见模块开头)。principal 用来去掉这个人读不了的只给拥有者的运行。""" + from app.auth.access import hidden_runs + from app.db.models import Conversation, ConversationTurn, Run + from app.engine.origin import counted_runs + + hidden = hidden_runs(principal) + turns = (await session.execute( + select(ConversationTurn.id, ConversationTurn.question, ConversationTurn.created_at, ConversationTurn.run_id, + Run.workflow_name) + .join(Conversation, Conversation.id == ConversationTurn.conversation_id) + .outerjoin(Run, Run.id == ConversationTurn.run_id) + .where(Conversation.kind == "chat", func.trim(ConversationTurn.question) != "", counted_runs(), not_(hidden)) + .order_by(ConversationTurn.created_at.desc()).limit(limit) + )).all() + runs = (await session.execute( + select(Run.id, Run.input, Run.workflow_name, Run.created_at, Run.origin_kind) + .where(Run.run_class != "formal", func.coalesce(Run.origin_kind, "") != "chat", counted_runs(), not_(hidden), + func.json_type(Run.input, "$.question") == "text") + .order_by(Run.created_at.desc()).limit(limit) + )).all() + out = [Question(text=question.strip(), at=at, run_id=run_id, turn_id=turn_id, workflow=workflow, origin="chat") + for turn_id, question, at, run_id, workflow in turns] + for run_id, payload, workflow, at, origin in runs: + text = (payload or {}).get("question") if isinstance(payload, dict) else None + if isinstance(text, str) and text.strip(): + out.append(Question(text=text.strip(), at=at, run_id=run_id, workflow=workflow, origin=origin)) + out.sort(key=lambda q: q.at.timestamp() if q.at is not None else 0.0, reverse=True) + return out[:limit] + + +async def cluster_questions(questions: Sequence[Question], *, threshold: float = DEFAULT_THRESHOLD, + ) -> list[QuestionCluster]: + """向量贪心聚类(顺序同 questions:从新到旧)。""" + from app.memory import embeddings + + if not questions: + return [] + vectors = await embeddings.embed_texts([q.text for q in questions]) + clusters: list[tuple[int, QuestionCluster]] = [] + for i, question in enumerate(questions): + for seed, cluster in clusters: + if embeddings.cosine(vectors[i], vectors[seed]) >= threshold: + cluster.members.append(question) + break + else: + clusters.append((i, QuestionCluster(members=[question]))) + return [cluster for _, cluster in clusters] + + +def embedder() -> str: + """聚类用的是哪个向量模型(memory/embeddings.embedder_id):本地哈希的结果只是下限,调用方照实写出来。""" + from app.memory import embeddings + + return embeddings.embedder_id() + + +__all__ = ["DEFAULT_THRESHOLD", "Question", "QuestionCluster", "cluster_questions", "embedder", "recent_questions"] diff --git a/backend/app/chat/turn.py b/backend/app/chat/turn.py new file mode 100644 index 00000000..993c0ad5 --- /dev/null +++ b/backend/app/chat/turn.py @@ -0,0 +1,129 @@ +"""问数据一轮里的几个「决定」(细案 01 做法 1,总方案 2.1)。 + +一轮问数据是:建图(助手的生成过程,api/copilot.answer_ops)→ 要不要运行 → 运行(输入怎么填)→ 复核 → 要不要带着原图 +重来一次、重来时说什么。以前这几个决定都写在问数据页里(frontend/src/store/chat.ts 的 plan / launch / settle / retry), +后端跑不了一轮问数据:评测要么照着抄一份(迟早走样,量出来的不是用户拿到的那一轮),要么做不了;这也违反「界面层不放 +逻辑」。现在决定写在这里,是纯函数: + +- next_step:生成过程交出结局之后做什么; +- fill_input:用户的问题填进这张图的入口字段; +- retry_after_review:复核之后要不要带着原图重来一次; +- retry_instruction:重来时交给助手的那句话(两种措辞)。 + +接口把决定交给问数据页:/copilot/generate-stream 的 final 带 next(run | ready | fail),/copilot/review 的结果带 +retry_instruction(要重来时是那句话,不重来是 null)。问数据页照后端给的走;老后端没有这两个字段时退回它自己那份同样的 +规则(tests/test_chat_turn_policy.py 逐条对照两边的常量和措辞)。评测(第 3 波)用同一组函数把一轮串起来。 + +只搬「决定」:重连、断点续看、取消、复核等多久这些仍在问数据页(把整轮编排搬到后端要重做这些,单独立项)。 +""" +from __future__ import annotations + +import re +from collections.abc import Mapping +from typing import Any + +#: 一轮最多自动试几次(含第一次):第二次还不行多半是问题本身没说清,该让用户介入(同 chat.ts 的 MAX_ATTEMPTS) +MAX_ATTEMPTS = 2 + +#: final 之后的三种走法(final.next):运行、只建不跑(摆着等用户点运行)、判失败(没过检查的图不拿去跑) +NEXT_RUN, NEXT_READY, NEXT_FAIL = "run", "ready", "fail" +NEXT_STEPS = (NEXT_RUN, NEXT_READY, NEXT_FAIL) +#: 别的结局:直接回答了(reply)、助手反问了等用户答(asked)、没自查过的老服务端交来没过检查的图时重来一次(retry) +NEXT_REPLY, NEXT_ASKED, NEXT_RETRY = "reply", "asked", "retry" + +#: 问题往哪个入口字段放:名字像「问题」的优先(同 chat.ts fillInput 的正则,大小写不敏感) +QUESTION_FIELD = re.compile(r"question|query|topic|input|问题|需求|主题", re.IGNORECASE) + +#: 带着原图重来时交给助手的话:原图交回去改(调大步数、改参数这种事就是改一个字段,不是推倒重来) +RETRY_WITH_BASE = ("这个工作流上一次运行没有成功:{reason}。\n\n" + "请针对这个原因改它(比如调大最大步数、改正工具参数、换个查询方式)," + "改完继续回答原来的问题:{question}") +#: 没有图可交回(上一次连图都没建出来)时:原问题后面缀上原因,从零再建 +RETRY_FROM_SCRATCH = "{question}\n\n(上一次没有成功:{reason}。请针对这个原因调整,重新给出完整的工作流。)" + + +def _blockers(final: Mapping[str, Any]) -> list[str]: + return [str(i.get("message") or "") for i in final.get("issues") or [] + if isinstance(i, Mapping) and i.get("level") == "error"] + + +def next_step(terminal: Mapping[str, Any] | None, *, self_checked: bool = True) -> str: + """生成过程交出结局之后做什么。terminal 是最后那条结局操作(final / reply / error);流结束时只有反问(ask)、没有 + 别的结局的给 None。 + + - reply:直接回答,这一轮到此为止;error:判失败;只有反问:等用户回答; + - final:有 error 级问题时,服务端已经自查过(收到过 check 的结论)就判失败——没过的问题已经交回模型在同一轮里改过 + 了,再整轮重来是再烧一次完整生成、改的还是同一批问题;没自查过(老服务端)才重来一次(retry);模型说不运行 + (autorun=false:用户要的是流程本身、或者问了用户)只建不跑;其余运行。 + """ + if terminal is None: + return NEXT_ASKED + kind = terminal.get("op") + if kind == "reply": + return NEXT_REPLY + if kind != "final": + return NEXT_FAIL + if _blockers(terminal): + return NEXT_FAIL if self_checked else NEXT_RETRY + if terminal.get("autorun") is False: + return NEXT_READY + return NEXT_RUN + + +def blocked_reason(final: Mapping[str, Any]) -> str: + """没过检查的图重来时的原因(只有老服务端那条路用得到)。""" + return "生成的工作流没有通过校验:" + ";".join(_blockers(final)) + + +def fill_input(graph: Mapping[str, Any] | None, question: str, history: str = "") -> dict[str, Any]: + """用户的问题填进这张图的入口字段(同 chat.ts 的 fillInput)。 + + 入口没声明字段:{question, history}(下游可能用 input.question)。声明了:每个字段先填默认值(没有就空串),问题放进 + 名字像「问题」的那个,其次第一个必填的,再不然第一个;history(前几轮压成的文字)照样带上——图跑起来时模型得知道 + 「它们」指的是上一轮那批数据。 + """ + nodes = (graph or {}).get("nodes") or [] + entry = next((n for n in nodes if isinstance(n, Mapping) and n.get("type") == "input"), None) + config = ((entry or {}).get("data") or {}).get("config") or {} + fields = [f for f in (config.get("fields") or []) if isinstance(f, Mapping)] + if not fields: + return {"question": question, "history": history} + payload: dict[str, Any] = {"history": history} + for f in fields: + if f.get("name"): + payload[str(f["name"])] = f["default"] if f.get("default") is not None else "" + named = next((f for f in fields if QUESTION_FIELD.search(str(f.get("name") or ""))), None) + target = named or next((f for f in fields if f.get("required")), None) or fields[0] + if target.get("name"): + payload[str(target["name"])] = question + return payload + + +def review_reason(review: Mapping[str, Any]) -> str: + """复核判不可用时重来的原因:各条信号的说明,没有就用给用户看的那句说明。""" + details = [str(s.get("detail") or "") for s in review.get("signals") or [] if isinstance(s, Mapping)] + return ";".join(d for d in details if d) or str(review.get("note") or "") + + +def retry_after_review(review: Mapping[str, Any] | None, attempt: int) -> bool: + """复核之后要不要带着原图重来一次:复核说换个配置重跑大概率就好(retry)、答案不可信(severity=broken)、这一轮还没 + 试满(attempt 从 0 数,attempt + 1 < MAX_ATTEMPTS)。""" + if not review: + return False + return bool(review.get("retry")) and review.get("severity") == "broken" and attempt + 1 < MAX_ATTEMPTS + + +def retry_instruction(question: str, reason: str, has_base: bool) -> str: + """重来时交给助手的话。has_base:有上一次建出的图可以交回去改。""" + template = RETRY_WITH_BASE if has_base else RETRY_FROM_SCRATCH + return template.format(question=question, reason=reason) + + +def has_nodes(graph: Any) -> bool: + """一张图有没有节点(有才交回去改)。""" + return isinstance(graph, Mapping) and bool(graph.get("nodes")) + + +__all__ = ["MAX_ATTEMPTS", "NEXT_ASKED", "NEXT_FAIL", "NEXT_READY", "NEXT_REPLY", "NEXT_RETRY", "NEXT_RUN", "NEXT_STEPS", + "QUESTION_FIELD", "RETRY_FROM_SCRATCH", "RETRY_WITH_BASE", "blocked_reason", "fill_input", "has_nodes", + "next_step", "retry_after_review", "retry_instruction", "review_reason"] diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 9dcf013a..955e7242 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -1,9 +1,10 @@ from __future__ import annotations import ipaddress +import re from functools import lru_cache from pathlib import Path -from typing import Annotated +from typing import Annotated, Any from pydantic import field_validator from pydantic_settings import BaseSettings, NoDecode, SettingsConfigDict @@ -11,6 +12,9 @@ from pydantic_settings import BaseSettings, NoDecode, SettingsConfigDict BACKEND_DIR = Path(__file__).resolve().parents[2] PROJECT_DIR = BACKEND_DIR.parent +#: 嵌入白名单里一项的主机部分:域名(可以 *. 开头)或方括号里的 IPv6,可以带端口;不许路径、查询、用户名 +_ORIGIN_HOST = re.compile(r"(?:\*\.)?(?:[A-Za-z0-9-]+\.)*[A-Za-z0-9-]+(?::\d{1,5})?|\[[0-9A-Fa-f:.]+\](?::\d{1,5})?") + # session_id 拼进路径前必须过这一关。允许的字符刻意收得很窄:字母数字加 # 连字符下划线——点号也不放行,免得出现 ".." 这种能往上走的名字。 _SESSION_MAX = 64 @@ -60,10 +64,18 @@ class Settings(BaseSettings): auth_session_max_days: int = 7 # 会话 cookie 带不带 Secure。不设时按请求本身是不是 https 定:本机 http 开发照常能登录 auth_cookie_secure: bool | None = None - # 对外的访问地址(反向代理后面时填,如 https://agentlab.example.com)。OIDC 的回调地址按它拼; - # 不设时按请求的 Host 拼 + # 对外的访问地址(反向代理后面时填,如 https://agentlab.example.com)。全站只认一个(core/public_url.py): + # 设了它,设置里的「对外访问地址」就不生效、不能改;通知和 Webhook 的链接、OIDC 的回调、来源核对都按它拼。 + # 不设时用设置里的那一项,再没有就按请求的 Host 拼(通知里不附链接) public_url: str = "" + # --- 安全响应头(app/web_security.py)--- + # 允许哪些站点用 iframe 嵌整站(同站的门户,如 https://portal.example.com),逗号分隔。本站自己总是允许的。 + # 不设时只有本站能嵌(frame-ancestors 'self',另加 X-Frame-Options: SAMEORIGIN) + frame_ancestors: Annotated[list[str], NoDecode] = [] + # 分享页(/s)允许被哪些站点嵌,逗号分隔,写法同上。分享页只认这一项,不认上面那一项 + embed_origins: Annotated[list[str], NoDecode] = [] + # --- 沙箱 --- sandbox_backend: str = "auto" # auto | microvm | seatbelt | bubblewrap | local | off sandbox_timeout: int = 30 @@ -121,7 +133,8 @@ class Settings(BaseSettings): "https://223.5.5.5/resolve", ] - @field_validator("http_tool_allowlist", "http_tool_fake_ip_ranges", "http_tool_doh_urls", mode="before") + @field_validator("http_tool_allowlist", "http_tool_fake_ip_ranges", "http_tool_doh_urls", "frame_ancestors", + "embed_origins", mode="before") @classmethod def _comma_list(cls, value: object) -> object: # 环境变量里写逗号分隔;也兼容 JSON 数组的写法 @@ -134,6 +147,37 @@ class Settings(BaseSettings): return [part.strip() for part in text.split(",") if part.strip()] return value + @field_validator("public_url") + @classmethod + def _valid_public_url(cls, value: str) -> str: + # 写错了启动时就报出来:以前拼出 agentlab.example.com/api/auth/oidc/callback 这样的回调,到登录时才发现 + from app.core.public_url import ENV_NAME, normalize, url_problem + + if not value.strip(): + return "" + if problem := url_problem(value): + what = "不能带 ? 或 # 之后的部分" if problem == "suffix" else "需要是 http:// 或 https:// 开头的完整地址" + raise ValueError(f"{ENV_NAME}={value!r} {what},例如 https://agentlab.example.com") + return normalize(value) or "" + + @field_validator("frame_ancestors", "embed_origins") + @classmethod + def _valid_origins(cls, value: list[str], info: Any) -> list[str]: + # 只收「协议://主机[:端口]」,主机可以写 *.example.com。路径、裸 *、没写协议的写错了启动时就报出来,别到嵌入 + # 失败(或者整站被谁都能嵌)才发现。'self' 本来就有,写了也不重复 + name = f"AGENTLAB_{(info.field_name or '').upper()}" + out: list[str] = [] + for raw in value: + item = raw.strip().rstrip("/") + if item == "'self'": + continue + scheme, _, host = item.partition("://") + if scheme.lower() not in ("http", "https") or not _ORIGIN_HOST.fullmatch(host): + raise ValueError(f"{name} 里的「{raw}」需要写成「协议://主机」(可以带端口),例如 https://portal.example.com") + if (origin := f"{scheme.lower()}://{host.lower()}") not in out: + out.append(origin) + return out + @field_validator("auth_cookie_secure", mode="before") @classmethod def _blank_is_auto(cls, value: object) -> object: diff --git a/backend/app/core/job_runtime.py b/backend/app/core/job_runtime.py new file mode 100644 index 00000000..98b641a9 --- /dev/null +++ b/backend/app/core/job_runtime.py @@ -0,0 +1,439 @@ +"""后台任务的共用运行时:数据目录的起草(data/catalog_jobs.py)和剖析(data/catalog_profile_jobs.py)各自建表、各自的步骤, +生命周期是同一套;之后的评测、后台探查也用它(docs/design/next-round/00-总方案.md X5)。 + +**为什么在 core。** 最早只服务数据目录,放在 data/catalog_job_runtime.py;评测(一次评测一个任务、每题一项)和后台探查 +(同源排队)要的是同一样东西,从 data 包里导入一个「数据目录的」运行时说不通。原路径留着再导出(同一个模块对象里的 +名字,lanes 也是同一条队),已有的 import 不用改,正在改起草、剖析的工作也不会因此冲突。 + +**为什么抽出来。** 剖析以前是一次同步长请求,和起草改成后台任务之前是同一类问题:关掉页面结果就没了、不能停、不留痕。 +两边要的是同一样东西——进程内跑、能停、重启标中断、写库不被停止打断、同源排队——各写一份,迟早一边修了另一边没修。 +共用的是生命周期代码,不是表:两种任务的进度完全不同(起草按段、块、模型用量,剖析按检查项、查询),合成一张 +「kind + JSON」的通用任务表要丢掉起草那些强类型列,还要迁移刚稳定的起草数据。 + +- **状态词**(ACTIVE、FINISHED、TASK_STATUSES、TASK_DONE、RETRYABLE、OPEN):任务 queued → running → succeeded / partial + / failed / cancelled / interrupted;逐表 pending → running → ok / partial / failed / skipped / cancelled。两种任务、 + 接口、界面都用这一套词。 +- **SerialWriter**:同一个任务的库写入一条一条来,shield 起来——停止任务时不会打断写到一半的库操作;收尾前 settle 等它们 + 写完。 +- **JobRun / JobManager**:进程内一个 asyncio 任务一件事(照 app/scheduler 的写法):start / stop(grace) / wait / running + / shutdown。JobRun.run 把「进队、等轮到自己、执行、按停止的原因收尾、出队」写一次,各种任务只写执行和收尾的那几步。 +- **SourceLanes**:每个数据源一条先来先到的队。起草、剖析都打业务库或模型、都写同一个源的目录:同源串行(先建的先跑), + 不同源并行。任务阻塞地占:建出来是 queued,轮到了才 running。同步起草、导入目录这类同步写操作不阻塞地试占:占不到 + 说明源上正有别的事,接口回 409;占到了,执行期间新建的任务排在它后面。**队里的事是开放的**:运行时不认具体哪几种, + 每种在自己的模块里 register_lane_kind 登记一次(kind → 叫法),进队时只核对登记过——第 2 波的后台探查(introspect) + 登记一行就排进同一条队,不用改这里;拼错的 kind 当场报错,不会悄悄排成一种谁都不认识的事。 +- **close_job / interrupt_on_startup**:收尾(还没结束的表记成某个状态,任务记成某个状态),和服务启动时的整理:排队、 + 进行中的标 interrupted,没结束的表记 cancelled。**不自动续跑**:复制出来的数据目录(检查脚本、验收副本)启动时不能 + 自己去调模型、查业务库,界面让人点「重试」接着做。 + +进程内登记、进程内的队:服务端是单进程部署,多进程部署时要换成库里的锁。 +""" +from __future__ import annotations + +import asyncio +import contextlib +import logging +from collections.abc import AsyncIterator, Awaitable, Callable, Iterator +from typing import Any + +from sqlalchemy import select, update +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.base import SessionLocal, utcnow + +logger = logging.getLogger(__name__) + +#: 任务还没结束的状态(同一个源同一种任务只能有一个)、结束了的状态 +ACTIVE = ("queued", "running") +FINISHED = ("succeeded", "partial", "failed", "cancelled", "interrupted") +#: 逐表的状态,其中算「已结束」的、重试时要带上的、还没结束的 +TASK_STATUSES = ("pending", "running", "ok", "partial", "failed", "skipped", "cancelled") +TASK_DONE = ("ok", "partial", "failed", "skipped", "cancelled") +RETRYABLE = ("failed", "partial", "cancelled") +OPEN = ("pending", "running") + +#: 停止进行中的任务时最多等它收尾多久(秒);关停服务时另给(shutdown 的 grace) +STOP_GRACE_S = 10.0 +SHUTDOWN_GRACE_S = 3.0 + + +class JobFailed(Exception): + """任务整体出错(数据源没了、快照没了、模型用不了……):message 就是任务的 error。""" + + +# ========================================================================== +# 库写入 +# ========================================================================== + + +class SerialWriter: + """一个任务的库写入:一条一条来,每条开一个会话;shield 起来,停止任务时不会打断写到一半的库操作。""" + + def __init__(self) -> None: + self._writes: set[asyncio.Task[Any]] = set() + self._lock = asyncio.Lock() + + async def __call__(self, fn: Callable[..., Awaitable[Any]], *args: Any) -> Any: + async def go() -> Any: + async with self._lock: + async with SessionLocal() as session: + return await fn(session, *args) + + task = asyncio.ensure_future(go()) + self._writes.add(task) + task.add_done_callback(self._writes.discard) + return await asyncio.shield(task) + + async def settle(self) -> None: + """等手上的库写入都写完(停止、出错收尾之前)。""" + while self._writes: + await asyncio.gather(*list(self._writes), return_exceptions=True) + + +# ========================================================================== +# 每个源一条队 +# ========================================================================== + + +#: 同源队里能排的事:{kind: 叫法}。开放的,各种事在自己的模块里登记(register_lane_kind):数据目录登记了 draft 起草任务、 +#: profile 剖析任务、sync_draft 同步起草、import 导入目录(data/catalog_jobs.py) +_LANE_KINDS: dict[str, str] = {} + + +def register_lane_kind(kind: str, label: str) -> str: + """登记一种同源队里的事(重复登记同一个叫法无害,换叫法报错:同一个 kind 只能有一种说法)。返回 kind。""" + if not kind or not label: + raise ValueError("同源队里的事要有 kind 和叫法") + known = _LANE_KINDS.get(kind) + if known is not None and known != label: + raise ValueError(f"同源队里的「{kind}」已经登记成「{known}」,不能再登记成「{label}」") + _LANE_KINDS[kind] = label + return kind + + +def lane_kinds() -> dict[str, str]: + """登记过的各种事:{kind: 叫法}。""" + return dict(_LANE_KINDS) + + +def lane_label(kind: str) -> str: + """一种事的叫法(「起草」「剖析」……);没登记的照原样给 kind。""" + return _LANE_KINDS.get(kind, kind) + + +class LaneTicket: + """在一个源的队里的一个位置。kind 是占着它的是什么(登记过的一种,见 register_lane_kind);job_id 是任务的(同步写 + 操作没有)。轮到它时 granted 置位。""" + + def __init__(self, lanes: SourceLanes, source_id: str, kind: str, job_id: str | None, + loop: asyncio.AbstractEventLoop) -> None: + self._lanes = lanes + self.source_id = source_id + self.kind = kind + self.job_id = job_id + self.loop = loop + self.granted = asyncio.Event() + self.released = False + + async def wait(self) -> None: + """等轮到自己。""" + await self.granted.wait() + + def release(self) -> None: + """出队(轮到了、没轮到都行,重复调用无害):排在后面的第一个接着开始。""" + self._lanes._release(self) + + +class LaneBusy(Exception): + """不阻塞地试占时这个源的队里已经有人:holder 是排在最前面(正在做)的那一个。""" + + def __init__(self, holder: LaneTicket) -> None: + super().__init__(holder.kind) + self.holder = holder + + +class SourceLanes: + """每个数据源一条先来先到的队。""" + + def __init__(self) -> None: + self._queues: dict[str, list[LaneTicket]] = {} + + def _queue(self, source_id: str) -> list[LaneTicket]: + """这个源的队。别的事件循环留下的(测试每个用例一个循环)去掉:那个循环里没人会再出队了。""" + queue = self._queues.get(source_id) or [] + loop = asyncio.get_running_loop() + alive = [t for t in queue if t.loop is loop and not t.released] + if alive: + self._queues[source_id] = alive + alive[0].granted.set() + else: + self._queues.pop(source_id, None) + return alive + + def enter(self, source_id: str, kind: str, job_id: str | None = None) -> LaneTicket: + """排进队尾,马上返回(不等)。排在最前面的立刻 granted;调用方 await ticket.wait() 等轮到自己。kind 要登记过。""" + if kind not in _LANE_KINDS: + raise ValueError(f"同源队里没有登记「{kind}」这种事,先 register_lane_kind") + queue = self._queue(source_id) + ticket = LaneTicket(self, source_id, kind, job_id, asyncio.get_running_loop()) + queue.append(ticket) + self._queues[source_id] = queue + if len(queue) == 1: + ticket.granted.set() + return ticket + + def try_enter(self, source_id: str, kind: str) -> LaneTicket: + """队空着才占(马上就轮到);有人在队里抛 LaneBusy。""" + queue = self._queue(source_id) + if queue: + raise LaneBusy(queue[0]) + return self.enter(source_id, kind) + + def holders(self, source_id: str) -> list[LaneTicket]: + """队里的各位,排在最前面(正在做)的在前。""" + return list(self._queue(source_id)) + + def ahead(self, source_id: str, job_id: str) -> list[str] | None: + """排在这个任务前面的各位是什么(kind,正在做的在前);这个任务不在队里(刚建、后台还没排进来)返回 None。""" + queue = self._queue(source_id) + for i, ticket in enumerate(queue): + if ticket.job_id == job_id: + return [t.kind for t in queue[:i]] + return None + + def _release(self, ticket: LaneTicket) -> None: + if ticket.released: + return + ticket.released = True + queue = self._queues.get(ticket.source_id) + if not queue or ticket not in queue: + return + head = queue[0] is ticket + queue.remove(ticket) + if not queue: + self._queues.pop(ticket.source_id, None) + elif head: + queue[0].granted.set() + + @contextlib.asynccontextmanager + async def hold(self, source_id: str, kind: str, job_id: str | None = None) -> AsyncIterator[LaneTicket]: + """阻塞地占:排队,轮到了再进去,出来时出队。""" + ticket = self.enter(source_id, kind, job_id) + try: + await ticket.wait() + yield ticket + finally: + ticket.release() + + @contextlib.contextmanager + def try_hold(self, source_id: str, kind: str) -> Iterator[LaneTicket]: + """不阻塞地试占(同步起草、导入目录):占不到抛 LaneBusy;占到了,执行期间新建的任务排在后面。""" + ticket = self.try_enter(source_id, kind) + try: + yield ticket + finally: + ticket.release() + + +lanes = SourceLanes() + + +# ========================================================================== +# 收尾、启动时的整理 +# ========================================================================== + + +async def close_job(session: AsyncSession, job_model: Any, task_model: Any, job_id: str, job_status: str, + job_error: str | None, task_status: str, task_error: str | None) -> None: + """收尾:还没结束的表记 task_status(原因 task_error),任务(还没结束的话)记 job_status。提交。""" + now = utcnow() + await session.execute( + update(task_model).where(task_model.job_id == job_id, task_model.status.in_(OPEN)) + .values(status=task_status, error=task_error, finished_at=now).execution_options(synchronize_session=False)) + await session.execute( + update(job_model).where(job_model.id == job_id, job_model.status.in_(ACTIVE)) + .values(status=job_status, error=job_error, finished_at=now).execution_options(synchronize_session=False)) + await session.commit() + + +async def cancel_queued(session: AsyncSession, job_model: Any, task_model: Any, job_id: str) -> bool: + """停止一个排队中的任务:任务和没结束的表直接记 cancelled。已经被后台认领(开始了)返回 False,交给调用方按进行中 + 的停。""" + now = utcnow() + claimed = await session.execute( + update(job_model).where(job_model.id == job_id, job_model.status == "queued") + .values(status="cancelled", finished_at=now).execution_options(synchronize_session=False)) + if claimed.rowcount != 1: + await session.rollback() + return False + await session.execute( + update(task_model).where(task_model.job_id == job_id, task_model.status.in_(OPEN)) + .values(status="cancelled", finished_at=now).execution_options(synchronize_session=False)) + await session.commit() + return True + + +async def claim(session: AsyncSession, job_model: Any, job_id: str) -> bool: + """认领:排队中 → 进行中。开始之前就被停止了返回 False。""" + result = await session.execute( + update(job_model).where(job_model.id == job_id, job_model.status == "queued") + .values(status="running", started_at=utcnow()).execution_options(synchronize_session=False)) + await session.commit() + return result.rowcount == 1 + + +async def interrupt_on_startup(job_model: Any, close: Callable[[AsyncSession, str], Awaitable[None]], *, + running: Callable[[str], bool]) -> int: + """服务启动时:排队、进行中、而且不在这个进程里跑的任务逐个 close(各自记成 interrupted、没结束的表记 cancelled)。 + 不自动续跑。返回整理了几个。""" + async with SessionLocal() as session: + ids = [job_id for job_id in (await session.execute( + select(job_model.id).where(job_model.status.in_(ACTIVE)))).scalars() if not running(job_id)] + for job_id in ids: + await close(session, job_id) + return len(ids) + + +# ========================================================================== +# 在后台执行 +# ========================================================================== + + +class JobRun: + """一个任务在后台的一次执行。子类给 kind(进队时的身份)、model(任务表,进队前读源和状态),写 _run(执行)和 + _stopped / _failed / _crashed(停止、整体出错、意外出错时的收尾)。""" + + kind = "" + model: Any = None + + def __init__(self, job_id: str) -> None: + self.job_id = job_id + self.task: asyncio.Task[None] | None = None + #: 为什么停:user 使用者点了停止,shutdown 服务关停(标中断) + self.stop_reason: str | None = None + self.writer = SerialWriter() + self.ticket: LaneTicket | None = None + + async def _write(self, fn: Callable[..., Awaitable[Any]], *args: Any) -> Any: + """一条库写入(SerialWriter:一条一条来,停止时不被打断)。""" + return await self.writer(fn, *args) + + async def _settle(self) -> None: + await self.writer.settle() + + async def run(self) -> None: + try: + try: + await self._queue() + await self._run() + except asyncio.CancelledError: + current = asyncio.current_task() + if current is not None: + current.uncancel() + await self._stopped() + except JobFailed as e: + await self._failed(str(e)) + except Exception as e: # noqa: BLE001 - 任务整体出错:照实记下,不留在「进行中」 + await self._crashed(e) + finally: + # 收尾写完才出队:排在后面的任务开始时,这个任务已经记成结束了 + if self.ticket is not None: + self.ticket.release() + + def lane_of(self, job: Any) -> str | None: + """这个任务排在哪条队里:缺省是任务的数据源(job.source_id)。不占源的任务(以后的评测:开始时拍目录快照,不挡 + 起草、剖析)覆盖成返回 None,不进队、直接执行。""" + return job.source_id + + async def _queue(self) -> None: + """进这个源的队,等轮到自己。任务不在了、已经不是排队中(开始之前就被停止了)的不进队:_run 认领不到,直接结束。""" + async with SessionLocal() as session: + job = await session.get(self.model, self.job_id) + source_id = self.lane_of(job) if job is not None and job.status == "queued" else None + if source_id is None: + return + self.ticket = lanes.enter(source_id, self.kind, self.job_id) + await self.ticket.wait() + + async def _run(self) -> None: + raise NotImplementedError + + async def _stopped(self) -> None: + raise NotImplementedError + + async def _failed(self, reason: str) -> None: + raise NotImplementedError + + async def _crashed(self, e: Exception) -> None: + raise NotImplementedError + + +class JobManager: + """进程内的一种后台任务:一个 asyncio 任务一件事,停的时候给宽限。factory 按任务编号造一次执行(JobRun);label 是 + 日志里的叫法(「数据目录起草任务」),name 是 asyncio 任务名的前缀。""" + + def __init__(self, factory: Callable[[str], JobRun], *, label: str, name: str) -> None: + self._runs: dict[str, JobRun] = {} + self._factory = factory + self.label = label + self.name = name + + def _alive(self, job_id: str) -> JobRun | None: + run = self._runs.get(job_id) + if run is None or run.task is None or run.task.done(): + return None + try: + if run.task.get_loop() is not asyncio.get_running_loop(): + self._runs.pop(job_id, None) # 别的事件循环留下的(测试每个用例一个循环) + return None + except RuntimeError: + return None + return run + + def start(self, job_id: str) -> None: + """开始在后台执行这个(排队中的)任务:先进这个源的队,轮到了才开始。""" + run = self._factory(job_id) + run.task = asyncio.get_running_loop().create_task(run.run(), name=f"{self.name}-{job_id}") + self._runs[job_id] = run + + def forget(_: asyncio.Task[None]) -> None: + if self._runs.get(job_id) is run: + self._runs.pop(job_id, None) + + run.task.add_done_callback(forget) + + def running(self, job_id: str) -> bool: + return self._alive(job_id) is not None + + async def stop(self, job_id: str, reason: str, *, grace: float = STOP_GRACE_S) -> bool: + """停掉这个进程里在跑(或在排队等)的任务,等它收尾(最多 grace 秒)。不在这个进程里返回 False。""" + run = self._alive(job_id) + if run is None: + return False + if run.stop_reason is None: + run.stop_reason = reason + run.task.cancel() # type: ignore[union-attr] + await asyncio.wait([run.task], timeout=grace) # type: ignore[list-item] + return True + + async def wait(self, job_id: str, timeout: float | None = None) -> None: + """等这个任务做完(测试、调试用)。不在跑直接返回。""" + run = self._alive(job_id) + if run is not None: + await asyncio.wait([run.task], timeout=timeout) # type: ignore[list-item] + + async def shutdown(self, *, grace: float = SHUTDOWN_GRACE_S) -> None: + """关停:停掉进行中、排队等着的任务(标中断),最多等 grace 秒收尾。""" + runs = [run for job_id in list(self._runs) if (run := self._alive(job_id)) is not None] + for run in runs: + if run.stop_reason is None: + run.stop_reason = "shutdown" + run.task.cancel() # type: ignore[union-attr] + if runs: + await asyncio.wait([run.task for run in runs], timeout=grace) # type: ignore[misc] + logger.info("已停止 %d 个%s", len(runs), self.label) + + +__all__ = ["ACTIVE", "FINISHED", "JobFailed", "JobManager", "JobRun", "LaneBusy", "LaneTicket", "OPEN", "RETRYABLE", + "SHUTDOWN_GRACE_S", "STOP_GRACE_S", "SerialWriter", "SourceLanes", "TASK_DONE", "TASK_STATUSES", "cancel_queued", + "claim", "close_job", "interrupt_on_startup", "lane_kinds", "lane_label", "lanes", "register_lane_kind"] diff --git a/backend/app/core/public_url.py b/backend/app/core/public_url.py new file mode 100644 index 00000000..1bd5a2cc --- /dev/null +++ b/backend/app/core/public_url.py @@ -0,0 +1,97 @@ +"""对外访问地址:别人打开 AgentLab 用的那个地址(https://agentlab.example.com),全站只有这一个说法(04 细案 B4)。 + +用到它的地方:通知和定时任务 Webhook 里的链接、单点登录的回调地址、非 GET 请求的来源核对(auth/middleware.py); +之后的分享链接、接口说明也从这里取。服务端不知道自己在反向代理后面叫什么,只能由人告诉它。 + +取值(public_base_url): +- 环境变量 AGENTLAB_PUBLIC_URL 优先:部署时定死,设置页上显示「由环境变量指定」、不能改(同登录方式的 env_locked); +- 没设时取设置里 notify 一组的 public_base_url:沿用这个键,界面上叫「对外访问地址」,在「设置 → 通用」里改; +- 都没有是 None:通知里不附链接,单点登录的回调和来源核对按请求自己的 Host 认。 + +以前两处各管各的:环境变量只管回调和来源核对、设置只管通知里的链接——只配了环境变量的部署,通知里不带链接;只在设置里 +填了的,回调又按请求的 Host 拼。违背「一种说法」。 + +设置的值缓存在进程里:中间件按请求核对来源,不能每次查库。启动时读一次(main._platform_startup),改设置的接口写完库就 +remember;手里本来就有会话的异步路径(通知、定时任务 Webhook)用 refresh 取,顺手按库里的值对齐缓存。和登录设置的缓存一样 +以单进程部署为前提。 + +这个模块不在顶层导入 core/config:config 的校验器要用这里的 url_problem,顶层互相导入会在启动时绕成环。 +""" +from __future__ import annotations + +from typing import Any +from urllib.parse import urlsplit + +#: 设置里存它的那一组、那一项(api/settings.py 的 NOTIFY_GROUP 就是这一组) +SETTING_GROUP = "notify" +SETTING_KEY = "public_base_url" +#: 环境变量的名字(core/config.Settings.public_url 按 AGENTLAB_ 前缀读它) +ENV_NAME = "AGENTLAB_PUBLIC_URL" + +_stored: str | None = None + + +def url_problem(value: str) -> str | None: + """地址的形状有什么问题:None 是没问题;"form" 不是 http:// 或 https:// 开头的完整地址;"suffix" 带了 ? 或 # + (链接要接在地址后面)。设置接口按它出中文的说法,环境变量按它在启动时报错。可以带路径(反向代理挂在子路径下)。""" + text = value.strip() + try: + parts = urlsplit(text) + except ValueError: + return "form" + if parts.scheme not in ("http", "https") or not parts.hostname or " " in text: + return "form" + if parts.query or parts.fragment: + return "suffix" + return None + + +def normalize(value: Any) -> str | None: + """存着的(或刚填的)值 → 去掉首尾空白和末尾的 /;空的、形状不对的当没填(None)。末尾带 / 的地址拼链接会多出一个斜杠。""" + if not isinstance(value, str) or not value.strip() or url_problem(value): + return None + return value.strip().rstrip("/") + + +def env_url() -> str | None: + """环境变量给的地址;没设为 None。形状在启动时由 config 的校验器查过、末尾的 / 去掉了。""" + from app.core.config import settings + + return settings.public_url or None + + +def env_locked() -> bool: + """环境变量设了:设置里的那一项不生效、不能改。""" + return env_url() is not None + + +def public_base_url() -> str | None: + """对外访问地址:环境变量优先,其次设置(进程内缓存),都没有是 None。不查库,中间件里也能调。""" + return env_url() or _stored + + +def remember(value: Any) -> None: + """改设置的接口写完库之后调:缓存换成新存的值(形状不对、空的当没填)。""" + global _stored + _stored = normalize(value) + + +async def refresh(session: Any = None) -> str | None: + """按库里存的设置对齐缓存,交回对外访问地址。启动时、手里有会话的异步路径用。""" + from app.db.base import SessionLocal + from app.db.models import Setting + + if session is None: + async with SessionLocal() as own: + row = await own.get(Setting, SETTING_GROUP) + else: + row = await session.get(Setting, SETTING_GROUP) + value = row.value if row is not None else None + remember(value.get(SETTING_KEY) if isinstance(value, dict) else None) + return public_base_url() + + +def origin_of(url: str) -> str: + """地址里的「协议://主机[:端口]」:来源核对按它比。""" + parts = urlsplit(url) + return f"{parts.scheme}://{parts.netloc}" diff --git a/backend/app/data/catalog.py b/backend/app/data/catalog.py index 6ca00287..25ac4c8c 100644 --- a/backend/app/data/catalog.py +++ b/backend/app/data/catalog.py @@ -420,6 +420,18 @@ async def read_catalog(session: AsyncSession, source_id: str) -> dict[str, Catal return {row.table_name: _entry(row) for row in rows} +async def read_catalogs(session: AsyncSession, source_ids: Iterable[str]) -> dict[str, dict[str, CatalogEntry]]: + """几个数据源的目录一次查询读出:{源 id: {表名: CatalogEntry}},没有目录的源不出现。给模型、给自查读目录的地方经 + data/catalog_view.py 调它,不直接查 catalog_notes。""" + ids = sorted({i for i in source_ids if i}) + if not ids: + return {} + out: dict[str, dict[str, CatalogEntry]] = {} + for row in (await session.execute(select(CatalogNote).where(CatalogNote.source_id.in_(ids)))).scalars(): + out.setdefault(row.source_id, {})[row.table_name] = _entry(row) + return out + + async def write_entry(session: AsyncSession, source_id: str, table: str, notes: dict[str, Any], *, if_version: int, actor: str | None, commit: bool = True) -> CatalogEntry: """整份写入一张表的目录(乐观锁),提交事务,返回写入后的目录。 @@ -567,18 +579,48 @@ def _undated(note: str) -> str: return _PROFILE_NOTE_DAY.sub("数据剖析", note, count=1) +#: 剖析备注改过的说法:(升级前的写法, 现在的写法)。大表看的是按存储顺序的前 N 行(多半是最老的数据)、关系取的是 +#: 数据库先给出的那几个键值,都不是随机抽样,说法改成照实的(data/catalog_profile.py 的 _relation_note、码值的 +#: 「统计范围」、FILTER_SAMPLED)。只换这几处固定的句式,换完和现在的写法一字不差才算同一句 +_LEGACY_PROFILE_WORDING = ( + (re.compile(r"抽样覆盖率 (?=\S+?,可能不是这条关系)"), "覆盖率 "), + (re.compile(r"子表抽样 (?=\d+ 个不同键值)"), "子表取了 "), + (re.compile(r"(只看前 (?=\d+ 行))"), "(按存储顺序只看前 "), + (re.compile(r"按前 (?=[\d,]+ 行统计)"), "按存储顺序取前 "), +) +#: 起草时核对有效行过滤写进备注的那句(catalog_profile.FILTER_SAMPLED / FILTER_SAMPLED_EDGE):接在模型给的依据后面, +#: 整段不以「数据剖析:」开头,只换这一句 +_LEGACY_FILTER_WORDING = re.compile(r"剖析核对:按前 (?=[\d,]+ 行统计)") + + +def _current_wording(note: str) -> str: + """剖析备注里升级前的说法换成现在的写法(_LEGACY_PROFILE_WORDING)。只用来比较,不改写目录里存着的备注。""" + for pattern, now in _LEGACY_PROFILE_WORDING: + note = pattern.sub(now, note) + return note + + def same_profile_note(a: Any, b: Any) -> bool: """两段备注是否相同;数据剖析写的备注只差开头的日期(升级前每次都写当天的日期),或者一段带日期、一段是不带 - 日期的新写法,也算相同。 + 日期的新写法,或者只差改过的几处说法(_LEGACY_PROFILE_WORDING:「抽样覆盖率」→「覆盖率」之类),也算相同。 隔天重跑、数据没变,结论一字不差、只有日期变了:要是这也算「变了」,目录就升一个版本,发布时记下的目录版本 - 对不上,之后每次正式运行都提醒「目录有变化」。升级后第一次重新剖析也是一样:结论没变,只是新写法不带日期。 + 对不上,之后每次正式运行都提醒「目录有变化」。升级后第一次重新剖析也是一样:结论没变,只是新写法不带日期、 + 措辞改了。算相同时保留旧的那段(目录不动),旧说法要等结论真的变了、重写备注时才换掉。 + + 起草写的备注(有效行过滤)里接着的剖析核对那句同理,只认那一句的旧说法(_LEGACY_FILTER_WORDING)。 """ if a == b: return True - if not (is_profile_note(a) and is_profile_note(b)): + if not (isinstance(a, str) and isinstance(b, str)): return False - return _undated(a) == _undated(b) + profile_a, profile_b = is_profile_note(a), is_profile_note(b) + if profile_a and profile_b: + return _current_wording(_undated(a)) == _current_wording(_undated(b)) + if profile_a or profile_b: + return False + return (_LEGACY_FILTER_WORDING.sub("剖析核对:按存储顺序取前 ", a) + == _LEGACY_FILTER_WORDING.sub("剖析核对:按存储顺序取前 ", b)) def _same_core(a: Mapping[str, Any], b: Mapping[str, Any]) -> bool: @@ -592,6 +634,14 @@ def visible_notes(notes: Mapping[str, Any] | None) -> dict[str, Any]: return _assemble({k: v for k, v in _slots(notes).items() if v.get("status") != "rejected"}) +def restrict_notes(notes: Mapping[str, Any] | None, statuses: Iterable[str]) -> dict[str, Any]: + """只留状态在 statuses 里的项(表级项、列级项、关系都按项算);驳回的项原样留着——给人、给模型看的照旧经 + visible_notes 去掉,冻结快照和 SQL 检查要靠驳回的关系编号认出不该按外键、命名现推回来的关系(frozen_catalog)。 + 目录视图「只用已核实、已确认的项」用它(data/catalog_view.py)。""" + keep = set(statuses) | {"rejected"} + return _assemble({k: v for k, v in _slots(notes).items() if v.get("status") in keep}) + + def status_counts(notes: Mapping[str, Any] | None) -> dict[str, int]: """各状态的项数(表级项、列级项、关系都算),四种状态都有键。""" counts = dict.fromkeys(ITEM_STATUSES, 0) @@ -1358,14 +1408,17 @@ _DATE_WORDS = frozenset({"date", "time", "day", "dt", "ts", "timestamp", "dateti _TEXT_TYPES = ("CHAR", "TEXT", "STRING", "CLOB") -def is_date_column(name: str, type_name: str | None) -> bool: +def is_date_column(name: str, type_name: str | None, *, enum_values: Any = None) -> bool: """日期类列:类型是日期、时间戳(DATE、DATETIME、TIMESTAMP……);或者类型是文字(或者不知道类型)、名字像日期 - (SQLite 的日期常存成文字)。只有时刻的 TIME、整数存的时间戳不算:取值不是日期。 + (SQLite 的日期常存成文字)。只有时刻的 TIME、整数存的时间戳不算:取值不是日期。有取值列表的(MySQL 的 ENUM / + SET、探查记下 enum_values 的 PostgreSQL 枚举)不算:PG 的枚举编译出来是类型名本身,update_reason 里也有「DATE」。 数据剖析(表里只有一个日期类列时提议它作业务日期)和 SQL 检查(按业务日期以外的时间列统计)共用这一个判断。 以前两边各写一套:剖析认的「唯一日期列」,SQL 检查可能认为还有别的时间列,反过来也一样,两边对不上。 """ - t = (type_name or "").upper() + t = _type_words(type_name) + if enum_values or _MYSQL_ENUM.match(t) or "INTERVAL" in t: + return False if "DATE" in t or "TIMESTAMP" in t: return True if t and not any(k in t for k in _TEXT_TYPES): @@ -1400,11 +1453,41 @@ def _name_index(tables: Mapping[str, Any]) -> dict[tuple[str, str], set[str]]: return index +#: 类型写法里带引号的部分(ENUM('int','date') 的取值):认类型时去掉,取值里的词不能让一列变成数字或时间 +_QUOTED_IN_TYPE = re.compile(r"'(?:[^']|'')*'") + + +def _type_words(type_name: str | None) -> str: + """认类型用的写法:大写、去掉引号里的取值。探查存的是按方言编译的完整写法(ENUM('paid','refund')、 + VARCHAR2(20 CHAR)、TIMESTAMP WITH TIME ZONE),不再是 SQLAlchemy 的通用写法。""" + return _QUOTED_IN_TYPE.sub("''", (type_name or "").upper()).strip() + + +#: MySQL 的 ENUM / SET 编译出来的写法:ENUM('a','b')、SET('x','y'),只认带括号的。PostgreSQL 的枚举编译出来是类型名本身, +#: setup_status、enum_kind 这种以 set、enum 开头的名字不能按前缀认成 MySQL 的 SET / ENUM——PG 的枚举一律靠探查记下的 +#: enum_values 认(column_family) +_MYSQL_ENUM = re.compile(r"^(ENUM|SET)\s*\(") + + +def _mysql_enum_kind(type_name: str | None) -> str | None: + """类型写法是 MySQL 的 ENUM(…) / SET(…) 时返回 "ENUM" / "SET",否则 None。""" + m = _MYSQL_ENUM.match(_type_words(type_name)) + return m.group(1) if m else None + + def _type_family(type_name: str | None) -> str | None: - """列类型的大类:数字、文字、时间;认不出来返回 None(不据此排除)。""" - t = (type_name or "").upper() + """列类型的大类:数字、文字、时间;认不出来返回 None(不据此排除)。 + + 认的是按方言编译的完整写法:MySQL 的 ENUM / SET 是文字(取值列表里的词不算),Oracle 的 INTERVAL DAY TO SECOND + 是时长、不算数字(以前按子串「INT」认成了数字)。PostgreSQL 的枚举编译出来是类型名本身,只看类型认不出来, + 要看列上的取值列表(column_family)。""" + t = _type_words(type_name) if not t: return None + if _MYSQL_ENUM.match(t): + return "text" + if "INTERVAL" in t: + return None if any(k in t for k in ("CHAR", "TEXT", "CLOB", "STRING", "UUID")): return "text" if any(k in t for k in ("INT", "NUMBER", "NUMERIC", "DECIMAL", "SERIAL", "REAL", "FLOAT", "DOUBLE")): @@ -1414,6 +1497,16 @@ def _type_family(type_name: str | None) -> str | None: return None +def column_family(col: Mapping[str, Any] | None) -> str | None: + """表结构里一列的类型大类:有取值列表(探查记下的 enum_values:MySQL 的 ENUM / SET、PostgreSQL 的枚举)的是 + 文字,其余按类型写法认(_type_family)。""" + if not isinstance(col, Mapping): + return None + if col.get("enum_values"): + return "text" + return _type_family(col.get("type")) + + def _id_stem(column: str) -> str | None: m = _SNAKE_ID.match(column) or _CAMEL_ID.match(column) return m.group("stem") if m else None @@ -1469,8 +1562,8 @@ def infer_name_relations(schema_cache: Mapping[str, Any] | None, table: str, *, pk = list(target_meta.get("primary_key") or []) if len(pk) != 1: continue - pk_type = next((c.get("type") for c in target_meta.get("columns") or [] if c.get("name") == pk[0]), None) - mine, theirs = _type_family(col.get("type")), _type_family(pk_type) + pk_col = next((c for c in target_meta.get("columns") or [] if c.get("name") == pk[0]), None) + mine, theirs = column_family(col), column_family(pk_col) if mine and theirs and mine != theirs: continue out.append(_relation(table, [name], target, pk, "name", cardinality=_cardinality(meta, [name]))) @@ -2408,15 +2501,18 @@ async def query_snapshot_tables(session: AsyncSession, source: Any) -> list[tupl 使用次数(table_usage)和「这张表被哪些运行用过」(data/catalog_review.py)共用这一份取法:数据源查询工具每次成功 查询都存一份查询快照工件(tools/datasource.py),工件引用表 artifacts 里按 (内容哈希, 运行) 各记一行;只收属于某次运行 - (runs 里有这次运行)、属于这个源的行,工具库里的试用(不属于任何运行)不算。 + (runs 里有这次运行)、属于这个源的行,工具库里的试用(不属于任何运行)不算;系统派生的运行(评测、口径对账、召回, + engine/origin.counted_runs)也不算——一跑评测,常用表的排序、起草的默认范围、审阅队列的权重就跟着被评测题刷高了。 表名在查询当时就记进了引用的 meta(query_snapshot_meta),这里只读 artifacts 一张表;升级前的老引用没有 meta,退回读 工件内容里的源名和 SQL,读过的按工件 id 缓存(内容永不改变)。SQL 里写的表名去掉 schema 前缀后按 name_key(只把 ASCII 字母转小写,和 SQLite 比较标识符的口径一致)对到表结构上,对不上的不计。 """ + from app.engine.origin import counted_runs + rows = (await session.execute( select(Artifact.id, Artifact.meta, Artifact.run_id).join(Run, Run.id == Artifact.run_id) - .where(Artifact.kind == QUERY_SNAPSHOT_KIND) + .where(Artifact.kind == QUERY_SNAPSHOT_KIND, counted_runs()) )).all() tables = (getattr(source, "schema_cache", None) or {}).get("tables") or {} lookup: dict[str, str] = {} @@ -2505,8 +2601,9 @@ __all__ = [ "fk_relations", "frozen_catalog", "infer_name_relations", "initial_status", "is_date_column", "is_profile_note", "join_paths", "make_item", "merge_notes", "name_words", "now_iso", "parse_path", "plan_patch", "profile_relation_holds", - "query_snapshot_meta", "read_catalog", "read_entry", "relation_graph", "relation_id", "render_table_index", - "render_table_notes", "resolve_draft_model", "resolve_table_name", "review_entry", "review_item", "same_notes", + "query_snapshot_meta", "read_catalog", "read_catalogs", "read_entry", "relation_graph", "relation_id", "render_table_index", + "render_table_notes", "resolve_draft_model", "resolve_table_name", "restrict_notes", "review_entry", "review_item", + "same_notes", "same_profile_note", "save_human_edit", "save_patch", "status_counts", "structure_relation_ids", "system_notes_source", "table_usage", "unreviewed_status", "validate_notes", "visible_notes", "write_entry", ] diff --git a/backend/app/data/catalog_job_runtime.py b/backend/app/data/catalog_job_runtime.py index 970159a4..664c5c51 100644 --- a/backend/app/data/catalog_job_runtime.py +++ b/backend/app/data/catalog_job_runtime.py @@ -1,400 +1,13 @@ -"""数据目录后台任务的共用运行时:起草(data/catalog_jobs.py)和剖析(data/catalog_profile_jobs.py)各自建表、各自的步骤, -生命周期是同一套。 +"""后台任务运行时的老路径:已上移到 app/core/job_runtime.py(docs/design/next-round/00-总方案.md X5)。 -**为什么抽出来。** 剖析以前是一次同步长请求,和起草改成后台任务之前是同一类问题:关掉页面结果就没了、不能停、不留痕。 -两边要的是同一样东西——进程内跑、能停、重启标中断、写库不被停止打断、同源排队——各写一份,迟早一边修了另一边没修。 -共用的是生命周期代码,不是表:两种任务的进度完全不同(起草按段、块、模型用量,剖析按检查项、查询),合成一张 -「kind + JSON」的通用任务表要丢掉起草那些强类型列,还要迁移刚稳定的起草数据。 - -- **状态词**(ACTIVE、FINISHED、TASK_STATUSES、TASK_DONE、RETRYABLE、OPEN):任务 queued → running → succeeded / partial - / failed / cancelled / interrupted;逐表 pending → running → ok / partial / failed / skipped / cancelled。两种任务、 - 接口、界面都用这一套词。 -- **SerialWriter**:同一个任务的库写入一条一条来,shield 起来——停止任务时不会打断写到一半的库操作;收尾前 settle 等它们 - 写完。 -- **JobRun / JobManager**:进程内一个 asyncio 任务一件事(照 app/scheduler 的写法):start / stop(grace) / wait / running - / shutdown。JobRun.run 把「进队、等轮到自己、执行、按停止的原因收尾、出队」写一次,各种任务只写执行和收尾的那几步。 -- **SourceLanes**:每个数据源一条先来先到的队。起草、剖析都打业务库或模型、都写同一个源的目录:同源串行(先建的先跑), - 不同源并行。任务阻塞地占:建出来是 queued,轮到了才 running。同步起草、导入目录这类同步写操作不阻塞地试占:占不到 - 说明源上正有别的事,接口回 409;占到了,执行期间新建的任务排在它后面。 -- **close_job / interrupt_on_startup**:收尾(还没结束的表记成某个状态,任务记成某个状态),和服务启动时的整理:排队、 - 进行中的标 interrupted,没结束的表记 cancelled。**不自动续跑**:复制出来的数据目录(检查脚本、验收副本)启动时不能 - 自己去调模型、查业务库,界面让人点「重试」接着做。 - -进程内登记、进程内的队:服务端是单进程部署,多进程部署时要换成库里的锁。 +这里把那个模块原样放在老名字下(sys.modules 指向同一个模块对象):起草、剖析和别处的 `from app.data import +catalog_job_runtime as runtime`、`from app.data.catalog_job_runtime import …` 拿到的是同一批名字、同一条队(lanes), +测试里改它的属性也就是改那个模块。新代码从 app.core.job_runtime 导入。 """ from __future__ import annotations -import asyncio -import contextlib -import logging -from collections.abc import AsyncIterator, Awaitable, Callable, Iterator -from typing import Any +import sys -from sqlalchemy import select, update -from sqlalchemy.ext.asyncio import AsyncSession +from app.core import job_runtime -from app.db.base import SessionLocal, utcnow - -logger = logging.getLogger(__name__) - -#: 任务还没结束的状态(同一个源同一种任务只能有一个)、结束了的状态 -ACTIVE = ("queued", "running") -FINISHED = ("succeeded", "partial", "failed", "cancelled", "interrupted") -#: 逐表的状态,其中算「已结束」的、重试时要带上的、还没结束的 -TASK_STATUSES = ("pending", "running", "ok", "partial", "failed", "skipped", "cancelled") -TASK_DONE = ("ok", "partial", "failed", "skipped", "cancelled") -RETRYABLE = ("failed", "partial", "cancelled") -OPEN = ("pending", "running") - -#: 停止进行中的任务时最多等它收尾多久(秒);关停服务时另给(shutdown 的 grace) -STOP_GRACE_S = 10.0 -SHUTDOWN_GRACE_S = 3.0 - - -class JobFailed(Exception): - """任务整体出错(数据源没了、快照没了、模型用不了……):message 就是任务的 error。""" - - -# ========================================================================== -# 库写入 -# ========================================================================== - - -class SerialWriter: - """一个任务的库写入:一条一条来,每条开一个会话;shield 起来,停止任务时不会打断写到一半的库操作。""" - - def __init__(self) -> None: - self._writes: set[asyncio.Task[Any]] = set() - self._lock = asyncio.Lock() - - async def __call__(self, fn: Callable[..., Awaitable[Any]], *args: Any) -> Any: - async def go() -> Any: - async with self._lock: - async with SessionLocal() as session: - return await fn(session, *args) - - task = asyncio.ensure_future(go()) - self._writes.add(task) - task.add_done_callback(self._writes.discard) - return await asyncio.shield(task) - - async def settle(self) -> None: - """等手上的库写入都写完(停止、出错收尾之前)。""" - while self._writes: - await asyncio.gather(*list(self._writes), return_exceptions=True) - - -# ========================================================================== -# 每个源一条队 -# ========================================================================== - - -class LaneTicket: - """在一个源的队里的一个位置。kind 是占着它的是什么:draft 起草任务、profile 剖析任务、sync_draft 同步起草、 - import 导入目录;job_id 是任务的(同步写操作没有)。轮到它时 granted 置位。""" - - def __init__(self, lanes: SourceLanes, source_id: str, kind: str, job_id: str | None, - loop: asyncio.AbstractEventLoop) -> None: - self._lanes = lanes - self.source_id = source_id - self.kind = kind - self.job_id = job_id - self.loop = loop - self.granted = asyncio.Event() - self.released = False - - async def wait(self) -> None: - """等轮到自己。""" - await self.granted.wait() - - def release(self) -> None: - """出队(轮到了、没轮到都行,重复调用无害):排在后面的第一个接着开始。""" - self._lanes._release(self) - - -class LaneBusy(Exception): - """不阻塞地试占时这个源的队里已经有人:holder 是排在最前面(正在做)的那一个。""" - - def __init__(self, holder: LaneTicket) -> None: - super().__init__(holder.kind) - self.holder = holder - - -class SourceLanes: - """每个数据源一条先来先到的队。""" - - def __init__(self) -> None: - self._queues: dict[str, list[LaneTicket]] = {} - - def _queue(self, source_id: str) -> list[LaneTicket]: - """这个源的队。别的事件循环留下的(测试每个用例一个循环)去掉:那个循环里没人会再出队了。""" - queue = self._queues.get(source_id) or [] - loop = asyncio.get_running_loop() - alive = [t for t in queue if t.loop is loop and not t.released] - if alive: - self._queues[source_id] = alive - alive[0].granted.set() - else: - self._queues.pop(source_id, None) - return alive - - def enter(self, source_id: str, kind: str, job_id: str | None = None) -> LaneTicket: - """排进队尾,马上返回(不等)。排在最前面的立刻 granted;调用方 await ticket.wait() 等轮到自己。""" - queue = self._queue(source_id) - ticket = LaneTicket(self, source_id, kind, job_id, asyncio.get_running_loop()) - queue.append(ticket) - self._queues[source_id] = queue - if len(queue) == 1: - ticket.granted.set() - return ticket - - def try_enter(self, source_id: str, kind: str) -> LaneTicket: - """队空着才占(马上就轮到);有人在队里抛 LaneBusy。""" - queue = self._queue(source_id) - if queue: - raise LaneBusy(queue[0]) - return self.enter(source_id, kind) - - def holders(self, source_id: str) -> list[LaneTicket]: - """队里的各位,排在最前面(正在做)的在前。""" - return list(self._queue(source_id)) - - def ahead(self, source_id: str, job_id: str) -> list[str] | None: - """排在这个任务前面的各位是什么(kind,正在做的在前);这个任务不在队里(刚建、后台还没排进来)返回 None。""" - queue = self._queue(source_id) - for i, ticket in enumerate(queue): - if ticket.job_id == job_id: - return [t.kind for t in queue[:i]] - return None - - def _release(self, ticket: LaneTicket) -> None: - if ticket.released: - return - ticket.released = True - queue = self._queues.get(ticket.source_id) - if not queue or ticket not in queue: - return - head = queue[0] is ticket - queue.remove(ticket) - if not queue: - self._queues.pop(ticket.source_id, None) - elif head: - queue[0].granted.set() - - @contextlib.asynccontextmanager - async def hold(self, source_id: str, kind: str, job_id: str | None = None) -> AsyncIterator[LaneTicket]: - """阻塞地占:排队,轮到了再进去,出来时出队。""" - ticket = self.enter(source_id, kind, job_id) - try: - await ticket.wait() - yield ticket - finally: - ticket.release() - - @contextlib.contextmanager - def try_hold(self, source_id: str, kind: str) -> Iterator[LaneTicket]: - """不阻塞地试占(同步起草、导入目录):占不到抛 LaneBusy;占到了,执行期间新建的任务排在后面。""" - ticket = self.try_enter(source_id, kind) - try: - yield ticket - finally: - ticket.release() - - -lanes = SourceLanes() - - -# ========================================================================== -# 收尾、启动时的整理 -# ========================================================================== - - -async def close_job(session: AsyncSession, job_model: Any, task_model: Any, job_id: str, job_status: str, - job_error: str | None, task_status: str, task_error: str | None) -> None: - """收尾:还没结束的表记 task_status(原因 task_error),任务(还没结束的话)记 job_status。提交。""" - now = utcnow() - await session.execute( - update(task_model).where(task_model.job_id == job_id, task_model.status.in_(OPEN)) - .values(status=task_status, error=task_error, finished_at=now).execution_options(synchronize_session=False)) - await session.execute( - update(job_model).where(job_model.id == job_id, job_model.status.in_(ACTIVE)) - .values(status=job_status, error=job_error, finished_at=now).execution_options(synchronize_session=False)) - await session.commit() - - -async def cancel_queued(session: AsyncSession, job_model: Any, task_model: Any, job_id: str) -> bool: - """停止一个排队中的任务:任务和没结束的表直接记 cancelled。已经被后台认领(开始了)返回 False,交给调用方按进行中 - 的停。""" - now = utcnow() - claimed = await session.execute( - update(job_model).where(job_model.id == job_id, job_model.status == "queued") - .values(status="cancelled", finished_at=now).execution_options(synchronize_session=False)) - if claimed.rowcount != 1: - await session.rollback() - return False - await session.execute( - update(task_model).where(task_model.job_id == job_id, task_model.status.in_(OPEN)) - .values(status="cancelled", finished_at=now).execution_options(synchronize_session=False)) - await session.commit() - return True - - -async def claim(session: AsyncSession, job_model: Any, job_id: str) -> bool: - """认领:排队中 → 进行中。开始之前就被停止了返回 False。""" - result = await session.execute( - update(job_model).where(job_model.id == job_id, job_model.status == "queued") - .values(status="running", started_at=utcnow()).execution_options(synchronize_session=False)) - await session.commit() - return result.rowcount == 1 - - -async def interrupt_on_startup(job_model: Any, close: Callable[[AsyncSession, str], Awaitable[None]], *, - running: Callable[[str], bool]) -> int: - """服务启动时:排队、进行中、而且不在这个进程里跑的任务逐个 close(各自记成 interrupted、没结束的表记 cancelled)。 - 不自动续跑。返回整理了几个。""" - async with SessionLocal() as session: - ids = [job_id for job_id in (await session.execute( - select(job_model.id).where(job_model.status.in_(ACTIVE)))).scalars() if not running(job_id)] - for job_id in ids: - await close(session, job_id) - return len(ids) - - -# ========================================================================== -# 在后台执行 -# ========================================================================== - - -class JobRun: - """一个任务在后台的一次执行。子类给 kind(进队时的身份)、model(任务表,进队前读源和状态),写 _run(执行)和 - _stopped / _failed / _crashed(停止、整体出错、意外出错时的收尾)。""" - - kind = "" - model: Any = None - - def __init__(self, job_id: str) -> None: - self.job_id = job_id - self.task: asyncio.Task[None] | None = None - #: 为什么停:user 使用者点了停止,shutdown 服务关停(标中断) - self.stop_reason: str | None = None - self.writer = SerialWriter() - self.ticket: LaneTicket | None = None - - async def _write(self, fn: Callable[..., Awaitable[Any]], *args: Any) -> Any: - """一条库写入(SerialWriter:一条一条来,停止时不被打断)。""" - return await self.writer(fn, *args) - - async def _settle(self) -> None: - await self.writer.settle() - - async def run(self) -> None: - try: - try: - await self._queue() - await self._run() - except asyncio.CancelledError: - current = asyncio.current_task() - if current is not None: - current.uncancel() - await self._stopped() - except JobFailed as e: - await self._failed(str(e)) - except Exception as e: # noqa: BLE001 - 任务整体出错:照实记下,不留在「进行中」 - await self._crashed(e) - finally: - # 收尾写完才出队:排在后面的任务开始时,这个任务已经记成结束了 - if self.ticket is not None: - self.ticket.release() - - async def _queue(self) -> None: - """进这个源的队,等轮到自己。任务不在了、已经不是排队中(开始之前就被停止了)的不进队:_run 认领不到,直接结束。""" - async with SessionLocal() as session: - job = await session.get(self.model, self.job_id) - source_id = job.source_id if job is not None and job.status == "queued" else None - if source_id is None: - return - self.ticket = lanes.enter(source_id, self.kind, self.job_id) - await self.ticket.wait() - - async def _run(self) -> None: - raise NotImplementedError - - async def _stopped(self) -> None: - raise NotImplementedError - - async def _failed(self, reason: str) -> None: - raise NotImplementedError - - async def _crashed(self, e: Exception) -> None: - raise NotImplementedError - - -class JobManager: - """进程内的一种后台任务:一个 asyncio 任务一件事,停的时候给宽限。factory 按任务编号造一次执行(JobRun);label 是 - 日志里的叫法(「数据目录起草任务」),name 是 asyncio 任务名的前缀。""" - - def __init__(self, factory: Callable[[str], JobRun], *, label: str, name: str) -> None: - self._runs: dict[str, JobRun] = {} - self._factory = factory - self.label = label - self.name = name - - def _alive(self, job_id: str) -> JobRun | None: - run = self._runs.get(job_id) - if run is None or run.task is None or run.task.done(): - return None - try: - if run.task.get_loop() is not asyncio.get_running_loop(): - self._runs.pop(job_id, None) # 别的事件循环留下的(测试每个用例一个循环) - return None - except RuntimeError: - return None - return run - - def start(self, job_id: str) -> None: - """开始在后台执行这个(排队中的)任务:先进这个源的队,轮到了才开始。""" - run = self._factory(job_id) - run.task = asyncio.get_running_loop().create_task(run.run(), name=f"{self.name}-{job_id}") - self._runs[job_id] = run - - def forget(_: asyncio.Task[None]) -> None: - if self._runs.get(job_id) is run: - self._runs.pop(job_id, None) - - run.task.add_done_callback(forget) - - def running(self, job_id: str) -> bool: - return self._alive(job_id) is not None - - async def stop(self, job_id: str, reason: str, *, grace: float = STOP_GRACE_S) -> bool: - """停掉这个进程里在跑(或在排队等)的任务,等它收尾(最多 grace 秒)。不在这个进程里返回 False。""" - run = self._alive(job_id) - if run is None: - return False - if run.stop_reason is None: - run.stop_reason = reason - run.task.cancel() # type: ignore[union-attr] - await asyncio.wait([run.task], timeout=grace) # type: ignore[list-item] - return True - - async def wait(self, job_id: str, timeout: float | None = None) -> None: - """等这个任务做完(测试、调试用)。不在跑直接返回。""" - run = self._alive(job_id) - if run is not None: - await asyncio.wait([run.task], timeout=timeout) # type: ignore[list-item] - - async def shutdown(self, *, grace: float = SHUTDOWN_GRACE_S) -> None: - """关停:停掉进行中、排队等着的任务(标中断),最多等 grace 秒收尾。""" - runs = [run for job_id in list(self._runs) if (run := self._alive(job_id)) is not None] - for run in runs: - if run.stop_reason is None: - run.stop_reason = "shutdown" - run.task.cancel() # type: ignore[union-attr] - if runs: - await asyncio.wait([run.task for run in runs], timeout=grace) # type: ignore[misc] - logger.info("已停止 %d 个%s", len(runs), self.label) - - -__all__ = ["ACTIVE", "FINISHED", "JobFailed", "JobManager", "JobRun", "LaneBusy", "LaneTicket", "OPEN", "RETRYABLE", - "SHUTDOWN_GRACE_S", "STOP_GRACE_S", "SerialWriter", "SourceLanes", "TASK_DONE", "TASK_STATUSES", "cancel_queued", - "claim", "close_job", "interrupt_on_startup", "lanes"] +sys.modules[__name__] = job_runtime diff --git a/backend/app/data/catalog_jobs.py b/backend/app/data/catalog_jobs.py index 19c009d9..557978c4 100644 --- a/backend/app/data/catalog_jobs.py +++ b/backend/app/data/catalog_jobs.py @@ -55,6 +55,11 @@ logger = logging.getLogger(__name__) _OPEN = runtime.OPEN +#: 起草这边占同源队的几种事(叫法登记在运行时里,见 core/job_runtime.register_lane_kind):后台起草任务、同步起草、导入目录 +runtime.register_lane_kind("draft", "起草") +runtime.register_lane_kind("sync_draft", "同步起草") +runtime.register_lane_kind("import", "导入目录") + #: 一个任务最多几张表;起草要求、本次说明最多几个字 MAX_TABLES = 500 GUIDANCE_MAX = 2000 diff --git a/backend/app/data/catalog_profile.py b/backend/app/data/catalog_profile.py index 8eee4488..17b59a39 100644 --- a/backend/app/data/catalog_profile.py +++ b/backend/app/data/catalog_profile.py @@ -41,9 +41,9 @@ import re import time from collections.abc import Awaitable, Callable from dataclasses import asdict, dataclass, field -from datetime import datetime, timezone +from datetime import date as _date, datetime, timezone from decimal import Decimal -from typing import Any +from typing import Any, NamedTuple from sqlalchemy import select, update from sqlalchemy.dialects import mysql, oracle, postgresql, sqlite @@ -99,7 +99,7 @@ _NUMBER_FIELDS: dict[str, tuple[str, float, float, bool]] = { "table_max_queries": ("每张表最多查询数", 1, 200, True), "table_max_s": ("每张表时长上限(秒)", 10, 600, False), "query_timeout_s": ("单条查询时限(秒)", 1, 60, False), - "sample_size": ("抽样键值数", 10, 10_000, True), + "sample_size": ("核对键值数", 10, 10_000, True), "max_scan_rows": ("整表统计行数上限", 0, 10_000_000, True), } _ENABLED_LABEL = "开启数据剖析" @@ -241,16 +241,96 @@ def sql_literal(value: Any, kind: str | None) -> str | None: return "'" + value.replace("'", "''") + "'" +#: 定长文字类型(CHAR(2)、NCHAR、bpchar、CHARACTER(2)):PostgreSQL、Oracle 取回的值带尾随空格补齐 +_FIXED_CHAR = re.compile(r"^\s*(?:N?CHAR|BPCHAR|CHARACTER)\b(?!\s+VARYING)", re.IGNORECASE) + + +def is_fixed_char(type_name: str | None) -> bool: + """定长文字列(CHAR(n)):值的尾随空格是补齐出来的,不是取值的一部分(SQL 比较时也不算)。""" + return bool(type_name) and bool(_FIXED_CHAR.match(str(type_name))) + + +class KeyLiterals(NamedTuple): + """key_literals 的结果。literals:去父表里对的 IN 列表字面量(去重、保持顺序);unmatchable:按类型就对不上父键、 + 算作没对上的键值个数(去重后);unsafe:值本身不能安全拼进 SQL、不计入核对的个数。""" + literals: list[str] + unmatchable: int + unsafe: int + + +def key_literals(values: list[Any], kind: str | None, *, parent: str | None, + child_type: str | None) -> KeyLiterals: + """从子表取回的键值 → 去父表里对的 IN 列表字面量(KeyLiterals)。 + + 字面量按**被指向列**的类型族写(catalog.column_family),不是按子表取回的值: + - 父列是文字、子列是数:写成 '123'。PostgreSQL 上 varchar 列 IN (123) 直接报「operator does not exist」; + - 父列是数、子列是文字:只拼整数写法的值('0012' 也照 12 写)。别的('A1'、'WALKIN')不拼——Oracle 上数字列 + IN ('A1') 会把字面量转成数报 ORA-01722,MySQL 会把 'A1' 转成 0 去比、对上不该对上的行——但它们**照样是子表 + 的键值**:整数主键里永远没有 'WALKIN',所以算作没对上,计入 unmatchable,由调用方留在覆盖率的分母里。要是 + 从分母里拿掉,散客代号占九成的一列只剩 1 个整数写法的键对上,就成了「覆盖率 100%」; + - 父列是时间:不在这里处理,_check_relation 先整条跳过(各家时间字面量写法不同、精度不同,键不核对); + - 父列类型认不出:照子表取回的值的类型写(sql_literal,以前的做法)。 + 子表是定长文字(CHAR(n))时先去掉尾随空格:PG、Oracle 取回的 'A ' 拿去和 VARCHAR 父列比就对不上,而按 CHAR 的 + 语义尾随空格本来就不算。不安全的值(反斜杠、「:名字」、控制字符、太长)照 sql_literal 的规矩跳过,记进 unsafe + (调用方另记「类型不支持核对,未计入」)。两个不同的取值写成同一个字面量('0012' 和 '12')只算一个,不算 unsafe。 + """ + if parent == "time": + raise ValueError("被指向列是日期时间的关系不核对键值(见 _check_relation)") + fixed = is_fixed_char(child_type) + out: list[str] = [] + unmatchable: set[str] = set() + unsafe = 0 + for value in values: + if fixed and isinstance(value, str): + value = value.rstrip(" ") + if parent == "text": + if isinstance(value, bool): + unsafe += 1 + continue + if isinstance(value, (int, float, Decimal)): + lit = sql_literal(value, "number") + lit = None if lit is None else "'" + lit + "'" + else: + lit = sql_literal(value, "text") + elif parent == "number": + if isinstance(value, str): + text = value.strip() + if not _INT_TEXT.fullmatch(text): + unmatchable.add(text) + continue + lit = str(int(text)) + else: + lit = sql_literal(value, "number") + else: + lit = sql_literal(value, kind) + if lit is None: + unsafe += 1 + else: + out.append(lit) + return KeyLiterals(list(dict.fromkeys(out)), len(unmatchable), unsafe) + + _DIALECT_MODULES = {"sqlite": sqlite, "postgres": postgresql, "postgresql": postgresql, "mysql": mysql, "mariadb": mysql, "oracle": oracle} +#: 日期列的最小值、最大值只看这个区间里的值(含下限、不含上限):0000-00-00(MySQL 关了严格模式写进去的零日期)、 +#: 0001-01-01、9999-12-31(「没有」「永久」的占位日期)、PostgreSQL 的 -infinity / infinity 都落在区间外。区间外的 +#: 值照实数出来(「有 N 个无效日期未计入」),不让一个占位日期把业务日期的范围撑成两千年。两端整天排除:带时刻的 +#: 9999-12-31 23:59:59、文字存的「0001-01-01 00:00:00」也算占位 +VALID_DATE_FROM = "0001-01-02" +VALID_DATE_BEFORE = "9999-12-31" +_ISO_DAY = re.compile(r"\d{4}-\d{2}-\d{2}") + + class SqlDialect: - """剖析 SQL 的方言差异:标识符怎么加引号、怎么只取前 n 行、统计信息在哪个系统视图里。只拼字符串,不连库。 + """剖析 SQL 的方言差异:标识符怎么加引号、怎么只取前 n 行、统计信息在哪个系统视图里、日期字面量和按天截断 + 怎么写、有没有块采样。只拼字符串,不连库。 标识符交给 SQLAlchemy 各方言的 identifier_preparer:需要时才加引号(memberTags 加、visits 不加), MySQL 用反引号。Oracle 尤其不能一律加双引号:探查拿到的表名是 SQLAlchemy 规整过的小写,库里其实是 - 大写,"visits" 加了引号就成了另一张不存在的表;不加引号 Oracle 自己转大写才对得上。 + 大写,"visits" 加了引号就成了另一张不存在的表;不加引号 Oracle 自己转大写才对得上。反过来,库里真是小写的 + 对象(建表时写了 "visits")探查时记了 quote_name,这种要强制加引号、不转大写(table、column、stats_sql)。 """ def __init__(self, kind: str | None) -> None: @@ -262,13 +342,64 @@ class SqlDialect: self._prep = self._dialect.identifier_preparer self.oracle = self.kind == "oracle" - def quote(self, name: str) -> str: - return self._prep.quote(name) + def quote(self, name: str, *, force: bool = False) -> str: + """标识符。force:探查记了 quote_name 的名字,原样加引号(Oracle 小写带引号的对象)。""" + return self._prep.quote_identifier(name) if force else self._prep.quote(name) def table(self, meta: dict[str, Any] | None, name: str) -> str: """写进 FROM 的表名:探查时指定了 schema 的带上它(Oracle 只读账号名下没有对象,全靠 schema 前缀)。""" schema = (meta or {}).get("schema") - return f"{self._prep.quote_schema(schema)}.{self.quote(name)}" if schema else self.quote(name) + quoted = self.quote(name, force=(meta or {}).get("quote_name") is True) + return f"{self._prep.quote_schema(schema)}.{quoted}" if schema else quoted + + def column(self, meta: dict[str, Any] | None, name: str) -> str: + """写进 SQL 的列名:表结构里这一列记了 quote_name 的强制加引号。""" + cols = (meta or {}).get("columns") or [] + entry = next((c for c in cols if isinstance(c, dict) and c.get("name") == name), None) + return self.quote(name, force=bool(entry and entry.get("quote_name") is True)) + + def date_literal(self, value: Any, *, text: bool = False) -> str: + """日期字面量(取值观察的时间窗口、日期列的有效区间用)。PostgreSQL、MySQL、Oracle 都认 DATE 'YYYY-MM-DD'; + SQLite 没有日期类型,日期存成文字,写字符串。text:列是文字存的日期(VARCHAR),按字符串比——PostgreSQL 上 + varchar 和 date 比较会直接报错。只收真实存在的日期(date 或 YYYY-MM-DD),别的抛 ValueError:拼进 SQL 的 + 只能是我们自己算出来的日子。""" + if isinstance(value, datetime): + value = value.date() + if isinstance(value, _date): + day = value.isoformat() + elif isinstance(value, str) and _ISO_DAY.fullmatch(value): + try: + day = _date.fromisoformat(value).isoformat() + except ValueError: + raise ValueError(f"不是有效的日期:{value}") from None + else: + raise ValueError(f"不是有效的日期:{value!r}") + if text or self.kind == "sqlite": + return f"'{day}'" + return f"DATE '{day}'" + + def day_trunc(self, column: str) -> str: + """时间列截到天(按日行数用)。PostgreSQL CAST(x AS DATE);MySQL DATE(x);Oracle 必须用 TRUNC(x):Oracle 的 + DATE 带时分秒,CAST(ts AS DATE) 截不掉;SQLite 的日期是文字,取前 10 个字符。""" + if self.kind in ("postgres", "postgresql"): + return f"CAST({column} AS DATE)" + if self.kind in ("mysql", "mariadb"): + return f"DATE({column})" + if self.oracle: + return f"TRUNC({column})" + return f"substr({column}, 1, 10)" + + def sample_from(self, table: str, percent: float) -> str | None: + """块采样的 FROM:PostgreSQL TABLESAMPLE SYSTEM (p)、Oracle SAMPLE BLOCK (p),按数据块随机取约 p% 的行; + MySQL、SQLite 没有块采样,返回 None(调用方退回「按存储顺序的前 N 行」,并照实写口径)。""" + p = float(percent) + if not (0 < p < 100): + raise ValueError(f"采样比例要在 0 到 100 之间:{percent}") + if self.kind in ("postgres", "postgresql"): + return f"{table} TABLESAMPLE SYSTEM ({p:g})" + if self.oracle: + return f"{table} SAMPLE BLOCK ({p:g})" + return None def limit(self, sql: str, n: int) -> str: """只取前 n 行。Oracle 11g 没有 LIMIT 也没有 FETCH FIRST,一律套一层 ROWNUM。""" @@ -304,7 +435,8 @@ class SqlDialect: if self.oracle: denorm = self._dialect.denormalize_name owner = sql_literal(denorm(schema), "text") if schema else "SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA')" - table = sql_literal(denorm(name), "text") + # 小写带引号的对象(quote_name)在数据字典里就是小写,不能转成大写去找 + table = sql_literal(name if (meta or {}).get("quote_name") is True else denorm(name), "text") if not (owner and table): return None return f"SELECT NUM_ROWS FROM ALL_TABLES WHERE OWNER = {owner} AND TABLE_NAME = {table}" @@ -335,8 +467,49 @@ class SqlDialect: f"WHERE {column} IS NOT NULL GROUP BY {column} ORDER BY cnt DESC, {column}") return self.limit(sql, n) - def min_max_sql(self, table: str, columns: list[str]) -> str: - return f"SELECT {', '.join(f'MIN({c}), MAX({c})' for c in columns)} FROM {table}" + def min_max_sql(self, table: str, columns: list[tuple[str, bool]]) -> str: + """日期列的最小值、最大值和无效日期的个数:每列三个值(MIN、MAX、区间外的个数)。columns 是 (写进 SQL 的列, + 是不是文字存的日期)。只看有效区间(VALID_DATE_FROM ≤ x < VALID_DATE_BEFORE)里的值,见 VALID_DATE_FROM。 + 整表统计,只对小表用。""" + parts = [] + for c, text in columns: + lo = self.date_literal(VALID_DATE_FROM, text=text) + hi = self.date_literal(VALID_DATE_BEFORE, text=text) + valid = f"{c} >= {lo} AND {c} < {hi}" + parts.append(f"MIN(CASE WHEN {valid} THEN {c} END), MAX(CASE WHEN {valid} THEN {c} END), " + f"SUM(CASE WHEN {c} < {lo} OR {c} >= {hi} THEN 1 ELSE 0 END)") + return f"SELECT {', '.join(parts)} FROM {table}" + + # ---- 取值观察(下一轮第 3 波用):窗口、按日行数、块采样。这里先把各方言的写法定下来,金样和容器集成测试都覆盖 + + def _window(self, column: str, since: Any, text: bool) -> str: + """时间窗口的条件:since 起(含),并且早于占位日期 9999-12-31——PostgreSQL 的 infinity、Oracle 的 9999-12-31 + 都大于任何一天,不截掉上端就会在「最近 35 天」里多出一天。""" + return (f"{column} >= {self.date_literal(since, text=text)} " + f"AND {column} < {self.date_literal(VALID_DATE_BEFORE, text=text)}") + + def day_volume_sql(self, table: str, column: str, since: Any, *, text: bool = False) -> str: + """最近若干天按天计数:since 起(含)每天几行。只在日期列打头有索引时对大表发(走索引的范围扫描)。""" + day = self.day_trunc(column) + return (f"SELECT {day} AS d, COUNT(*) AS n FROM {table} " + f"WHERE {self._window(column, since, text)} GROUP BY {day} ORDER BY d") + + def window_value_counts_sql(self, table: str, column: str, date_column: str, since: Any, n: int, *, + text: bool = False) -> str: + """时间窗口里的取值分布:业务日期 since 起(含)的行里,出现最多的 n 个取值和各自的行数。""" + sql = (f"SELECT {column}, COUNT(*) AS cnt FROM {table} " + f"WHERE {self._window(date_column, since, text)} AND {column} IS NOT NULL " + f"GROUP BY {column} ORDER BY cnt DESC, {column}") + return self.limit(sql, n) + + def sample_value_counts_sql(self, table: str, column: str, percent: float, n: int) -> str | None: + """块采样里的取值分布;没有块采样的方言返回 None。""" + source = self.sample_from(table, percent) + if source is None: + return None + sql = (f"SELECT {column}, COUNT(*) AS cnt FROM {source} WHERE {column} IS NOT NULL " + f"GROUP BY {column} ORDER BY cnt DESC, {column}") + return self.limit(sql, n) # ========================================================================== @@ -581,8 +754,10 @@ async def estimate_size(run: Callable[..., Awaitable[QueryResult]], dialect: Sql """先读统计信息;读不到(SQLite、没分析过的表)再数到 max_scan_rows + 1 行为止。 统计信息为 0 或负数(PostgreSQL 没分析过的表是 -1,MySQL 的估算值可能是 0)当作不知道:一张其实很大的表 - 要是被当成空表,后面就会对它做整表统计。返回 (行数, 途中没查成的几条)。这张表的查询次数、时长用完(或者任务止损) -时抛 _Stop。 + 要是被当成空表,后面就会对它做整表统计。统计信息说的行数不超过 max_scan_rows 时也再数一次(多一条有上限的 + 计数):统计信息可能是很久以前的(表后来涨到了几千万行,ANALYZE 没再跑过),照它把大表当小表,就会对它做 + 整表的 COUNT(DISTINCT)、MIN/MAX。数过的结果是准数(count),数到上限还没数完就按大表算(at_least)。 + 返回 (行数, 途中没查成的几条)。这张表的查询次数、时长用完(或者任务止损)时抛 _Stop。 """ skips: list[_Skip] = [] sql = dialect.stats_sql(meta, name) @@ -592,7 +767,9 @@ async def estimate_size(run: Callable[..., Awaitable[QueryResult]], dialect: Sql except _Skip as e: skips.append(e) else: - if rows is not None and rows > 0: + if rows is not None and rows > max(max_scan_rows, 0): + return TableSize(rows, "stats"), skips + if rows is not None and rows > 0 and max_scan_rows <= 0: return TableSize(rows, "stats"), skips if max_scan_rows <= 0: return TableSize(None, "unknown"), skips @@ -752,6 +929,8 @@ class _CodesOutcome: rows: int note: str complete: bool = False + #: 列定义(ENUM)列出的取值:这次没出现的也写进码值(含义待填),码值是「可能的取值」 + declared: list[str] = field(default_factory=list) @dataclass @@ -901,6 +1080,11 @@ def _column(meta: dict[str, Any], name: str) -> str | None: if isinstance(c, dict) and str(c.get("name", "")).lower() == lowered), None) +def _column_meta(meta: dict[str, Any], name: str) -> dict[str, Any] | None: + """表结构里这一列(类型、取值列表、quote_name);name 是表结构里的写法(_column 的返回值)。""" + return next((c for c in meta.get("columns") or [] if isinstance(c, dict) and c.get("name") == name), None) + + def _constraint(meta: dict[str, Any], column: str) -> str | None: """这一列单独是主键或某个唯一约束时返回「主键」/「唯一约束」,否则 None。""" wanted = [column.lower()] @@ -950,7 +1134,7 @@ async def _unique(cx: _TableCx, table: str, meta: dict[str, Any], column: str, if not size.within(cx.settings.max_scan_rows): return None, "too_large" d = cx.dialect - res = await cx.run(d.unique_check_sql(d.table(meta, table), d.quote(column))) + res = await cx.run(d.unique_check_sql(d.table(meta, table), d.column(meta, column))) row = res.rows[0] if res.rows else [None, None] total, distinct = _as_int(row[0]), _as_int(row[1]) if total is None or distinct is None: @@ -969,20 +1153,39 @@ def _fk_note(child: tuple[bool | None, str]) -> str: return f"{catalog.PROFILE_NOTE_PREFIX}外键约束已保证引用完整,未抽样核对覆盖率;子表一侧{basis}:{shape}" +#: MySQL 上比较文字按列的排序规则:常见的排序规则不分大小写(_ci),PAD SPACE 的还忽略尾随空格。覆盖率、唯一性、 +#: 码值的分组都按它算,写进说明,人才知道「A01」和「a01」在这里算同一个值 +MYSQL_COLLATION_NOTE = "MySQL 按列的排序规则比较文字(常见的排序规则不分大小写),覆盖率和唯一性按它计算" +MYSQL_CODES_COLLATION_NOTE = "MySQL 按列的排序规则分组(常见的排序规则不分大小写),大小写不同的写法可能并成一个取值" + + +def _collation_note(dialect: SqlDialect, family: str | None, note: str) -> str | None: + """MySQL、MariaDB 上文字列(或类型认不出的列)的排序规则说明;别的方言、数字列没有。""" + return note if dialect.kind in ("mysql", "mariadb") and family in ("text", None) else None + + def _relation_note(*, sample: int, matched: int, coverage: float, scan_cap: int | None, skipped_values: int, parent: tuple[bool | None, str], child: tuple[bool | None, str] | None, - holds: bool) -> str: - """关系的剖析备注:抽样规模、覆盖率、父键和子键唯一的依据。人据此判断这条结论有多可靠。 - 不写日期:什么时候剖析的看这一项的 updated_at(界面按本地时间显示)。""" + holds: bool, unmatchable: int = 0, collation: str | None = None) -> str: + """关系的剖析备注:取了多少个键值、看了多少行、覆盖率、父键和子键唯一的依据。人据此判断这条结论有多可靠。 + 不写日期:什么时候剖析的看这一项的 updated_at(界面按本地时间显示)。 + + 不说「抽样」:DISTINCT … LIMIT 取的是数据库先给出的那几个值,大表上是按存储顺序的前若干行里的值,不是随机抽的 + (多半是最老的数据)。unmatchable:被指向列是数字、子表取到的却不是整数写法的键值个数(key_literals)——已经算在 + sample 里、算作没对上,这里说明为什么没去父表里对。""" parts: list[str] = [] low = coverage < catalog.PROFILE_VERIFY_COVERAGE if low: - parts.append(f"抽样覆盖率 {_percent(coverage)},可能不是这条关系") - scope = f"(只看前 {scan_cap} 行)" if scan_cap else "" - sampled = f"子表抽样 {sample} 个不同键值{scope},父表对上 {matched} 个" + parts.append(f"覆盖率 {_percent(coverage)},可能不是这条关系") + scope = f"(按存储顺序只看前 {scan_cap} 行)" if scan_cap else "" + sampled = f"子表取了 {sample} 个不同键值{scope},父表对上 {matched} 个" parts.append(sampled if low else f"{sampled},覆盖率 {_percent(coverage)}") if skipped_values: parts.append(f"另有 {skipped_values} 个键值的类型不支持核对,未计入") + if unmatchable: + parts.append(f"其中 {unmatchable} 个不是整数写法,被指向列是数字,算作没对上") + if collation: + parts.append(collation) unique, how = parent if unique is True: parts.append(f"被指向列是{how}" if how in ("主键", "唯一约束") else "被指向列经计数核对唯一") @@ -1026,24 +1229,32 @@ async def _check_relation(cx: _TableCx, table: str, rel: dict[str, Any], out: _T return d, s = cx.dialect, cx.settings + child_col, parent_col = _column_meta(meta, col), _column_meta(pmeta, pcol) + parent_family = catalog.column_family(parent_col) + if parent_family == "time": + # 时间键不核对:各家时间字面量写法、精度不同,拼出来的 IN 列表对不对得上说不准。不发子表的取值查询 + raise _Skip("unsupported", f"{parent}.{pcol} 是日期时间,各库的时间写法和精度不同,不核对这种关系的覆盖率") size = await cx.size_with_data(table, result) scan_cap = cx.scan_cap(size) child_table, parent_table = d.table(meta, table), d.table(pmeta, parent) - res = await cx.run(d.distinct_sample_sql(child_table, d.quote(col), s.sample_size, scan_cap=scan_cap), + res = await cx.run(d.distinct_sample_sql(child_table, d.column(meta, col), s.sample_size, scan_cap=scan_cap), max_rows=s.sample_size) kind = (res.column_types or {}).get(res.columns[0]) if res.columns else None values = [row[0] for row in res.rows if row] - literals = list(dict.fromkeys(lit for v in values if (lit := sql_literal(v, kind)) is not None)) + literals, unmatchable, unsafe = key_literals(values, kind, parent=parent_family, + child_type=(child_col or {}).get("type")) if not values: raise _Skip("no_data", f"{table}.{col} 没有非空值,无法核对") - if not literals: + if not literals and not unmatchable: raise _Skip("unsupported", f"{table}.{col} 的取值类型不支持核对(只核对整数和文字)") + # 分母是子表取到的全部键值:不是整数写法、按类型就对不上整数父键的(unmatchable)也在里面,算作没对上 + sample = len(literals) + unmatchable matched = 0 for start in range(0, len(literals), IN_CHUNK): - sql = d.match_count_sql(parent_table, d.quote(pcol), literals[start:start + IN_CHUNK]) + sql = d.match_count_sql(parent_table, d.column(pmeta, pcol), literals[start:start + IN_CHUNK]) matched += _as_int(_first(await cx.run(sql))) or 0 matched = min(matched, len(literals)) - coverage = round(matched / len(literals), 4) + coverage = round(matched / sample, 4) parent_unique = await _unique(cx, parent, pmeta, pcol, result) child_unique = await _unique(cx, table, meta, col, result) if parent_unique[0] else None @@ -1051,12 +1262,13 @@ async def _check_relation(cx: _TableCx, table: str, rel: dict[str, Any], out: _T if child_unique is not None: cardinality = "one_to_one" if child_unique[0] is True else "many_to_one" holds = coverage >= catalog.PROFILE_VERIFY_COVERAGE and cardinality is not None - note = _relation_note(sample=len(literals), matched=matched, coverage=coverage, scan_cap=scan_cap, - skipped_values=len(values) - len(literals), parent=parent_unique, child=child_unique, - holds=holds) + note = _relation_note(sample=sample, matched=matched, coverage=coverage, scan_cap=scan_cap, + skipped_values=unsafe, parent=parent_unique, child=child_unique, holds=holds, + unmatchable=unmatchable, + collation=_collation_note(d, parent_family, MYSQL_COLLATION_NOTE)) checked = child_unique is not None and child_unique[0] is not None out.relations.append(_RelationOutcome(rid=str(rel["id"]), target=_relation_target(rel), coverage=coverage, - cardinality=cardinality, holds=holds, sample=len(literals), + cardinality=cardinality, holds=holds, sample=sample, matched=matched, note=note, cardinality_checked=checked)) @@ -1076,17 +1288,28 @@ _NOT_CODE_TAILS = frozenset({"id", "no", "num", "number", "name", "desc", "descr "at", "date", "time", "on", "amount", "price", "count", "qty", "rate", "url"}) #: 像日期的取值:2026-07-01、2026/07/01,后面可以跟时间 _DATE_TEXT = re.compile(r"^\d{4}[-/]\d{2}[-/]\d{2}") +#: 日期列最小值、最大值不计入的值(见 VALID_DATE_FROM)在说明里的说法 +INVALID_DATES = "零日期 0000-00-00、0001-01-01、9999-12-31 这类占位日期或无穷大" -def _code_type_ok(type_name: str | None) -> bool: - """整数,或者短文字(定长不超过 MAX_CODE_LEN;SQLite 的 TEXT 不带长度,也算)。""" - t = (type_name or "").upper() +def _is_enum(type_name: str | None, enum_values: Any = None) -> bool: + """取值列表有限的列:MySQL 的 ENUM(…) / SET(…),或者探查记了取值列表(PostgreSQL 的枚举:类型写法只是类型名, + setup_status 这种以 set 开头的名字不算 SET,见 catalog._MYSQL_ENUM)。""" + return bool(enum_values) or catalog._mysql_enum_kind(type_name) is not None + + +def _code_type_ok(type_name: str | None, enum_values: Any = None) -> bool: + """ENUM / SET(取值列表有限,直接算码值候选)、整数,或者短文字(定长不超过 MAX_CODE_LEN;SQLite 的 TEXT 不带 + 长度,也算)。类型是按方言编译的完整写法:VARCHAR2(20 CHAR) 的长度按 20 算。""" + if _is_enum(type_name, enum_values): + return True + t = catalog._type_words(type_name) if not t or any(k in t for k in ("CLOB", "BLOB", "BINARY", "JSON", "DATE", "TIME", "INTERVAL", "REAL", "FLOAT", "DOUBLE", "BOOL")): return False if "INT" in t: return True - size = re.search(r"\(\s*(\d+)\s*(?:,\s*(\d+)\s*)?\)", t) + size = re.search(r"\(\s*(\d+)(?:\s+(?:CHAR|BYTE))?\s*(?:,\s*(\d+)\s*)?\)", t) if any(k in t for k in ("NUMBER", "NUMERIC", "DECIMAL")): return size is None or not size.group(2) or int(size.group(2)) == 0 if any(k in t for k in ("CHAR", "TEXT", "STRING")): @@ -1100,9 +1323,9 @@ def _looks_like_code(name: str) -> bool: def _code_columns(meta: dict[str, Any], notes: dict[str, Any]) -> list[str]: - """要取值分布的列:目录里度量类型记为状态的、剖析以前提过码值候选的(再核一次)、或者类型和命名像状态、 - 类型、渠道的整数或短文字列。主键、唯一约束列、关系里的键列不算;码值人工确认或驳回过的不再取。 - 被遮罩的列也列出来,由检查记为「被遮罩」跳过——要让人看得见为什么它没有码值。""" + """要取值分布的列:目录里度量类型记为状态的、剖析以前提过码值候选的(再核一次)、ENUM / SET 列(取值列表有限, + 不看名字)、或者类型和命名像状态、类型、渠道的整数或短文字列。主键、唯一约束列、关系里的键列不算;码值人工确认 + 或驳回过的不再取。被遮罩的列也列出来,由检查记为「被遮罩」跳过——要让人看得见为什么它没有码值。""" pk = [c.lower() for c in meta.get("primary_key") or []] unique = {str(g[0]).lower() for g in meta.get("unique") or [] if isinstance(g, list) and len(g) == 1} keys = {str(c).lower() for r in notes.get("relations") or [] if isinstance(r, dict) for c in r.get("columns") or []} @@ -1117,23 +1340,28 @@ def _code_columns(meta: dict[str, Any], notes: dict[str, Any]) -> list[str]: status_measure = (isinstance(measure, dict) and measure.get("value") == "status" and measure.get("status") != "rejected") profiled = isinstance(codes, dict) and codes.get("source") == "profile" - if not (status_measure or profiled or _looks_like_code(name)): + enum = _is_enum(col.get("type"), col.get("enum_values")) + if not (status_measure or profiled or enum or _looks_like_code(name)): continue lowered = name.lower() - if (pk == [lowered]) or lowered in unique or lowered in keys or not _code_type_ok(col.get("type")): + if (pk == [lowered]) or lowered in unique or lowered in keys or not _code_type_ok(col.get("type"), + col.get("enum_values")): continue out.append(name) return out -def _code_key(value: Any) -> str | None: - """取值分布里的一个取值 → 码值(文字)。只认整数和不太长的文字。""" +def _code_key(value: Any, *, fixed: bool = False) -> str | None: + """取值分布里的一个取值 → 码值(文字)。只认整数和不太长的文字。fixed:定长文字列(CHAR(n)),去掉补齐的尾随 + 空格(PG、Oracle 取回的是 'A ',WHERE status = 'A' 照样对得上,码值写 'A')。""" if value is None or isinstance(value, bool): return None if isinstance(value, int): return str(value) if isinstance(value, float): return str(int(value)) if value.is_integer() else None + if isinstance(value, str) and fixed: + value = value.rstrip(" ") if isinstance(value, str) and value.strip() and len(value) <= MAX_CODE_LEN and not _CONTROL.search(value): return value return None @@ -1151,34 +1379,53 @@ async def _check_codes(cx: _TableCx, table: str, column: str, out: _TableOutcome if cx.is_masked(column): raise _Skip("masked", f"{table}.{column} 在数据源设置中被遮罩,不取值", carry=False) d, meta = cx.dialect, cx.tables[table] + col_meta = _column_meta(meta, column) or {} size = await cx.size_with_data(table, result) scan_cap = cx.scan_cap(size) - res = await cx.run(d.value_counts_sql(d.table(meta, table), d.quote(column), CODES_MAX + 1, scan_cap=scan_cap), + res = await cx.run(d.value_counts_sql(d.table(meta, table), d.column(meta, column), CODES_MAX + 1, + scan_cap=scan_cap), max_rows=CODES_MAX + 1) if not res.rows: raise _Skip("no_data", f"{table}.{column} 没有非空值,无法取值分布") if len(res.rows) > CODES_MAX: raise _Skip("high_cardinality", f"{table}.{column} 的取值超过 {CODES_MAX} 个,不作为码值候选", carry=False) - values: list[tuple[str, int]] = [] + fixed = is_fixed_char(col_meta.get("type")) + enum = _is_enum(col_meta.get("type"), col_meta.get("enum_values")) + counts: dict[str, int] = {} for row in res.rows: - key = _code_key(row[0]) + key = _code_key(row[0], fixed=fixed) if key is None: raise _Skip("unsupported", f"{table}.{column} 的取值不是整数或短文字,不作为码值候选", carry=False) - values.append((key, _as_int(row[1]) or 0)) - if len(values) >= 3 and all(n == 1 for _, n in values): - # 每个取值只出现一次(票种编码 T01…T09):这是一列标识,不是状态码 + counts[key] = counts.get(key, 0) + (_as_int(row[1]) or 0) + values = list(counts.items()) + if len(values) >= 3 and all(n == 1 for _, n in values) and not enum: + # 每个取值只出现一次(票种编码 T01…T09):这是一列标识,不是状态码。ENUM 的取值列表是列定义给的,不按这条判 raise _Skip("high_cardinality", f"{table}.{column} 的取值各不相同,像标识而不是码值,不作为码值候选", carry=False) rows = sum(n for _, n in values) - scope = f"按前 {scan_cap} 行统计" if scan_cap else "统计全表" + # 不说「抽样」:大表看的是按存储顺序的前若干行(多半是最老的数据),不是随机抽的 + scope = f"按存储顺序取前 {scan_cap} 行统计" if scan_cap else "统计全表" spread = "、".join(f"{k}({_share(n, rows)})" for k, n in values) - note = f"{NOTE_PREFIX}{scope},{rows} 行非空值共 {len(values)} 个取值:{spread}。" - out.codes.append(_CodesOutcome(column=column, values=values, rows=rows, note=note, complete=scan_cap is None)) + extra: list[str] = [] + # ENUM 的取值列表(SET 的是成员,取值是成员的组合,不算):这次没出现的也是可能的取值 + is_set = catalog._mysql_enum_kind(col_meta.get("type")) == "SET" + declared = [] if is_set else [str(v) for v in col_meta.get("enum_values") or []] + unseen = [v for v in declared if v not in counts] + if unseen: + extra.append(f"列定义里另有 {len(unseen)} 个取值这次没有出现:{'、'.join(unseen[:CODES_MAX])}") + family = "text" if enum else catalog.column_family(col_meta) + collation = _collation_note(d, family, MYSQL_CODES_COLLATION_NOTE) + if collation: + extra.append(collation) + tail = "".join(f";{x}" for x in extra) + note = f"{NOTE_PREFIX}{scope},{rows} 行非空值共 {len(values)} 个取值:{spread}{tail}。" + out.codes.append(_CodesOutcome(column=column, values=values, rows=rows, note=note, complete=scan_cap is None, + declared=unseen)) def _is_date_column(col: dict[str, Any]) -> bool: """日期类列(catalog.is_date_column:和 SQL 检查认时间列是同一个判断)。""" - return catalog.is_date_column(str(col.get("name") or ""), col.get("type")) + return catalog.is_date_column(str(col.get("name") or ""), col.get("type"), enum_values=col.get("enum_values")) async def _check_dates(cx: _TableCx, table: str, columns: list[str], total: int, out: _TableOutcome, @@ -1190,20 +1437,31 @@ async def _check_dates(cx: _TableCx, table: str, columns: list[str], total: int, if not size.within(cx.settings.max_scan_rows): raise _Skip("too_large", f"{table} 的行数超过整表统计的行数上限({cx.settings.max_scan_rows} 行)," "不取日期列的最小值和最大值") - res = await cx.run(d.min_max_sql(d.table(meta, table), [d.quote(c) for c in columns])) + # 文字存的日期(SQLite 的 TEXT、业务库里的 VARCHAR 日期)按字符串和有效区间比;日期类型的按日期字面量比 + texts = [catalog.column_family(_column_meta(meta, c)) != "time" for c in columns] + res = await cx.run(d.min_max_sql(d.table(meta, table), [(d.column(meta, c), t) for c, t in zip(columns, texts)])) row = res.rows[0] if res.rows else [] + invalid: dict[str, int] = {} for i, column in enumerate(columns): - low, high = (row[2 * i], row[2 * i + 1]) if len(row) >= 2 * i + 2 else (None, None) + low, high, bad = (row[3 * i], row[3 * i + 1], row[3 * i + 2]) if len(row) >= 3 * i + 3 else (None, None, 0) + invalid[column] = _as_int(bad) or 0 if low is None or high is None or not (_DATE_TEXT.match(str(low)) and _DATE_TEXT.match(str(high))): continue - result.date_ranges.append({"column": column, "min": str(low), "max": str(high)}) + found: dict[str, Any] = {"column": column, "min": str(low), "max": str(high)} + if invalid[column]: + found["invalid"] = invalid[column] + result.date_ranges.append(found) if total != 1: return if not result.date_ranges: + if invalid.get(columns[0]): + raise _Skip("no_data", f"{table}.{columns[0]} 只有 {invalid[columns[0]]} 个无效日期({INVALID_DATES})," + "没有可识别的日期值") raise _Skip("no_data", f"{table}.{columns[0]} 没有可识别的日期值") found = result.date_ranges[0] + dropped = (f"另有 {found['invalid']} 个无效日期({INVALID_DATES})未计入," if found.get("invalid") else "") note = (f"{NOTE_PREFIX}表里只有这一个日期类列,取值从 {str(found['min'])[:19]} 到 " - f"{str(found['max'])[:19]},可作为业务日期的参考。") + f"{str(found['max'])[:19]},{dropped}可作为业务日期的参考。") out.business_date = _DateOutcome(column=found["column"], low=found["min"], high=found["max"], note=note) @@ -1226,7 +1484,7 @@ def _relation_finding(o: _RelationOutcome, rel: dict[str, Any], *, status: str, elif status == "verified": summary = f"覆盖率 {_percent(o.coverage)}," + ("仍为有确证" if previous == "verified" else "升为有确证") elif o.coverage < catalog.PROFILE_VERIFY_COVERAGE: - summary = f"抽样覆盖率 {_percent(o.coverage)},保持推断" + summary = f"覆盖率 {_percent(o.coverage)},保持推断" else: summary = "被指向列未核实唯一,保持推断" return {"kind": "relation", "path": f"relations.{o.rid}", "target": o.target, @@ -1241,6 +1499,8 @@ def _codes_value(c: _CodesOutcome, item: dict[str, Any] | None) -> dict[str, str 沿用它的含义;它写了、数据里没出现的取值也留着——码值是「可能的取值」,没出现不等于不存在。""" old = item.get("value") if isinstance(item, dict) and isinstance(item.get("value"), dict) else {} value = {key: str(old.get(key) or "") for key, _ in c.values} + for key in c.declared: + value.setdefault(key, str(old.get(key) or "")) for key, meaning in old.items(): if key not in value and isinstance(meaning, str) and meaning.strip(): value[key] = meaning @@ -1744,8 +2004,8 @@ def ensure_enabled(row: Any, settings: ProfileSettings | None = None) -> Profile #: 有效行过滤核对的说法(写进这一项的 note;前面是模型给的依据) FILTER_CHECKED = "剖析核对:排除 {share} 的行(共 {rows} 行)" -FILTER_SAMPLED = "剖析核对:按前 {rows} 行统计,排除 {share} 的行" -FILTER_SAMPLED_EDGE = "剖析核对:按前 {rows} 行统计,{what},请人工核对" +FILTER_SAMPLED = "剖析核对:按存储顺序取前 {rows} 行统计,排除 {share} 的行" +FILTER_SAMPLED_EDGE = "剖析核对:按存储顺序取前 {rows} 行统计,{what},请人工核对" FILTER_UNCHECKED = "没有核对排除比例({why})" FILTER_NOT_ENABLED = "数据源没有开数据剖析" diff --git a/backend/app/data/catalog_profile_jobs.py b/backend/app/data/catalog_profile_jobs.py index 726e0417..74f9c058 100644 --- a/backend/app/data/catalog_profile_jobs.py +++ b/backend/app/data/catalog_profile_jobs.py @@ -58,6 +58,9 @@ from app.db.models import CatalogDraftJob, CatalogProfileJob, CatalogProfileTask logger = logging.getLogger(__name__) +#: 剖析任务占同源队(叫法登记在运行时里,见 core/job_runtime.register_lane_kind) +runtime.register_lane_kind("profile", "剖析") + #: 一个任务最多几张表 MAX_TABLES = 500 #: 每个源保留几个结束了的剖析任务(某张表最近一次剖析所在的任务另外保留) diff --git a/backend/app/data/catalog_view.py b/backend/app/data/catalog_view.py new file mode 100644 index 00000000..5af6e2f9 --- /dev/null +++ b/backend/app/data/catalog_view.py @@ -0,0 +1,181 @@ +"""目录视图:运行和助手读数据目录统一经这一个接缝(docs/design/next-round/01-效果度量与回归评测.md 做法 5,总方案 X7)。 + +**为什么要接缝。** 目录在四处被读:运行时的 db_schema / db_query 工具(目录同时冻结进表结构快照)、助手查看(look 表详情)、 +自查用的 SQL 检查器、双盲复算(交给盲算的说明、比较两条 SQL 用的中文名)。以前各读各的、没有统一的开关,「开目录和关 +目录」「只用已核实的项」「起草模型 A 的目录和 B 的」这些对照都做不了,评测期间目录被起草或剖析改了也不知道。现在四处都 +经这里取,一处定读哪一份;lint 测试(tests/test_catalog_view.py)不许这几个模块绕过它直接 read_catalog、直接查 CatalogNote。 + +**视图**(CatalogView): +- mode:live 当前目录(缺省,行为和以前一样);snapshot 一份冻结的目录快照(工件 id 在 snapshot,内容见 SNAPSHOT_KIND); + off 不给目录。 +- statuses:all 全部(驳回的照旧不给模型看);reviewed 只要已核实(verified)和已确认(confirmed)的项,推断的 + (proposed)去掉——量的是模型起草、还没审过的那些项是帮忙还是添乱。 + +**视图从哪来。** 运行从 Run.options['catalog_view'] 读(launch_run 收 options,发起时定下、随 run.started 封存,审批恢复、 +重启续跑都沿用——用上下文变量传的话恢复之后就丢了);生成过程(问数据的 answer_ops、评测的作答)从参数读。都不给就是 live。 +冻结进表结构快照时,不是缺省视图的在工件引用的 meta 里记下视图(describe),证据面板据此写得出「本次运行目录:关」。 + +**快照的内容**(第 2 波的评测题集包写,这里只定读法):kind=eval_catalog_snapshot 的内容寻址工件, +{"sources": {源 id: {"tables": {表名: {"version": 版本, "notes": 目录}}}}}。快照里没有这个源就是这个源没有目录; +快照工件不在了照实报错(CatalogSnapshotMissing),不悄悄退回当前目录——那样对照就白做了。 +""" +from __future__ import annotations + +import logging +from collections.abc import Iterable, Mapping +from dataclasses import dataclass +from typing import Any + +from app.data import catalog +from app.data.catalog import CatalogEntry + +logger = logging.getLogger(__name__) + +#: Run.options 里放视图的键 +OPTION_KEY = "catalog_view" +MODES = ("live", "snapshot", "off") +#: 各状态集合留下哪些项的状态(all 不筛:驳回的由各处照旧经 visible_notes 去掉) +STATUS_SETS: dict[str, tuple[str, ...] | None] = {"all": None, "reviewed": ("verified", "confirmed")} +#: 目录快照工件的 kind(细案 01 做法 6) +SNAPSHOT_KIND = "eval_catalog_snapshot" + + +class CatalogViewInvalid(ValueError): + """视图写得不对(服务端代码写错了才会有:客户端写不了 Run.options)。""" + + +class CatalogSnapshotMissing(RuntimeError): + """视图指定的目录快照工件不在了、或者内容认不出。""" + + +@dataclass(frozen=True) +class CatalogView: + mode: str = "live" + statuses: str = "all" + #: mode=snapshot 时快照工件的 id + snapshot: str | None = None + + def __post_init__(self) -> None: + if self.mode not in MODES: + raise CatalogViewInvalid(f"目录视图的 mode 只能是 {'、'.join(MODES)},当前为「{self.mode}」") + if self.statuses not in STATUS_SETS: + raise CatalogViewInvalid(f"目录视图的 statuses 只能是 {'、'.join(STATUS_SETS)},当前为「{self.statuses}」") + if (self.mode == "snapshot") != bool(self.snapshot): + raise CatalogViewInvalid("目录视图只有 mode=snapshot 时带快照工件 id,而且必须带") + + @property + def is_default(self) -> bool: + return self.mode == "live" and self.statuses == "all" + + def as_option(self) -> dict[str, Any]: + """写进 Run.options['catalog_view'] 的形状。""" + return {"mode": self.mode, "statuses": self.statuses, **({"snapshot": self.snapshot} if self.snapshot else {})} + + @classmethod + def parse(cls, raw: Any) -> CatalogView: + """Run.options['catalog_view']、生成参数 → 视图。None 是缺省(live);写得不对抛 CatalogViewInvalid。""" + if raw is None: + return LIVE + if isinstance(raw, CatalogView): + return raw + if not isinstance(raw, Mapping): + raise CatalogViewInvalid(f"目录视图需要是一个对象,当前为「{raw}」") + unknown = set(raw) - {"mode", "statuses", "snapshot"} + if unknown: + raise CatalogViewInvalid(f"目录视图里不认识的键:{'、'.join(sorted(unknown))}") + return cls(mode=str(raw.get("mode") or "live"), statuses=str(raw.get("statuses") or "all"), + snapshot=str(raw["snapshot"]) if raw.get("snapshot") else None) + + +LIVE = CatalogView() + + +def of_options(options: Mapping[str, Any] | None) -> CatalogView: + """一次运行的视图(Run.options 里的 catalog_view)。没有就是 live。""" + return CatalogView.parse((options or {}).get(OPTION_KEY)) + + +def of_run(run_ctx: Any) -> CatalogView: + """运行里的节点构造 ToolContext 时用:RunContext.extra['run_options'] 里的视图(runner 每段执行都放这份)。""" + from app.engine.runner import RUN_OPTIONS_KEY + + return of_options((getattr(run_ctx, "extra", None) or {}).get(RUN_OPTIONS_KEY)) + + +def describe(view: CatalogView | None) -> dict[str, Any] | None: + """记进冻结快照 meta 的说法:缺省视图(live、全部)不记,别的照 as_option 记下。""" + view = view or LIVE + return None if view.is_default else view.as_option() + + +def restrict(view: CatalogView | None, notes: Mapping[str, Any] | None) -> dict[str, Any]: + """一张表的目录在这个视图下留下什么(状态集合;驳回的项原样留着,见 catalog.restrict_notes)。""" + allowed = STATUS_SETS[(view or LIVE).statuses] + if allowed is None: + return dict(notes or {}) + return catalog.restrict_notes(notes, allowed) + + +def _restricted(view: CatalogView, entries: Mapping[str, CatalogEntry]) -> dict[str, CatalogEntry]: + if STATUS_SETS[view.statuses] is None: + return dict(entries) + out: dict[str, CatalogEntry] = {} + for table, entry in entries.items(): + if notes := restrict(view, entry.notes): + out[table] = CatalogEntry(table=entry.table, notes=notes, version=entry.version, + updated_by=entry.updated_by, updated_at=entry.updated_at) + return out + + +def _snapshot_tables(view: CatalogView, source_id: str) -> dict[str, CatalogEntry]: + from app.core.artifact_store import load + + try: + content = load(view.snapshot or "") + except ValueError as e: # 内容和哈希对不上 + raise CatalogSnapshotMissing(f"目录快照 {str(view.snapshot)[:12]}… 读不出:{e}") from e + if not isinstance(content, dict) or not isinstance(content.get("sources"), dict): + raise CatalogSnapshotMissing(f"目录快照 {str(view.snapshot)[:12]}… 不在了或内容认不出,这次不能按它读目录") + tables = ((content["sources"].get(source_id) or {}).get("tables")) or {} + out: dict[str, CatalogEntry] = {} + for table, body in tables.items(): + if isinstance(body, Mapping) and isinstance(body.get("notes"), Mapping): + out[str(table)] = CatalogEntry(table=str(table), notes=dict(body["notes"]), + version=int(body.get("version") or 0)) + return out + + +async def read_many(session: Any, view: CatalogView | None, source_ids: Iterable[str]) -> dict[str, dict[str, CatalogEntry]]: + """几个数据源在这个视图下的目录:{源 id: {表名: CatalogEntry}},没有目录的源不出现。live 一次查询读出。""" + view = view or LIVE + ids = [i for i in dict.fromkeys(source_ids) if i] + if view.mode == "off" or not ids: + return {} + if view.mode == "snapshot": + raw = {sid: found for sid in ids if (found := _snapshot_tables(view, sid))} + else: + raw = await catalog.read_catalogs(session, ids) + return {sid: kept for sid, entries in raw.items() if (kept := _restricted(view, entries))} + + +async def read(view: CatalogView | None, source: Any) -> dict[str, CatalogEntry]: + """一个数据源在这个视图下的目录:{表名: CatalogEntry}。用自己的短会话读(调用方的会话里已经加载的对象不会因为回滚 + 过期)。live 读不到(库出错)当没有目录、记日志:目录是附加说明,不能挡住查询;快照不在了照实抛 CatalogSnapshotMissing。""" + from app.db.base import SessionLocal + + view = view or LIVE + source_id = getattr(source, "id", None) + if view.mode == "off" or not source_id: + return {} + if view.mode == "snapshot": + return _restricted(view, _snapshot_tables(view, source_id)) + try: + async with SessionLocal() as own: + return (await read_many(own, view, [source_id])).get(source_id, {}) + except Exception: # noqa: BLE001 + logger.exception("读取数据源 %s 的数据目录失败,本次不附目录", getattr(source, "name", source_id)) + return {} + + +__all__ = ["CatalogSnapshotMissing", "CatalogView", "CatalogViewInvalid", "LIVE", "MODES", "OPTION_KEY", "SNAPSHOT_KIND", + "STATUS_SETS", "describe", "of_options", "of_run", "read", "read_many", "restrict"] diff --git a/backend/app/data/csv_grid.py b/backend/app/data/csv_grid.py new file mode 100644 index 00000000..7fb8ece7 --- /dev/null +++ b/backend/app/data/csv_grid.py @@ -0,0 +1,206 @@ +"""CSV 走配方(Excel 期 5 第 4 项):把 CSV 适配成配方读取层认的样子——一张可见工作表的扫描结果(WorkbookScan) +和网格(Grid / 逐行流),之后的执行器、核对、说明、溯源、按期累积全部复用,不另写一套 CSV 的解析(否则会和配方 +语言分叉,违背「一种说法」)。 + +**怎么认出是 CSV。** 和简单导入(tabular._CSV)一样按扩展名定:只有 .csv / .tsv 按 CSV 读,并且内容得是文字 +(不是 zip、不是旧版 .xls 的 OLE 文件头、前 64 KB 里没有 NUL 字节)。以前只看内容,企业系统常见的「HTML 表格冒充 +.xls」、误传的文本文件都会被拆成一张叫「CSV」的工作表去起草、核对,不再像以前那样明确拒收;现在别的扩展名照旧交给 +xlsx_scan 拒收,内容是网页或 XML 的另给一句能照着做的话。编码识别复用 tabular 读 CSV 的那一段(UTF-8 / GB18030 / +带 BOM 的 UTF-16);.tsv 一律按制表符分隔(同 tabular),.csv 嗅探逗号、分号、制表符、竖线。扫描时定下的分隔符 +记进工作表的 path(CSV_PATH 后面跟分隔符),之后读格子用同一个,不再各嗅各的。 + +**坐标沿用「工作表!A1」。** 细案原写「格子坐标记成 R{行}C{列}」,这里没这么做(见交付说明的偏离项):坐标在执行器、 +核对、修复按钮、框选、溯源、裁判摘录、前端网格里一共有十来处解析和拼写,引入第二种写法就要处处认两种,漏一处就是 +着色错位、下钻找不到格子;期 1 读 CSV 时(tabular._shape_reasons)用的也是 A1。CSV 没有工作表,名字固定为 +CSV_SHEET:每月文件名不同(orders_2026-08.csv、orders_2026-09.csv),配方按工作表名认,名字必须稳定。行号是第几条 +记录(引号里的换行不另起一行),列号是第几个字段,和用 Excel 打开这个 CSV 看到的 A1 坐标一致。 + +**格子的值。** CSV 里只有文字。只有写法毫无歧义的才转成数(-12、80.5,不超过 15 位有效数字),其余一律原样留作文字: +「1,234」(千分位,交给配方的 text_number 决定)、「007」(前导零是编码的一部分)、「+5」「1e5」「12%」、日期写法 +(执行器按 DATE 列解析文字日期,起草按文字认日期)、「=SUM(…)」(CSV 里不是公式,Excel 打开时才会当公式算)。 +空字段、只有空白的字段是空格(和 xlsx_scan 的口径一致)。没有公式、合并单元格、隐藏行列。 +""" +from __future__ import annotations + +import csv +import io +import re +from dataclasses import dataclass +from typing import Any, Iterator + +from app.data.recipe_types import GridCell +from app.data.xlsx_scan import Bounds, SheetScan, UnsupportedTable, WorkbookScan + +__all__ = ["CSV_PATH", "CSV_SHEET", "CSV_SUFFIXES", "MAX_COLS", "MAX_ROWS", "CsvTable", "cell_value", "is_csv_sheet", + "looks_like_csv", "looks_like_markup", "parse", "rows", "scan", "scan_any", "sheet_delimiter"] + +#: CSV 的「工作表名」:配方按它认(SheetMatch.name),每月的文件名不同也认得出 +CSV_SHEET = "CSV" +#: SheetScan.path 的开头:读取层据此知道这张「工作表」要从 CSV 里读,不去压缩包里找 XML。后面跟着扫描时定下的分隔符 +#: ("csv:,"、"csv:\t");只有「csv:」的(这一版之前的暂存区)读的时候再嗅一次 +CSV_PATH = "csv:" +#: 按 CSV 读的扩展名,和简单导入(tabular._CSV)一致 +CSV_SUFFIXES = (".csv", ".tsv") +#: 嗅探认的分隔符 +_DELIMITERS = ",;\t|" +#: 内容是网页或 XML(「HTML 表格冒充 .xls」、Excel 2003 的 XML 表格)时的说法 +MARKUP_MESSAGE = ("无法打开该文件:它其实是网页或 XML 格式(有的系统导出的「Excel」文件是这样),不是 Excel 工作簿。" + "请用 Excel 打开后另存为 .xlsx 或 .csv 再上传") +NOT_TEXT_MESSAGE = ("无法按 CSV 读取这个文件:扩展名是 CSV,内容却不是文字(可能是图片、PDF 之类的文件改了扩展名)。" + "请确认上传的文件,或在 Excel 中另存为 .csv 后重新上传") +#: 和 Excel 的上限一致:超过的 CSV 用 Excel 打开也装不下,交给模型的格子坐标也写不出来 +MAX_ROWS = 1_048_576 +MAX_COLS = 16_384 +#: 嗅探看多少字节 +_SNIFF = 64 * 1024 +#: 写法毫无歧义的整数、小数(不带正号、不带前导零、不用科学计数法) +_INT = re.compile(r"-?(?:0|[1-9][0-9]*)") +_DEC = re.compile(r"-?(?:0|[1-9][0-9]*)\.[0-9]+") +#: 超过这么多位有效数字的不转成数:转成浮点会丢位(19 位卡号撞值),留作文字 +_DIGITS_MAX = 15 + + +def looks_like_csv(raw: bytes) -> bool: + """这份原件按 CSV 读:不是 zip、不是 OLE(旧版 .xls)、开头没有 NUL 字节、不是空文件。带 BOM 的 UTF-16 例外 + (每个字符都带 NUL,Excel「Unicode 文本」另存出来的就是它)。""" + if not raw or raw[:4] == b"PK\x03\x04" or raw[:8] == b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1": + return False + if raw[:2] in (b"\xff\xfe", b"\xfe\xff"): + return True + return b"\x00" not in raw[:_SNIFF] + + +def looks_like_markup(raw: bytes) -> bool: + """内容是网页或 XML:去掉 BOM 和开头的空白后以「<」开头。真的工作簿以 zip(PK)或 OLE 文件头开头,不会是这样。""" + head = raw[:4096] + if head[:2] in (b"\xff\xfe", b"\xfe\xff"): + try: + text = head.decode("utf-16", errors="ignore") + except UnicodeDecodeError: + return False + else: + text = head.decode("latin-1") + return text.lstrip("\ufeff\xef\xbb\xbf \t\r\n").startswith("<") + + +def _suffix(filename: str | None) -> str: + name = str(filename or "").rsplit("/", 1)[-1] + return "." + name.rsplit(".", 1)[-1].lower() if "." in name else "" + + +def cell_value(text: str) -> Any: + """CSV 的一个字段 → 格子的值:空 / 只有空白是 None(空格),写法毫无歧义的整数、小数转成数,其余原样(文字)。""" + t = text.strip() + if not t: + return None + digits = len(t.lstrip("-").replace(".", "").lstrip("0")) or 1 + if digits <= _DIGITS_MAX: + if _INT.fullmatch(t): + return int(t) + if _DEC.fullmatch(t): + return float(t) + return text + + +@dataclass +class CsvTable: + """解析好的 CSV:records[i] 是第 i + 1 条记录的字段(原文)。""" + + records: list[list[str]] + delimiter: str + + +def parse(raw: bytes, delimiter: str | None = None) -> CsvTable: + """原件 → 记录。编码照 tabular 读 CSV 的做法;delimiter 不给时嗅探(同 tabular)。字段太长、引号不配对读不了时抛 + UnsupportedTable。""" + from app.data import tabular + + text = tabular._decode_csv(raw) + if delimiter is None: + try: + delimiter = csv.Sniffer().sniff(text[:8192], delimiters=_DELIMITERS).delimiter + except csv.Error: + delimiter = "," # 嗅不出来就按最常见的来 + records: list[list[str]] = [] + try: + for record in csv.reader(io.StringIO(text, newline=""), delimiter=delimiter): + if len(records) >= MAX_ROWS: + raise UnsupportedTable(f"CSV 文件超过 {MAX_ROWS:,} 行,超出可以导入的上限") + if len(record) > MAX_COLS: + raise UnsupportedTable(f"CSV 文件第 {len(records) + 1} 行有 {len(record):,} 列,超过 {MAX_COLS:,} 列的上限") + records.append(record) + except csv.Error as e: + raise UnsupportedTable(f"无法按 CSV 读取这个文件:第 {len(records) + 1} 行的写法不正确({e})。" + "请在 Excel 中重新另存为 CSV 后上传") from e + return CsvTable(records=records, delimiter=delimiter) + + +def _cells(table: CsvTable) -> Iterator[tuple[int, list[tuple[int, GridCell]]]]: + for r, record in enumerate(table.records, 1): + out = [(c, GridCell(v)) for c, field in enumerate(record, 1) if (v := cell_value(field)) is not None] + if out: + yield r, out + + +def is_csv_sheet(sheet: Any) -> bool: + """这张「工作表」是 CSV 适配出来的(path 以 CSV_PATH 开头)。""" + return str(getattr(sheet, "path", "") or "").startswith(CSV_PATH) + + +def sheet_delimiter(sheet: Any) -> str | None: + """扫描时定下的分隔符(记在 path 里);这一版之前的暂存区没记,返回 None(读的时候再嗅)。""" + tail = str(getattr(sheet, "path", "") or "")[len(CSV_PATH):] + return tail if len(tail) == 1 and tail in _DELIMITERS else None + + +def scan(raw: bytes, *, delimiter: str | None = None) -> WorkbookScan: + """CSV 原件 → 扫描结果:一张可见工作表(名字 CSV_SHEET、路径 CSV_PATH 加分隔符),边界、非空格数、逐行非空格数的 + 游程。没有合并区、隐藏行列、公式、错误值、离群格(CSV 里每一格都是写出来的,没有残留格式)。""" + table = parse(raw, delimiter) + sheet = SheetScan(name=CSV_SHEET, state="visible", path=CSV_PATH + table.delimiter) + lo_r = lo_c = hi_r = hi_c = 0 + runs: list[tuple[int, int, int]] = [] + for r, cells in _cells(table): + n = len(cells) + sheet.nonempty += n + first, last = cells[0][0], cells[-1][0] + if not lo_r: + lo_r, lo_c, hi_c = r, first, last + lo_c, hi_c, hi_r = min(lo_c, first), max(hi_c, last), r + if runs and runs[-1][1] == r - 1 and runs[-1][2] == n: + runs[-1] = (runs[-1][0], r, n) + else: + runs.append((r, r, n)) + if lo_r: + sheet.bounds = Bounds(lo_r, lo_c, hi_r, hi_c) + sheet.dimension = sheet.bounds.a1() + sheet.row_runs = runs + return WorkbookScan(sheets=[sheet], format="csv") + + +def scan_any(raw: bytes, filename: str | None) -> WorkbookScan: + """配方导入的扫描入口:.csv / .tsv 走这里的适配(.tsv 按制表符分隔),其余照旧交给 xlsx_scan(读不了照旧拒收, + 文案不变);内容是网页或 XML 的另给一句能照着做的话(MARKUP_MESSAGE)。扩展名是 CSV、内容却是工作簿(xlsx 改了 + 扩展名)的照工作簿读。""" + from app.data import xlsx_scan + + suffix = _suffix(filename) + if suffix in CSV_SUFFIXES: + if looks_like_csv(raw): + return scan(raw, delimiter="\t" if suffix == ".tsv" else None) + if raw[:4] != b"PK\x03\x04": + raise UnsupportedTable(NOT_TEXT_MESSAGE) + elif looks_like_markup(raw): + raise UnsupportedTable(MARKUP_MESSAGE) + return xlsx_scan.scan(raw) + + +def rows(table: CsvTable, *, min_row: int, max_row: int, min_col: int, + max_col: int) -> Iterator[tuple[int, list[tuple[int, GridCell]]]]: + """按行流式给一个矩形里的非空格:(行号, [(列号, 格)…]),口径同 xlsx_cells.sheet_rows(整行为空的不给)。""" + for r in range(max(min_row, 1), min(max_row, len(table.records)) + 1): + record = table.records[r - 1] + out = [(c, GridCell(v)) for c in range(max(min_col, 1), min(max_col, len(record)) + 1) + if (v := cell_value(record[c - 1])) is not None] + if out: + yield r, out diff --git a/backend/app/data/introspect.py b/backend/app/data/introspect.py index ea8875be..7d7f478b 100644 --- a/backend/app/data/introspect.py +++ b/backend/app/data/introspect.py @@ -165,6 +165,89 @@ def _unique_groups(constraints: list[dict[str, Any]], indexes: list[dict[str, An return groups +def _storage_type(col_type: Any) -> Any: + """PostgreSQL 的域(DOMAIN)换成它底下的类型(域套域的一路往下找);别的类型原样返回。""" + from sqlalchemy.dialects.postgresql import DOMAIN + + seen = 0 + while isinstance(col_type, DOMAIN) and getattr(col_type, "data_type", None) is not None and seen < 8: + col_type, seen = col_type.data_type, seen + 1 + return col_type + + +def type_text(col_type: Any, dialect: Any) -> str: + """反射到的列类型 → 存进表结构的写法:按这个数据源的方言编译的完整写法(VARCHAR2(20 CHAR)、NUMBER(10, 0)、 + ENUM('paid','refund')、TIMESTAMP WITH TIME ZONE),编译不了的退回 str()。PostgreSQL 的域(DOMAIN)记它底下的类型: + 域编译出来只是域名(settlement_amount),看不出是数字还是文字,名字里的词(set、UPDATE 里的 DATE)还会让类型大类、 + 日期列、码值候选认错;模型照着写 SQL 也要知道它存的是什么。 + + 以前存的是 str(类型),即 SQLAlchemy 的通用写法:Oracle 的 VARCHAR2、NUMBER 丢了长度和精度,MySQL 的 ENUM 丢了 + 取值列表,PostgreSQL 的 TIMESTAMP 丢了时区。类型写不全,表结构前后比对就看不出类型变了,ENUM 状态列也进不了 + 码值候选。这一改会让新探查的表结构快照哈希变化(提示词里的类型写法变长),旧运行封存的快照不受影响。""" + if col_type is None: + return "" + col_type = _storage_type(col_type) + try: + compiled = str(col_type.compile(dialect=dialect)) + except Exception: # noqa: BLE001 - 编译不了(NullType 等)就用通用写法,不让一列拖垮整次探查 + return str(col_type) + if getattr(dialect, "name", "") == "oracle" and compiled.startswith("VARCHAR("): + # SQLAlchemy 把 Oracle 的 VARCHAR2 反射成通用的 VARCHAR,按 Oracle 方言编译出来是 VARCHAR(10 CHAR)(容器集成 + # 测试实测)。Oracle 的数据字典里只有 VARCHAR2(建表写 VARCHAR 也存成 VARCHAR2),照数据字典写 + compiled = "VARCHAR2" + compiled[len("VARCHAR"):] + return compiled + + +def _enum_values(col_type: Any) -> list[str] | None: + """ENUM(MySQL、PostgreSQL 的枚举,都是 sqltypes.Enum 的子类)的取值列表,MySQL SET 的成员列表;别的类型 None。 + 套在域(DOMAIN)里的枚举也算(同 type_text)。""" + from sqlalchemy.dialects.mysql import SET + from sqlalchemy.types import Enum + + col_type = _storage_type(col_type) + if isinstance(col_type, Enum): + values = list(col_type.enums or []) + elif isinstance(col_type, SET): + values = list(col_type.values or []) + else: + return None + return [str(v) for v in values] + + +def _forced_quote(name: Any) -> bool: + """名字必须原样加引号:Oracle 反射回来的全小写对象(建表时写了 "visits")是 quoted_name(quote=True)。 + 它存进 JSON 就成了普通字符串,剖析拼 SQL 时不加引号,Oracle 把它转成大写 VISITS,查的是另一个不存在的对象。""" + return getattr(name, "quote", None) is True + + +def column_entry(col: Mapping[str, Any], dialect: Any) -> dict[str, Any]: + """反射到的一列 → 表结构里的一列:名字、按方言编译的类型、可空、注释;ENUM / SET 另记取值列表(enum_values), + 要强制加引号的名字另记 quote_name。只在有的时候写这两个键:其余列和以前一字不差。""" + entry: dict[str, Any] = { + "name": str(col["name"]), + "type": type_text(col.get("type"), dialect), + "nullable": bool(col.get("nullable", True)), + "comment": col.get("comment") or None, + } + values = _enum_values(col.get("type")) + if values: + entry["enum_values"] = values + if _forced_quote(col.get("name")): + entry["quote_name"] = True + return entry + + +def index_first_columns(indexes: list[dict[str, Any]], pk: list[str]) -> list[str]: + """哪些列打头有索引:各索引(含主键)的第一列,去重、按名字排序。取值观察看大表的最新业务日期、按日行数时, + 只在业务日期列打头有索引时才发窗口查询(按它走索引,不扫全表)。表达式索引(列名是 None)不算。""" + first = {str(pk[0])} if pk else set() + for ix in indexes: + cols = ix.get("column_names") or [] + if cols and cols[0] is not None: + first.add(str(cols[0])) + return sorted(first) + + async def introspect(source: Any, *, schema: str | None = None) -> dict[str, Any]: """探查整个库的结构,结果直接存进 DataSource.schema_cache。 @@ -266,24 +349,20 @@ async def introspect(source: Any, *, schema: str | None = None) -> dict[str, Any # 只读账号名下没有这些对象,全靠跨 schema 授权 "qualified": f"{target_schema}.{name}" if target_schema else name, "schema": target_schema, - "columns": [ - { - "name": c["name"], - "type": str(c.get("type") or ""), - "nullable": bool(c.get("nullable", True)), - "comment": c.get("comment") or None, - } - for c in columns - ], + "columns": [column_entry(c, sync_conn.dialect) for c in columns], "primary_key": pk, "comment": comment, "is_view": name in views, } + if _forced_quote(name): + tables[name]["quote_name"] = True if name in fk_ok: tables[name]["foreign_keys"] = _foreign_keys(fk_by.get(key) or [], target_schema, default_schema) if name in uq_ok: tables[name]["unique"] = _unique_groups(uq_by.get(key) or [], (ix_by.get(key) or []) if name in ix_ok else [], pk) + if name in ix_ok: + tables[name]["index_first_columns"] = index_first_columns(ix_by.get(key) or [], pk) payload: dict[str, Any] = {"tables": tables, "truncated": truncated, "total": len(names) + len(views)} if failures and not tables: @@ -471,6 +550,10 @@ def describe_table(source: Any, table: str, *, catalog: Mapping[str, Mapping[str marks.append("非空") if col.get("comment"): marks.append(col["comment"]) + values = col.get("enum_values") + if values and "(" not in str(col.get("type") or ""): + # PostgreSQL 的枚举类型写法只有类型名(order_status):取值列表另外写出来,模型才知道能写哪些值 + marks.append("取值:" + "、".join(str(v) for v in values[:30]) + ("…" if len(values) > 30 else "")) suffix = f" {'、'.join(marks)}" if marks else "" lines.append(f" {col['name']} {col['type']}{suffix}") if catalog: diff --git a/backend/app/data/recipe.py b/backend/app/data/recipe.py index 6e4d4a6d..5bc4cad5 100644 --- a/backend/app/data/recipe.py +++ b/backend/app/data/recipe.py @@ -43,6 +43,9 @@ from app.data.recipe_parsers import ( text_label, ) from app.data.recipe_types import ( + LIST_PERIOD_END, + LIST_PERIOD_START, + LIST_SUBTOTAL_DIM, LIST_TOTAL_DIM, ColumnOut, CrosstabBlock, @@ -271,6 +274,8 @@ _FIELD_LABEL = { "parts": "组成列", "claims": "认领", "a": "一方", "b": "另一方", "reason": "理由", "ignore_rows": "按行标签忽略的行", "ignore_columns": "按表头忽略的列", "ignore_outside": "按同一行文字忽略的数字", "label": "标签", "anchor": "同一行的文字", + "subtotals": "小计行", "no_subtotals": "没有小计行", "scope": "小计管哪些明细", "group_column": "分组的列", "period": "每行属于哪一期", + "column": "日期列", "check": "核对方式", "synthesize": "加统计期列", } @@ -608,15 +613,25 @@ class _Checker: for si, bi, block in self.lists: base = ("sheets", si, "blocks", bi) cols = [(c.name, base + ("columns", n, "name")) for n, c in enumerate(block.columns)] + if block.period is not None and block.period.synthesize: + # 期 5:加上的统计期起止两列也是这张表的列,和表头推出的列名不能撞 + pp = base + ("period", "synthesize") + cols += [(LIST_PERIOD_START, pp), (LIST_PERIOD_END, pp)] out.append(_Writer(block.table, base + ("table",), "data", cols, origin=_pointer(base))) + numeric = [(c.name, base + ("columns", n, "name")) for n, c in enumerate(block.columns) if c.type in _NUMERIC] total = block.rows.total_row if total is not None and total.keep_as: tp = base + ("rows", "total_row", "keep_as") - tcols = [(LIST_TOTAL_DIM, tp)] - tcols += [(c.name, base + ("columns", n, "name")) - for n, c in enumerate(block.columns) if c.type in _NUMERIC] - out.append(_Writer(total.keep_as, tp, "reported_total", tcols, + out.append(_Writer(total.keep_as, tp, "reported_total", [(LIST_TOTAL_DIM, tp)] + numeric, origin=_pointer(base + ("rows", "total_row")))) + sub = block.rows.subtotals + if sub is not None and sub.keep_as: + sp = base + ("rows", "subtotals", "keep_as") + scols = [(LIST_SUBTOTAL_DIM, sp)] + if sub.scope == "group_column" and sub.group_column: + scols.append((sub.group_column, base + ("rows", "subtotals", "group_column"))) + out.append(_Writer(sub.keep_as, sp, "reported_total", scols + numeric, + origin=_pointer(base + ("rows", "subtotals")))) return out # ---- 8. 名字 ---- @@ -680,6 +695,10 @@ class _Checker: if sheet.context: continue for bi, block in enumerate(sheet.blocks): + if isinstance(block, ListBlock) and block.period is not None: + # 期 5:每行属于哪一期要拿统计期来核对(按日期列)或填进去(加起止两列) + self.add(("sheets", si, "blocks", bi, "period"), "covers_without_context", + "要按统计期核对或填写每一行的统计期,但这个工作表没有配置统计期") if not isinstance(block, CrosstabBlock): continue ap = ("sheets", si, "blocks", bi, "axis") @@ -873,6 +892,10 @@ class _Checker: if total is not None and any(ch.isdigit() for ch in total.pick): self.add(("sheets", si, "blocks", bi, "rows", "total_row", "pick"), "period_literal", f"合计行的开头词「{total.pick}」不能含数字(配方里不能写死年份或期次)") + sub = block.rows.subtotals + if sub is not None and any(ch.isdigit() for ch in sub.pick): + self.add(("sheets", si, "blocks", bi, "rows", "subtotals", "pick"), "period_literal", + f"小计行的开头词「{sub.pick}」不能含数字(配方里不能写死年份或期次)") # ---- 4、9、13. 单位:键是本表列名,值在词表里,与原表标签、表头里的单位一致 ---- @@ -902,10 +925,13 @@ class _Checker: self.unit_against(seg.keep_as.table, seg.keep_as.value, target.locate.title, "分段标题") for _si, _bi, block in self.lists: total = block.rows.total_row + sub = block.rows.subtotals for c in block.columns: self.unit_against(block.table, c.name, c.header, "表头") if total is not None and total.keep_as and c.type in _NUMERIC: self.unit_against(total.keep_as, c.name, c.header, "表头") + if sub is not None and sub.keep_as and c.type in _NUMERIC: + self.unit_against(sub.keep_as, c.name, c.header, "表头") def after_target(self, si: int, bi: int, seg: DerivedSegment) -> DimensionSegment | None: """合计分段紧跟的那个 dimension 分段(同一块里按 id 找);找不到或不是 dimension 段时已报 derived_after_invalid。""" @@ -998,6 +1024,8 @@ class _Checker: if total is not None and total.label_column not in names: self.add(base + ("rows", "total_row", "label_column"), "column_unknown", f"合计标签所在的列「{total.label_column}」不在这个列表的列里{self._near(total.label_column, names)}") + self.check_subtotals(base, block, names) + self.check_list_period(base, block, names) seen: dict[str, str] = {} for n, c in enumerate(block.columns): key = match_key(c.header) @@ -1006,6 +1034,53 @@ class _Checker: f"表头「{c.header}」与「{seen[key]}」是同一个表头") seen.setdefault(key, c.header) + def check_subtotals(self, base: tuple[Any, ...], block: ListBlock, names: list[str]) -> None: + """期 5:小计行的标签列、分组列要是这个列表的列;按组小计要给分组列;小计和合计的开头词不能互为前缀 + (「合计」「合计小计」:一行就分不清是小计还是合计);配了小计又声明没有小计(no_subtotals)是矛盾的。""" + sub = block.rows.subtotals + if sub is None: + return + sp = base + ("rows", "subtotals") + if block.rows.no_subtotals: + self.add(base + ("rows", "no_subtotals"), "subtotals_conflict", + "配了小计行,又声明了这张列表没有小计行:二选一") + if sub.label_column not in names: + self.add(sp + ("label_column",), "column_unknown", + f"小计标签所在的列「{sub.label_column}」不在这个列表的列里{self._near(sub.label_column, names)}") + if sub.scope == "group_column": + if not sub.group_column: + self.add(sp + ("group_column",), "subtotal_group_missing", "按组小计时要指明按哪一列分组") + elif sub.group_column not in names: + self.add(sp + ("group_column",), "column_unknown", + f"小计分组的列「{sub.group_column}」不在这个列表的列里{self._near(sub.group_column, names)}") + elif sub.group_column: + self.add(sp + ("group_column",), "subtotal_group_missing", + "小计管「上一行小计之后的全部明细」时不按列分组,请去掉分组的列,或改为按组小计") + total = block.rows.total_row + if total is not None: + a, b = match_key(sub.pick), match_key(total.pick) + if a.startswith(b) or b.startswith(a): + self.add(sp + ("pick",), "subtotal_pick_conflict", + f"小计行的开头词「{sub.pick}」和合计行的「{total.pick}」一个是另一个的开头,分不清是小计还是合计") + + def check_list_period(self, base: tuple[Any, ...], block: ListBlock, names: list[str]) -> None: + """期 5:每行属于哪一期,按日期列核对和加起止两列只能二选一;按日期列时那一列要是这个列表的日期列。""" + period = block.period + if period is None: + return + pp = base + ("period",) + if (period.column is None) == (period.synthesize is None): + self.add(pp, "list_period_invalid", "每一行属于哪一期只能二选一:按一个日期列核对,或加统计期起止两列") + return + if period.column is not None: + col = next((c for c in block.columns if c.name == period.column), None) + if col is None: + self.add(pp + ("column",), "column_unknown", + f"核对统计期的列「{period.column}」不在这个列表的列里{self._near(period.column, names)}") + elif col.type != "DATE": + self.add(pp + ("column",), "list_period_invalid", + f"核对统计期的列「{period.column}」的类型要是日期") + # ---- 期 3:忽略规则、表头上方的标题(P3-SPEC 9.2) ---- def check_ignores(self) -> None: diff --git a/backend/app/data/recipe_accumulate.py b/backend/app/data/recipe_accumulate.py index 5f6a8b54..ac689387 100644 --- a/backend/app/data/recipe_accumulate.py +++ b/backend/app/data/recipe_accumulate.py @@ -30,6 +30,7 @@ from app.data.recipe_parsers import hour_range, hour_range_total, match_key, tex from app.data.recipe_types import ( UNION_VER, AccumulatePlan, ChangeClass, CheckResult, ColumnOut, CrosstabBlock, DerivedSegment, DimensionSegment, PartInfo, Recipe, UnionPart, UnionReport, accumulate_blockers, derive_tables, + list_period_columns, ) from app.data.table_versions import settle_journal, sha_json @@ -508,14 +509,19 @@ def union_options(parts: list[tuple[str, str, str]], target_recipe_sha: str, *, class _Table: - """并集里的一张表:列(目标配方的在前、退役的按给定顺序追加在后)、主键、日期轴列。""" + """并集里的一张表:列(目标配方的在前、退役的按给定顺序追加在后)、主键、日期轴列。期 5:列表写入的表按 period + 分期(list_period_columns):按日期列的,U3 核对那一列落在该期统计期内;加统计期起止两列的,核对两列就是该期。""" - def __init__(self, name: str, cols: list[ColumnOut], grain: list[str]) -> None: + def __init__(self, name: str, cols: list[ColumnOut], grain: list[str], + period: tuple[str, ...] | None = None) -> None: self.name = name self.cols = cols self.grain = grain self.axis = next((c.name for c in cols if c.role == "axis"), None) self.names = [c.name for c in cols] + self.period = period if period is None or all(c in self.names for c in period[1:]) else None + if self.axis is None and self.period is not None and self.period[0] == "column": + self.axis = self.period[1] def _copy_col(col: ColumnOut) -> ColumnOut: @@ -551,6 +557,7 @@ def _layout(target: Recipe, parts: list[UnionPart], out: dict[str, _Table] = {} retired = retired or {} + periods = list_period_columns(target) for name, cols in tables.items(): mine = [_copy_col(c) for c in cols] known = {c.name for c in mine} @@ -561,7 +568,7 @@ def _layout(target: Recipe, parts: list[UnionPart], mine.append(_copy_col(found) if found is not None else _copy_col(col)) known.add(col.name) spec = specs.get(name) - out[name] = _Table(name, mine, list(spec.grain) if spec is not None else []) + out[name] = _Table(name, mine, list(spec.grain) if spec is not None else [], periods.get(name)) for name, cols in retired.items(): if name in out: continue @@ -831,20 +838,36 @@ def _finish(conn: sqlite3.Connection, layout: dict[str, _Table], parts: list[Uni u2.append(f"表「{table.name}」有 {dup} 组主键重复") checks.append(_check("U2", "union_pk", "累积后主键不重复", checked=len(keyed), failed=u2, sql=sql2)) - # U3 统计期:第 k 期的 rowid 区间里,日期都在 [start_k, end_k] 之内 + # U3 统计期:第 k 期的 rowid 区间里,日期都在 [start_k, end_k] 之内。期 5:按日期列分期的列表,日期为空也算不在; + # 加统计期起止两列的列表,两列必须正好是这一期 u3: list[str] = [] sql3, params3, checked3 = None, [], 0 for k, (part, ranges) in enumerate(zip(parts, part_rows), 1): for name, span in ranges.items(): - axis = layout[name].axis - if axis is None or axis not in part_cols[k - 1].get(name, []): - continue + table = layout[name] + have = part_cols[k - 1].get(name, []) a, b = span["union"] + if table.period is not None and table.period[0] == "synthesize": + start_col, end_col = table.period[1], table.period[2] + if start_col not in have or end_col not in have or b < a: + continue + checked3 += 1 + sql3 = (f"SELECT COUNT(*) FROM {_q(name)} WHERE rowid BETWEEN ? AND ? " + f"AND ({_q(start_col)} IS NOT ? OR {_q(end_col)} IS NOT ?)") + params3 = [a, b, part.start, part.end] + bad = conn.execute(sql3, params3).fetchone()[0] + if bad: + u3.append(f"第 {k} 期({_period_text(part.start, part.end)})的「{name}」有 {bad} 行的统计期不是这一期") + continue + axis = table.axis + if axis is None or axis not in have: + continue if b < a: continue checked3 += 1 + nulls = f"{_q(axis)} IS NULL OR " if table.period is not None else "" sql3 = (f"SELECT COUNT(*) FROM {_q(name)} WHERE rowid BETWEEN ? AND ? " - f"AND ({_q(axis)} < ? OR {_q(axis)} > ?)") + f"AND ({nulls}{_q(axis)} < ? OR {_q(axis)} > ?)") params3 = [a, b, part.start, part.end] bad = conn.execute(sql3, params3).fetchone()[0] if bad: diff --git a/backend/app/data/recipe_checks.py b/backend/app/data/recipe_checks.py index e095ada7..79a21b96 100644 --- a/backend/app/data/recipe_checks.py +++ b/backend/app/data/recipe_checks.py @@ -23,6 +23,7 @@ from typing import Any, Callable, Iterable from app.data.engine import open_checked_sqlite from app.data.recipe_types import ( + SUBTOTAL_KEY, CheckResult, ColumnOut, CrosstabBlock, @@ -34,6 +35,7 @@ from app.data.recipe_types import ( Recipe, SumEq, derive_tables, + list_period_columns, ) #: 无法核对的原因(契约 CheckResult.reasons 的键)的固定顺序:细节、说明都按这个顺序列 @@ -71,24 +73,33 @@ class CheckPlan: k: dict[str, str] = field(default_factory=dict) #: derived 分段键 → G 编号(只给有公式格的 derived 段) g: dict[str, str] = field(default_factory=dict) - #: 列表块键 → T 编号(每个有合计行的列表块一条) + #: 列表块键 → T 编号(每个有合计行的列表块一条);期 5:块键 + SUBTOTAL_KEY → 这个块的小计核对(一条) t: dict[str, str] = field(default_factory=dict) items: dict[str, list[DerivedItem]] = field(default_factory=dict) def _recipe_keys(recipe: Recipe) -> tuple[list[str], list[str]]: - """配方里 derived 分段键、带合计行的列表块键,按配方里的先后。""" + """配方里 derived 分段键、带合计行的列表块键,按配方里的先后。期 5:带小计的列表块另有一个键(块键 + + SUBTOTAL_KEY),排在这个块的合计行后面:同一块的小计和合计各编一个 T,结论分开写。""" derived: list[str] = [] totals: list[str] = [] for sheet in recipe.sheets: for block in sheet.blocks: if isinstance(block, CrosstabBlock): derived += [s.id for s in block.segments if isinstance(s, DerivedSegment)] - elif block.rows.total_row is not None: + continue + if block.rows.total_row is not None: totals.append(block.id) + if block.rows.subtotals is not None: + totals.append(f"{block.id}{SUBTOTAL_KEY}") return derived, totals +def is_subtotal_key(key: str) -> bool: + """T 编号的键是不是一个块的小计核对(期 5)。""" + return key.endswith(SUBTOTAL_KEY) + + def _ordered(recipe_keys: list[str], items: list[DerivedItem]) -> list[str]: """先按配方里的先后;配方里找不到的分段(不该出现)按执行结果里第一次出现的先后排在后面,不丢。""" present: list[str] = [] @@ -570,6 +581,25 @@ def _check_pk(sql: _Sql, cid: str, title: str, table: str, grain: list[str]) -> # ---------------------------------------------------------------- 入口 +def _check_period_rows(sql: _Sql, cid: str, title: str, table: str, column: str, period: Any, + lineage: dict[str, dict[str, list[list[Any]]]]) -> CheckResult: + """期 5:列表按日期列分期(ListPeriod.column、check=within_context)时,每行的这一列都落在本期统计期内。 + 日期为空的行也算不在:看不出它属于哪一期。不在的行按期累积会混进别的期(统计期互不重叠的前提就破了), + 所以是结构类、不能写理由接受。统计期没定下来(要人录入)时不会走到这里:那时不建库。""" + if period is None or not period.start or not period.end: + raise _Broken("本期没有统计期,无法核对每行的日期") + q = (f"SELECT rowid FROM {_q(table)} WHERE {_q(column)} IS NULL OR {_q(column)} < ? OR {_q(column)} > ? " + "ORDER BY rowid") + params = [period.start, period.end] + bad = [r[0] for r in sql.all(q, params)] + total = sql.one(f"SELECT COUNT(*) FROM {_q(table)}")[0] + cells = [c for rowid in bad[:_DETAIL_MAX] if (c := lineage_cell(lineage, table, column, rowid))] + details = [f"有 {len(bad)} 行的「{column}」为空或不在 {period.start} 至 {period.end} 之内"] if bad else [] + return CheckResult(id=cid, kind="period_rows", title=title, status="mismatch" if bad else "passed", + category="structure", checked=int(total), failed=len(bad), sql=q, params=params, + details=details, cells=cells) + + def _axis_names(recipe: Recipe) -> dict[str, str]: """derived 分段键 → 所在交叉表块的轴列名(K 的铺满判定不按日期分)。""" out: dict[str, str] = {} @@ -629,7 +659,9 @@ def run_checks(db_path: str, extraction: Extraction, recipe: Recipe) -> list[Che jobs.append((cid, "formula_refs", f"合计公式引用(分段「{seg}」)", lambda t, seg=seg, cid=cid: _check_g(sql, cid, t, plan.items[seg]))) for block, cid in plan.t.items(): - jobs.append((cid, "column_sum", f"合计行按列求和(块「{block}」)", + title = (f"小计行按各组明细求和(块「{block.removesuffix(SUBTOTAL_KEY)}」)" if is_subtotal_key(block) + else f"合计行按列求和(块「{block}」)") + jobs.append((cid, "column_sum", title, lambda t, block=block, cid=cid: _check_t(sql, cid, t, plan.items[block], full_calc))) for rel in recipe.relations: if isinstance(rel, SumEq): @@ -646,6 +678,12 @@ def run_checks(db_path: str, extraction: Extraction, recipe: Recipe) -> list[Che for i, table in enumerate([t for t in tables if grains.get(t)], 1): jobs.append((f"P{i}", "pk_unique", f"主键唯一:「{table}」", lambda t, table=table, i=i: _check_pk(sql, f"P{i}", t, table, grains[table]))) + period = extraction.period + dated = [(t, spec[1]) for t, spec in list_period_columns(recipe).items() if spec[0] == "column" and t in tables] + for i, (table, column) in enumerate(dated, 1): + jobs.append((f"D{i}", "period_rows", f"「{table}」每行的「{column}」都在本期统计期内", + lambda t, table=table, column=column, i=i: _check_period_rows( + sql, f"D{i}", t, table, column, period, extraction.lineage))) try: return list(extraction.context_checks) + [_guarded(sql, *job) for job in jobs] finally: diff --git a/backend/app/data/recipe_confirm.py b/backend/app/data/recipe_confirm.py index f2a9419c..015abf54 100644 --- a/backend/app/data/recipe_confirm.py +++ b/backend/app/data/recipe_confirm.py @@ -43,7 +43,7 @@ from app.data.recipe_diff import ( ) from app.data.recipe_parsers import UNITS, match_key, period_residue, split_unit_suffix from app.data.recipe_types import ( - ConfirmItem, CrosstabBlock, DerivedSegment, Dismissed, ListBlock, MeasuresSegment, NotComparable, Recipe, + LIST_PERIOD_END, LIST_PERIOD_START, SUBTOTAL_KEY, ConfirmItem, CrosstabBlock, DerivedSegment, Dismissed, ListBlock, MeasuresSegment, NotComparable, Recipe, SheetRecipe, SumEq, derive_tables, ) @@ -523,6 +523,13 @@ def _signatures(recipe: Recipe, sheet_name: Callable[[SheetRecipe], str]) -> dic total = block.rows.total_row out[f"total_row:{block.id}"] = _sig( None if total is None else (total.label_column, total.pick, total.keep_as)) + # 期 5:穿插的小计、每行属于哪一期 + subt = block.rows.subtotals + out[f"subtotals:{block.id}"] = _sig( + None if subt is None else (subt.label_column, subt.pick, subt.scope, subt.group_column, subt.keep_as)) + out[f"no_subtotals:{block.id}"] = _sig(block.rows.no_subtotals) + period = block.period + out[f"list_period:{block.id}"] = _sig(None if period is None else (period.column, period.synthesize)) for r in recipe.relations: if isinstance(r, SumEq): out[f"relation:{r.id}"] = _sig("sum_eq", r.table, r.total, sorted(r.parts)) @@ -1058,6 +1065,29 @@ def _recipe_items(bag: _Bag, recipe: Recipe, ex: dict[str, Any], facts: Any, *, detail = (f"按明细核对的列:{'、'.join(nums)};其余列合计行里的内容不核对" if nums else "这个块没有数字列,合计行里的内容不核对") bag.add(f"total_row:{b.id}", f"{prefix}改作核对{keep}", detail) + for b in lists: + # 期 5:穿插在明细中间的小计行:不进数据,按各组明细核对。要人确认:这几行从此不在本表里 + subt = b.rows.subtotals + if subt is not None: + rows = rows_text(derived_rows.get(f"{b.id}{SUBTOTAL_KEY}")) + prefix = f"「{b.id}」中间的小计行" + (f"({rows})" if rows else "") + keep = f",原值另存表「{subt.keep_as}」" if subt.keep_as else ",原值不另存" + scope = ("每行小计核对上一行小计之后的全部明细" if subt.scope == "since_previous" + else f"每行小计核对紧挨在它上面、「{subt.group_column}」相同的那几行明细") + nums = [c.name for c in b.columns if c.type in ("INTEGER", "REAL")] + detail = f"{scope};按明细核对的列:{'、'.join(nums)}" if nums else f"{scope};这个块没有数字列,不核对" + bag.add(f"subtotals:{b.id}", f"{prefix}改作核对、不进数据{keep}", detail) + elif b.rows.no_subtotals: + # 声明没有小计行是放宽:以合计字样开头的行从此照常进数据,不再报「像小计」。配上(或改过)的那一次确认 + bag.add(f"no_subtotals:{b.id}", f"「{b.id}」没有小计行:以合计字样开头的行都按数据导入,不再提醒像小计的行", + "如果其实有穿插的小计,对这些列求和会把小计重复算进去") + period = b.period + if period is not None and period.column: + bag.add(f"list_period:{b.id}", f"「{b.id}」每一行的「{period.column}」都要在本期统计期内", + "按期累积时各期的行按这一列区分;有一行不在本期统计期内(或为空),这一期整份拒收") + elif period is not None and period.synthesize: + bag.add(f"list_period:{b.id}", f"「{b.id}」每一行加上本期统计期的起、止两列", + f"列名「{LIST_PERIOD_START}」「{LIST_PERIOD_END}」,取自表内的统计期或人工录入的统计期") # 年份取自统计期:表头是日期格时年份来自格子本身,不算 forms: dict[str, set[str]] = {} @@ -1235,6 +1265,14 @@ def _period_items(bag: _Bag, recipe: Recipe, ex: dict[str, Any], ctx: ConfirmCon bag.add(f"rows_after_stop:{blk}", ";".join(str(p.get("message") or "") for p in probs), "、".join(cells[:20]), source="outside") + # 期 5:没配小计的列表里像小计的数据行:照常导入了,要人确认它们确实是数据(不是的话用修复按钮改作小计核对) + for p in ex.get("problems") or []: + p = plain(p) + if isinstance(p, dict) and p.get("code") == "subtotal_suspect": + blk = str((p.get("fix_args") or {}).get("block") or "、".join(str(c) for c in p.get("cells") or [])) + bag.add(f"subtotal_suspect:{blk}", str(p.get("message") or ""), + "、".join(str(c) for c in (p.get("cells") or [])[:20]), source="outside") + # 列表块只有表头、没有数据行(AU-1):问题对象里没有块键,按消息里的「列表「块」」认 for p in ex.get("problems") or []: p = plain(p) diff --git a/backend/app/data/recipe_engine.py b/backend/app/data/recipe_engine.py index 7c33e010..7f0ac181 100644 --- a/backend/app/data/recipe_engine.py +++ b/backend/app/data/recipe_engine.py @@ -39,6 +39,14 @@ pk_duplicate 同时出现)。块级的定位失败(找不到日期表头、 个矩形。 - 回执里的「排除的行」rows_excluded(H2:排除的行要写进回执):隐藏行、列表跳过的空行、按规则忽略的行、 停止之后没导入的文字行、只核对不另存的合计行。 + +**期 5(下一轮 03 细案 E)又加了两件事,同样没用新字段的配方写出的库逐字节不变:** +- 列表中间穿插的小计(ListRows.subtotals):小计行不进数据,每个数字格一条合计格,按它所管的那组明细(上一行小计 + 之后的全部明细,或紧挨在上面、分组列相同的那几行)交给核对模块用 SQL 求和比对;整行记进排除的行(subtotal)。 +- 列表按期累积时由系统加的统计期起止两列(ListPeriod.synthesize):列表在统计期定下来之前就抽完了,这两格先放 + 占位,统计期定下来以后写库时换成真值(_Sink.fill_period);按日期列分期的(ListPeriod.column)由核对模块的 D + 核对查每行都在本期统计期内。 +CSV 原件经 csv_grid 适配成一张「工作表」,这里照常按网格或逐行流读(xlsx_cells 按 SheetScan.path 分派)。 """ from __future__ import annotations @@ -59,15 +67,16 @@ from app.data import engine as _data_engine from app.data import tabular, xlsx_cells from app.data.names import canon from app.data.recipe_parsers import ( - AxisDate, HourRange, HourTotal, Period, classify_outside, cn_date_range, cn_year_month, filename_period, + TOTAL_WORDS, AxisDate, HourRange, HourTotal, Period, classify_outside, cn_date_range, cn_year_month, + filename_period, hour_range, hour_range_total, is_pure_period_cell, looks_numeric_text, match_key, month_day_or_date, period_residue, text_label, thousands_number, ) from app.data.recipe_types import ( - AxisOut, CanonEntry, CheckResult, ColumnOut, ContextSpec, CrosstabBlock, DerivedItem, DimensionSegment, - ExcludedRows, Extraction, Grid, GridCell, LedgerSheet, ListBlock, MeasuresSegment, OutsideText, + SUBTOTAL_KEY, AxisOut, CanonEntry, CheckResult, ColumnOut, ContextSpec, CrosstabBlock, DerivedItem, + DimensionSegment, ExcludedRows, Extraction, Grid, GridCell, LedgerSheet, ListBlock, MeasuresSegment, OutsideText, PeriodInput, PeriodOut, Problem, Recipe, RegionMark, SegmentLabels, SheetRecipe, TableOut, TableSpec, - derive_tables, + derive_tables, list_period_columns, ) from app.data.xlsx_scan import MAX_MERGED, SheetScan, WorkbookScan, cell_ref, col_letter @@ -101,7 +110,8 @@ PROBLEM_CODES: dict[str, str] = { # 列表定位 "header_not_found": "structure", "header_ambiguous": "structure", "column_missing": "structure", "column_extra": "structure", "rows_after_stop": "confirm", "total_row_missing": "structure", - "total_row_blank": "structure", "list_empty": "confirm", + "total_row_blank": "structure", "list_empty": "confirm", "subtotal_blank": "structure", + "subtotal_suspect": "confirm", # 值 "value_not_integer": "structure", "value_text_number": "structure", "value_blank": "structure", "value_error": "structure", "value_not_number": "structure", "value_not_date": "structure", @@ -125,6 +135,7 @@ PROBLEM_FIX: dict[str, str] = { "axis_extra_cells": "ignore_cells", "column_extra": "ignore_cells", "row_unclaimed": "ignore_cells", "outside_number": "ignore_cells", "hidden_rows": "declare_hidden", "hidden_cols": "declare_hidden", "sheet_missing": "rename_sheet", + "subtotal_suspect": "declare_subtotals", } #: 列表找表头的窗口:从 after_title 的下一行(或已用区域首行)起往下这么多行 @@ -143,7 +154,7 @@ PLACEHOLDER_TEXTS_MAX = 8 PLACEHOLDER_TEXT_LEN = 8 #: 排除的行在回执里的顺序(同一工作表内按原因、块、锚点、首行排) _EXCLUDED_ORDER = ("hidden_excluded", "blank_skipped", "ignored_rows", "ignored_outside", "after_stop", - "total_not_kept") + "total_not_kept", "subtotal") #: 公式区域展开的上限:更大的区域不可能全落在基表里,直接判「引用了基表之外的格」 REF_CELLS_MAX = 200_000 @@ -613,11 +624,18 @@ class _Problems: class _Sink: """一张表的行:主键去重、计数、溯源、批量写库(有库时)。rowid 就是插入序号(新库、1 起)。""" - def __init__(self, eng: _Engine, name: str, cols: list[ColumnOut], spec: TableSpec | None) -> None: + def __init__(self, eng: _Engine, name: str, cols: list[ColumnOut], spec: TableSpec | None, + synth: tuple[str, str] | None = None) -> None: self.eng = eng self.name = name self.cols = cols names = [c.name for c in cols] + #: 期 5:由系统加的统计期起止两列(ListPeriod.synthesize)在一行里的位置。列表是在统计期定下来之前抽完的, + #: 这两格先放占位(_PERIOD_START / _PERIOD_END),统计期定下来以后(fill_period)写库时换成真值;在那之前 + #: 不分批写库(flush 等着),整张表留在内存里 + self.synth_idx = (names.index(synth[0]), names.index(synth[1])) \ + if synth and synth[0] in names and synth[1] in names else None + self.period: tuple[str | None, str | None] | None = None self.grain = [g for g in (spec.grain if spec else []) if g in names] self.grain_idx = [names.index(g) for g in self.grain] self.check_pk = bool(self.grain_idx) and not eng.partial @@ -682,9 +700,22 @@ class _Sink: self.eng.problems.issue("value_blank", sheet, f"工作表「{sheet}」的表「{self.name}」的主键列{_quoted(names)}", coords, text=_PK_NULL, group="pk_null") - def flush(self) -> None: + def fill_period(self, start: str | None, end: str | None) -> None: + """统计期定下来了:加的统计期起止两列从此写真值(start、end 是 YYYY-MM-DD)。""" + self.period = (start, end) + + def flush(self, *, final: bool = False) -> None: + if self.synth_idx is not None and self.period is None and not final: + return # 统计期还没定:加的两列还是占位,先不写 if self.buffer and self.eng.conn is not None: - self.eng.conn.executemany(self.insert_sql, self.buffer) + rows = self.buffer + if self.synth_idx is not None: + a, b = self.synth_idx + start, end = self.period or (None, None) + fill = {a: start, b: end} + rows = [tuple(fill.get(i, v) if v is _PERIOD_START or v is _PERIOD_END else v + for i, v in enumerate(row)) for row in rows] + self.eng.conn.executemany(self.insert_sql, rows) self.buffer.clear() @@ -695,6 +726,12 @@ class _Sink: _FAIL = object() #: 列表数据区里不能填充的合并单元格覆盖的空格(已经报了 merged_in_values) _MERGED = object() +#: 期 5:加的统计期起止两列在统计期定下来之前的占位(_Sink.fill_period)。主键判重时它们和自己相等: +#: 同一期的行统计期都一样,判重只看其余的主键列,结论不变 +_PERIOD_START = object() +_PERIOD_END = object() +#: 小计按组时「还没有组」 +_NO_GROUP = object() # -------------------------------------------------------------------------- @@ -1625,6 +1662,31 @@ class _ListRun: self.ignore_keys = {k for rule in block.ignore_columns if (k := match_key(rule.header))} total = block.rows.total_row self.total_idx = next((i for i, c in enumerate(self.cols) if total and c.name == total.label_column), None) + # 期 5:穿插的小计行(ListRows.subtotals)。sub_first:上一行小计(或表头)之后第一行明细的 rowid;按组小计 + # 时 run_first / run_value 是紧挨在上面、分组列取值相同的那一段明细的首个 rowid 和取值。sub_rows:小计所在的 + # 行 → 它管的明细的 rowid(合计行的公式引用了小计格时,按它换算回明细) + self.sub = block.rows.subtotals + sub = self.sub + self.sub_idx = next((i for i, c in enumerate(self.cols) if sub and c.name == sub.label_column), None) + self.sub_key = match_key(sub.pick) if sub else "" + self.group_idx = next((i for i, c in enumerate(self.cols) + if sub and sub.scope == "group_column" and c.name == sub.group_column), None) + self.sub_first: int | None = None + self.run_first: int | None = None + self.run_value: Any = _NO_GROUP + self.sub_hidden = False + self.sub_rows: dict[int, list[int]] = {} + #: 期 5:加统计期起止两列(ListPeriod.synthesize):每行末尾两格占位,统计期定下来以后由 _Sink 换成真值 + self.synth = block.period is not None and block.period.synthesize is not None + #: 期 5:没配小计时,像夹在明细中间的小计的数据行(行号)和命中的合计字样:照常按数据导入,收尾时报 + #: subtotal_suspect 要人确认(带修复按钮:改作小计核对)。人声明过没有小计(no_subtotals)的不看 + self.suspects: list[int] = [] + self.suspect_word: str | None = None + self.watch_suspects = block.rows.subtotals is None and not block.rows.no_subtotals + #: 上一行普通明细在哪些文字、日期列(列序号,不含标签所在的首列)上有值;还没有明细时是空集 + self.prev_filled: frozenset[int] = frozenset() + #: 等着下一行明细确认的候选:(行号, 合计字样, 上一行明细有值、这一行空着的列) + self.pending: list[tuple[int, str, frozenset[int]]] = [] # ---------------- 喂入 ---------------- @@ -1692,6 +1754,7 @@ class _ListRun: if self.located: for i, text in self.headers_read.items(): self.eng.headers[(self.block.table, self.cols[i].name)] = text + self._report_suspects() # ---------------- 表头 ---------------- @@ -2001,6 +2064,7 @@ class _ListRun: fills = self._merges_at(r, inb) if self.sh.hidden_row(r): self.hidden_seen = True + self.sub_hidden = True if self.sh.sr.hidden.rows == "exclude": self.excluded_rows.add(r) self.sh.layout(r, self.layout_hidden) @@ -2010,10 +2074,85 @@ class _ListRun: label = inb.get(self.colmap.get(self.total_idx, -1)) pick = match_key(block.rows.total_row.pick) # type: ignore[union-attr] if label is not None and match_key(_text(label.value)).startswith(pick): + if self.watch_suspects: + self._settle_suspects(None) self._total_row(r, inb, cells, label) return + if self.sub_idx is not None: + label = inb.get(self.colmap.get(self.sub_idx, -1)) + if label is not None and match_key(_text(label.value)).startswith(self.sub_key): + self._subtotal_row(r, inb, cells, label) + return + elif self.watch_suspects: + self._watch_suspect(r, inb) self._record(r, inb, cells, fills) + def _watch_suspect(self, r: int, inb: dict[int, GridCell]) -> None: + """期 5:没配小计、也没声明没有小计时,认夹在明细中间的小计(照常按数据导入,收尾时报 subtotal_suspect)。 + + 要同时满足才算:首列以合计字样开头、其余文字和日期列都空着、至少一格有数(_suspect_word);上一行明细在某个 + 文字或日期列上有值、这一行那一列空着;下一行明细在同一列上也有值,或者下面紧跟着合计行。所以候选先挂着, + 等下一行明细来了再定(_settle_suspects)。 + + 为什么这么严(复核反例):只有「标签列 + 数字列」的键值清单(「累计收入 300」「汇总调整 5」),「其余文字列 + 为空」自然成立,什么也说明不了;放在最后、后面没有明细的行也不是「夹在中间」。报错了就是每期都要勾的 + 确认项,按配方导入一期的无人值守运行会被它挡住。""" + filled = frozenset(i for i, col in enumerate(self.cols) + if col.type in ("TEXT", "DATE") and (c := self.colmap.get(i)) is not None + and c != self.c1 and c in inb) + word = self._suspect_word(inb) if self.prev_filled else None + if word is not None: + self.pending.append((r, word, self.prev_filled)) + return + self._settle_suspects(filled) + self.prev_filled = filled + + def _settle_suspects(self, filled: frozenset[int] | None) -> None: + """挂着的候选有了下文:filled 是紧跟着的那行明细有值的文字、日期列(None 是紧跟着合计行)。上一行明细有值、 + 候选空着的列,下一行明细也有值,才算夹在明细中间。""" + for row, word, gap in self.pending: + if filled is None or gap & filled: + self.suspects.append(row) + self.suspect_word = self.suspect_word or word + self.pending = [] + + def _suspect_word(self, inb: dict[int, GridCell]) -> str | None: + """期 5:这一行本身像不像小计:首列是以合计字样(TOTAL_WORDS)开头的文字,其余文字、日期列都空着,至少一格 + 有数。像的话返回那个字样。以前这种行静默当数据导入,求和就把小计重复算了进去(H2);现在照常导入,但收尾时 + 报出来要人确认,并给「改作小计核对」的修复按钮。合计行已经在前面认走了。上下文的条件见 _watch_suspect。""" + cell = inb.get(self.c1) + if cell is None or cell.formula is not None or not isinstance(cell.value, str): + return None + key = match_key(cell.value) + word = next((w for w in TOTAL_WORDS if key.startswith(w)), None) + if word is None or not any(self.colmap.get(i) == self.c1 for i in range(len(self.cols))): + return None + numeric = False + for i, col in enumerate(self.cols): + c = self.colmap.get(i) + if c is None or c == self.c1 or c not in inb: + continue + if col.type in ("TEXT", "DATE"): + return None + numeric = True + return word if numeric else None + + def _report_suspects(self) -> None: + """收尾:像小计的数据行合成一条 subtotal_suspect(需确认),修复参数带块 id、标签列、字样、行号和标签格。""" + if not self.suspects or not self.located: + return + label = next(col.name for i, col in enumerate(self.cols) if self.colmap.get(i) == self.c1) + cells = [self.sh.coord(r, self.c1) for r in self.suspects] + rows = _rows_text(self.suspects) + word = self.suspect_word or "" + where = (f"{self.where}:第 {rows} 行像夹在明细中间的小计(「{label}」列以「{word}」开头,上下明细都有值的" + "文字列在这一行空着),按数据导入了") + msg = (f"{where}。如果它们是小计,对这一列求和会把小计重复算进去:请用修复按钮改作小计核对;" + "确实是数据时,确认后照常导入;以后各期不想再确认,在配方的「小计行」里勾选「没有小计行」") + self.eng.problems.add("subtotal_suspect", msg, where, cells[:CELLS_MAX], sheet=self.sh.name, + fix_args={"block": self.block.id, "label_column": label, "pick": word, + "rows": list(self.suspects), "cells": cells[:CELLS_MAX]}) + def _plan(self) -> list[tuple[Any, int, bool, str]]: """每列的(列定义、所在列号、是不是主键、位置说明):定下表头之后算一次,逐行复用。""" plan = self.__dict__.get("_plan_cache") @@ -2073,6 +2212,8 @@ class _ListRun: v = None values.append(v) lineage.append((col.name, r, c)) + if self.synth: + values += [_PERIOD_START, _PERIOD_END] ignored = self.ignored claims = {c: ("ignored_column" if c in ignored else "value") for c in inb} self.sh.layout(r, self.layout_data) @@ -2084,6 +2225,12 @@ class _ListRun: if self.rowid_first is None: self.rowid_first = rowid self.rowid_last = rowid + if self.sub_first is None: + self.sub_first = rowid + if self.group_idx is not None: + group = values[self.group_idx] + if self.run_value is _NO_GROUP or group != self.run_value: + self.run_first, self.run_value = rowid, group self.emit(r, cells, claims, True) def _total_row(self, r: int, inb: dict[int, GridCell], cells: list[tuple[int, GridCell]], @@ -2142,7 +2289,67 @@ class _ListRun: self.stop = ("total", r) self.emit(r, cells, claims, True) - def _ref_lookup(self, rc: tuple[int, int]) -> str | int: + def _subtotal_row(self, r: int, inb: dict[int, GridCell], cells: list[tuple[int, GridCell]], + label: GridCell) -> None: + """期 5:穿插在明细中间的一行小计。不进数据:标签格和数字格的去向是 total_label / total_value(同合计行), + 每个数字格一条合计格(DerivedItem,kind=column_sum,分段键是块 id + SUBTOTAL_KEY),核对模块按它所管的那组 + 明细(rowid 闭区间)用 SQL 逐列求和比对,执行器不做加法(H1);整行记进排除的行(reason=subtotal)。配了 + keep_as 的另存一行(小计项、按组时的分组取值、各数字列)。 + + 它管哪几行:since_previous 是上一行小计(或表头)之后的全部明细;group_column 是其中紧挨在它上面、分组列 + 取值相同的那一段。一行明细都没有时区间为空,核对判「无法核对」(合计行之上没有明细行)。数字格全空的小计 + 没有一列能核对,报 subtotal_blank(结构类):不报的话这一行就悄悄没了核对。""" + eng, sh, block, sub = self.eng, self.sh, self.block, self.sub + assert sub is not None + claims = {c: ("ignored_column" if c in self.ignored else "total_label") for c in inb} + label_raw = _text(label.value) + first = self.sub_first + if sub.scope == "group_column" and first is not None and self.run_first is not None: + first = max(first, self.run_first) + last = self.rowid_last if first is not None else None + numeric = [(i, col) for i, col in enumerate(self.cols) if col.type in ("INTEGER", "REAL")] + if numeric and not any(self.colmap[i] in inb for i, _ in numeric): + coord = sh.coord(r, self.colmap[self.sub_idx]) # type: ignore[index] + where = f"{self.where}:小计行({coord.split('!')[-1]})的数字格全部为空" + eng.problems.add( + "subtotal_blank", + f"{where},无法按这一组明细核对。请在文件中补上小计后重新上传;原表不再有小计时,在配方面板的" + "「小计行」中取消「有小计行」", + f"{where},无法核对", [coord], sheet=sh.name) + key = f"{block.id}{SUBTOTAL_KEY}" + keep_values: list[Any] = [] + for i, col in numeric: + c = self.colmap[i] + cell = inb.get(c) + if cell is None: + keep_values.append(None) + continue + claims[c] = "total_value" + got = eng.convert_number(cell, col.type, block.values, sh.name, f"{self.where}的小计行", r, c, + data_area=False) + keep_values.append(got) + item = DerivedItem( + kind="column_sum", segment=key, sheet=sh.name, cell=sh.coord(r, c), label_raw=label_raw, + label=canon(label_raw), value=None if got is _FAIL else got, is_formula=cell.formula is not None, + formula=cell.formula, base_table=block.table, base_value=col.name, rowid_first=first, + rowid_last=last, hidden_in_range=self.sub_hidden, keep_table=sub.keep_as) + if item.is_formula: + eng.resolve_refs(item, cell.formula or "", self._ref_lookup) + eng.ex.derived.append(item) + if sub.keep_as and sub.keep_as in eng.sinks: + ks = eng.sinks[sub.keep_as] + ks.sources.append(f"{sh.sr.id}/{block.id}") + group = [None if self.run_value is _NO_GROUP else self.run_value] if self.group_idx is not None else [] + lin = [(col.name, r, self.colmap[i]) for i, col in numeric] + ks.add([canon(label_raw), *group, *keep_values], sheet=sh.name, + at=(r, self.colmap[self.sub_idx]), cells=lin, fails=True) # type: ignore[index] + n = sum(1 for role in claims.values() if role in ("total_label", "total_value")) + sh.exclude("subtotal", r, r, n, block=block.id, anchor=sub.pick) + self.sub_rows[r] = list(range(first, last + 1)) if first is not None and last is not None else [] + self.sub_first, self.run_first, self.run_value, self.sub_hidden = None, None, _NO_GROUP, False + self.emit(r, cells, claims, True) + + def _ref_lookup(self, rc: tuple[int, int]) -> str | int | list[int]: r, c = rc if self.c1 <= c <= self.c2: rowid = self.row_rowid.get(r) @@ -2150,6 +2357,9 @@ class _ListRun: return rowid if r in self.excluded_rows: return "hidden" + if r in self.sub_rows: + # 合计行的公式引用了小计格(=C5+C9):小计格代表它管的那组明细 + return self.sub_rows[r] return "outside" @@ -2343,7 +2553,8 @@ class _Engine: self.canon_log[key] = CanonEntry(table, col.name, text, stored, 1, _coord(sheet, r, c)) return stored - def resolve_refs(self, item: DerivedItem, formula: str, lookup: Callable[[tuple[int, int]], str | int]) -> None: + def resolve_refs(self, item: DerivedItem, formula: str, + lookup: Callable[[tuple[int, int]], str | int | list[int]]) -> None: """合计公式的引用经溯源:全在基表 → ref_rowids;有隐藏行或分类汇总函数 → hidden; 有不在基表的 → outside;形状认不出 → unrecognized。执行器不做加法(H1)。""" shape = formula_refs(formula) @@ -2357,7 +2568,9 @@ class _Engine: outside = not cells for rc in cells: hit = lookup(rc) - if isinstance(hit, int): + if isinstance(hit, list): + rowids.extend(hit) # 期 5:引用了小计格,换算成它管的明细 + elif isinstance(hit, int): rowids.append(hit) elif hit == "hidden": hidden = True @@ -2607,8 +2820,9 @@ class _Engine: self.ex.problems = self.problems.finish(set()) return self.ex self.cols = cols + synth = {t: (spec[1], spec[2]) for t, spec in list_period_columns(self.recipe).items() if spec[0] == "synthesize"} for name, cs in cols.items(): - self.sinks[name] = _Sink(self, name, cs, self.specs.get(name)) + self.sinks[name] = _Sink(self, name, cs, self.specs.get(name), synth.get(name)) if self.db_path is not None: _remove_db(self.db_path) self.conn = _open_sqlite(self.db_path, readonly=False) @@ -2622,6 +2836,11 @@ class _Engine: crosstabs.extend(self._phase1(sh)) t2 = time.perf_counter() period, period_reported, _ = self._period() + if self.ex.period is not None: + # 期 5:列表加的统计期起止两列现在才有值(列表在统计期定下来之前就抽完了) + for sink in self.sinks.values(): + if sink.synth_idx is not None: + sink.fill_period(self.ex.period.start, self.ex.period.end) for ct in crosstabs: ct.complete_axis(period, period_reported) ct.extract() @@ -2650,7 +2869,7 @@ class _Engine: _remove_db(self.db_path) # type: ignore[arg-type] else: for sink in self.sinks.values(): - sink.flush() + sink.flush(final=True) self.conn.execute("COMMIT") self.conn.close() self.conn = None @@ -2748,7 +2967,9 @@ class _Engine: sh.failed = True return [] b = s.bounds - if b is not None and not self.partial and b.area > tabular.SPARSE_FACTOR * max(s.nonempty, tabular.SPARSE_FLOOR): + # 稀疏判定挡的是 Excel 里误输入、残留格式撑大的已用区域;CSV 的每一格都是写出来的,空字段多只是数据稀疏 + if b is not None and not self.partial and self.scan.format != "csv" \ + and b.area > tabular.SPARSE_FACTOR * max(s.nonempty, tabular.SPARSE_FLOOR): far = "、".join(list(dict.fromkeys(s.far_cells))[:3]) msg = (f"工作表「{sh.name}」的已用区域 {b.a1()} 共 {b.area:,} 个单元格,但只有 {s.nonempty:,} 个单元格有内容;" f"远处的单元格(如 {far})可能是误输入或残留格式。请在 Excel 中删除这些单元格后重新上传") diff --git a/backend/app/data/recipe_fixes.py b/backend/app/data/recipe_fixes.py index 54f6be61..5921ebfb 100644 --- a/backend/app/data/recipe_fixes.py +++ b/backend/app/data/recipe_fixes.py @@ -253,6 +253,8 @@ def _key(kind: str, target: dict[str, Any]) -> str: return f"declare_hidden:{t['sheet']}:{t['axis']}" if kind == "rename_sheet": return f"rename_sheet:{t['sheet_id']}" + if kind == "declare_subtotals": + return f"declare_subtotals:{t['block']}" raise KeyError(kind) @@ -750,6 +752,62 @@ def _o_declare_total(recipe: dict, p: FixProposal, opt: FixOption, reason: str | return ops, summary +# ========================================================================== +# 期 5:列表中间像小计的行改作小计核对 +# ========================================================================== + + +def _subtotal_keep_name(recipe: dict, block: dict) -> str: + used = {collide_key(t.get("name") or "") for t in recipe.get("tables") or []} + return unique_name(f"{block.get('table')}_小计", used) + + +def _p_declare_subtotals(recipe: dict, prob: Any, args: dict, anchor: FixAnchor) -> FixProposal | None: + """subtotal_suspect → 列表块加 rows.subtotals(标签列、开头词取自执行器给的 fix_args:文件里的合计字样)。 + 块已经配了小计、标签列不在块里、开头词含数字时不提议。""" + found = find_block(recipe, args.get("block")) + if found is None: + return None + _si, _bi, block = found + label, pick = args.get("label_column"), args.get("pick") + if block.get("layout") != "list" or (block.get("rows") or {}).get("subtotals"): + return None + if label not in [c.get("name") for c in block.get("columns") or []] or not isinstance(pick, str) \ + or not pick or has_digit(pick): + return None + target = {"block": block["id"], "label_column": label, "pick": pick} + keep = _subtotal_keep_name(recipe, block) + return _proposal("declare_subtotals", _key("declare_subtotals", target), code=_get(prob, "code"), + title=f"列表「{block['id']}」中间「{pick}」开头的行是小计", + cells=list(args.get("cells") or _get(prob, "cells") or []), target=target, + options=[FixOption("check_only", "改作小计核对,不另存", + detail="每行小计按上一行小计之后的明细逐列求和核对,小计行不进数据,记进回执的" + "「排除的行」"), + FixOption("keep", f"改作小计核对,原值另存表「{keep}」", + detail="同上,原表的小计另存成一张表;另存的小计表不能按期累积")], + anchor=anchor) + + +def _o_declare_subtotals(recipe: dict, p: FixProposal, opt: FixOption, reason: str | None) -> tuple[list, list[str]]: + found = find_block(recipe, p.target.get("block")) + if found is None: + raise _Stale + si, bi, block = found + if (block.get("rows") or {}).get("subtotals"): + raise _Stale + keep = _subtotal_keep_name(recipe, block) if opt.value == "keep" else None + value = {"label_column": p.target["label_column"], "pick": p.target["pick"], "scope": "since_previous", + "group_column": None, "keep_as": keep} + ops: list[dict[str, Any]] = [{"op": "add", "path": ptr("sheets", si, "blocks", bi, "rows", "subtotals"), + "value": value}] + if keep: + ops.append({"op": "add", "path": "/tables/-", "value": { + "name": keep, "grain": [], "kind": "reported_total", "units": {}, "note": ""}}) + summary = [f"列表「{block['id']}」中间「{p.target['pick']}」开头的行改作小计核对,不进数据" + + (f",原值另存表「{keep}」" if keep else ",不另存")] + return ops, summary + + # ========================================================================== # ⑥ 忽略(按文字记) # ========================================================================== @@ -998,12 +1056,13 @@ _Builder = Callable[[dict, Any, dict, FixAnchor], "FixProposal | None"] _BUILDERS: dict[str, _Builder] = { "remove_label": _p_remove_label, "add_label": _p_add_label, "rename_title": _p_rename_title, "declare_total": _p_declare_total, "ignore_cells": _p_ignore_cells, "declare_placeholder": _p_declare_placeholder, - "declare_hidden": _p_declare_hidden, "rename_sheet": _p_rename_sheet, + "declare_hidden": _p_declare_hidden, "rename_sheet": _p_rename_sheet, "declare_subtotals": _p_declare_subtotals, } _OPS: dict[str, Callable[[dict, FixProposal, FixOption, str | None], tuple[list, list[str]]]] = { "remove_label": _o_remove_label, "add_label": _o_add_label, "edit_members": _o_edit_members, "rename_title": _o_rename_title, "declare_total": _o_declare_total, "ignore_cells": _o_ignore_cells, "declare_placeholder": _o_declare_placeholder, "declare_hidden": _o_declare_hidden, "rename_sheet": _o_rename_sheet, + "declare_subtotals": _o_declare_subtotals, } @@ -1064,6 +1123,8 @@ def _edit_title(p: FixProposal, opt: FixOption) -> str: return f"工作表「{t.get('sheet')}」:{opt.label}" if p.kind == "rename_sheet": return opt.label + if p.kind == "declare_subtotals": + return f"列表「{t.get('block')}」中间「{t.get('pick')}」开头的行{opt.label}" return p.title diff --git a/backend/app/data/recipe_imports.py b/backend/app/data/recipe_imports.py index 40dba2b1..ee8d338c 100644 --- a/backend/app/data/recipe_imports.py +++ b/backend/app/data/recipe_imports.py @@ -177,7 +177,8 @@ class Pipeline: recipe_sha256: Callable[[Recipe], str] canonical: Callable[[Recipe], dict[str, Any]] apply_patch: Callable[[dict[str, Any], list[dict[str, Any]]], dict[str, Any]] - scan: Callable[[bytes], WorkbookScan] + #: (原件, 文件名) → 扫描结果。文件名定按 CSV 读还是按工作簿读(csv_grid.scan_any) + scan: Callable[[bytes, str], WorkbookScan] scan_to_json: Callable[[WorkbookScan], dict[str, Any]] scan_from_json: Callable[[dict[str, Any]], WorkbookScan] read_grid: Callable[..., Grid] @@ -225,7 +226,7 @@ class Pipeline: def default_pipeline() -> Pipeline: """接上真模块。函数体里惰性导入:这些模块(openpyxl、执行器、起草器)导入不便宜,接口模块加载时不必付。""" from app.data import ( - recipe, recipe_accumulate, recipe_ai, recipe_checks, recipe_compare, recipe_confirm, recipe_diff, + csv_grid, recipe, recipe_accumulate, recipe_ai, recipe_checks, recipe_compare, recipe_confirm, recipe_diff, recipe_engine, recipe_fixes, recipe_notes, recipe_select, recipe_suggest, xlsx_cells, xlsx_scan, ) @@ -234,7 +235,8 @@ def default_pipeline() -> Pipeline: recipe_sha256=recipe.recipe_sha256, canonical=recipe.canonical_recipe, apply_patch=recipe.apply_patch, - scan=xlsx_scan.scan, + # CSV / TSV 也走配方:csv_grid 适配成一张「工作表」,其余照旧交给 xlsx_scan(Excel 期 5) + scan=csv_grid.scan_any, scan_to_json=xlsx_scan.scan_to_json, scan_from_json=xlsx_scan.scan_from_json, read_grid=xlsx_cells.read_grid, @@ -473,7 +475,7 @@ def _open_book(p: Pipeline, raw: bytes, scan: WorkbookScan) -> _Book: def _book_of(p: Pipeline, snap: _Snap) -> _Book: raw = _read_raw(snap.raw_sha256) - scan = p.scan_from_json(snap.scan) if snap.scan else p.scan(raw) + scan = p.scan_from_json(snap.scan) if snap.scan else p.scan(raw, snap.file_name) return _open_book(p, raw, scan) @@ -771,7 +773,7 @@ class _Staged: def _stage_work(p: Pipeline, raw: bytes, filename: str) -> _Staged: """首次导入(或切换)的暂存:扫描、网格预览、规则起草(带干跑)、对草稿做一次静态校验。""" - scan = p.scan(raw) + scan = p.scan(raw, filename) book = _open_book(p, raw, scan) dry = _Capture(_dry_fn(p, book, filename, {})) d = p.draft(scan, book.grids, filename, validate=lambda data, f, o: p.validate(data, facts=f, origin=o), @@ -859,7 +861,7 @@ async def stage_upload(session: AsyncSession, *, raw: bytes, filename: str, name def _replay_work(p: Pipeline, raw: bytes, current: dict[str, Any], filename: str, snap: _Snap, *, evaluate: bool) -> tuple[_Staged, _Eval | None]: """上传新一期 / 修改配方:扫描(或取存下的扫描)、按现行配方的写法取系统发现,修改配方时再评估一遍。""" - scan = p.scan_from_json(snap.scan) if snap.scan else p.scan(raw) + scan = p.scan_from_json(snap.scan) if snap.scan else p.scan(raw, filename) book = _open_book(p, raw, scan) facts = p.detect_facts(book.grids, scan, recipe=current) staged = _Staged(scan_json=p.scan_to_json(scan), previews=[p.preview(g) for g in book.grids.values()], @@ -1859,7 +1861,7 @@ def _trial_work(p: Pipeline, snap: _Snap, work: dict[str, Any], facts: DraftFact if parsed is None or problems: return _TrialRun(parsed=parsed, problems=list(problems)) raw = _read_raw(snap.raw_sha256) - scan = p.scan_from_json(snap.scan) if snap.scan else p.scan(raw) + scan = p.scan_from_json(snap.scan) if snap.scan else p.scan(raw, snap.file_name) path.parent.mkdir(parents=True, exist_ok=True) # 先写 .tmp- 再改名:半成品归启动清理(_sweep_tmp)管,发布、回收不会把半个库当成试运行库 tmp = path.parent / f"{path.name}{raw_store.TMP_MARK}{secrets.token_hex(6)}" @@ -2361,7 +2363,7 @@ def _manifest(*, staging_id: str, kind: str, source_id: str, info: table_version ex_dict: dict[str, Any], file: dict[str, Any], recipe_meta: dict[str, Any], checks: list[dict[str, Any]], overrides: list[dict[str, Any]], waivers: list[dict[str, Any]], confirmed: list[dict[str, Any]], notes: SchemaNotes, ai: dict[str, Any], signed_by: str | None, - edits: list[dict[str, Any]] | None = None) -> dict[str, Any]: + edits: list[dict[str, Any]] | None = None, signer_verified: bool = False) -> dict[str, Any]: receipt = copy.deepcopy(ex_dict) receipt.pop("problems", None) return { @@ -2383,7 +2385,8 @@ def _manifest(*, staging_id: str, kind: str, source_id: str, info: table_version "ai": ai, # 期 3:修复、框选的修改记录(含已被整份替换覆盖的,作为历史;不带撤销用的修改前起点) "edits": copy.deepcopy(list(edits or [])), - "signed_by": {"name": signed_by, "verified": False}, + # 是否认证照实记(05-已知遗留 (c)):开了登录、署名是登录的人时为真。以前写死 False,封存的证据记错了事实 + "signed_by": {"name": signed_by, "verified": bool(signer_verified and signed_by)}, "staging_id": staging_id, "kind": kind, "created_at": _now_iso(), } @@ -2411,7 +2414,8 @@ def _file_sha(path: Path) -> str | None: async def commit_staging(session: AsyncSession, staging: ImportStaging, *, trial_id: str, confirmations: list[str], - acceptances: list[Acceptance], signed_by: str | None) -> CommitResult: + acceptances: list[Acceptance], signed_by: str | None, + signer_verified: bool = False) -> CommitResult: """POST imports/{id}/commit(7.8):预检 → (与现行导入相同时)不新建记录 → 发布构建、写配方、导入记录、 快照、清单,切指针。失败时指针不动,暂存区退回 drafting(试运行库已被发布流程消耗)。 @@ -2532,6 +2536,8 @@ async def commit_staging(session: AsyncSession, staging: ImportStaging, *, trial ex = extraction_from_dict(ex_dict) who = signed_by or staging.signed_by + # 署名是这次请求的(signed_by)才谈得上认证;退回暂存区上记的那个,是否认证这里不知道,照实记未认证 + who_verified = bool(signer_verified and signed_by) acc = [Acceptance(check_id=a.check_id, reason=a.reason.strip(), signed_by=who) for a in given.values()] # 这一期自己的说明(带实际的接受)进它的导入清单:版本页逐期显示「改写后的那一句」从这里取(第 5 节)。 # 快照冻结的说明在多期时另按各期最弱的结论合并(2.8),只剩本期时就是它 @@ -2603,7 +2609,8 @@ async def commit_staging(session: AsyncSession, staging: ImportStaging, *, trial file={"name": file_name, "size": file_size, "raw_sha256": raw_sha}, recipe_meta={"id": recipe_id, "sha256": recipe_sha, "origin": origin, "canonical": canonical}, checks=checks_full, overrides=overrides, waivers=waivers, confirmed=confirmed, - notes=notes if union_info is None else period_notes, ai=ai, signed_by=who, edits=edits) + notes=notes if union_info is None else period_notes, ai=ai, signed_by=who, edits=edits, + signer_verified=who_verified) async def schema_cache_for(view: table_versions.SourceView, info: table_versions.PublishInfo) -> dict[str, Any]: from app.core import artifact_store @@ -2663,7 +2670,7 @@ async def commit_staging(session: AsyncSession, staging: ImportStaging, *, trial "table_hashes": dict(union_report.get("table_hashes") or {}), "union_ver": recipe_types.UNION_VER}, parts=ordered, report=union_report, label_sets=list((plan or {}).get("label_sets") or []), gaps=list((plan or {}).get("gaps") or []), replaced=(plan or {}).get("replaces"), removed=None, - notes=notes, signed_by=who) + notes=notes, signed_by=who, signer_verified=who_verified) smid = await artifact_store.put_json(doc, kind="snapshot_manifest", meta={"source_id": source_id, "snapshot_id": info.snapshot_id}) cache["import_manifests"] = [mid if iid is None else by_imp[iid].info.manifest_artifact diff --git a/backend/app/data/recipe_notes.py b/backend/app/data/recipe_notes.py index b4464cf7..48c8da20 100644 --- a/backend/app/data/recipe_notes.py +++ b/backend/app/data/recipe_notes.py @@ -27,6 +27,9 @@ from app.data.names import canon from app.data.recipe_checks import date_columns, plan_checks, split_coord from app.data.recipe_parsers import UNITS from app.data.recipe_types import ( + LIST_PERIOD_END, + LIST_PERIOD_START, + SUBTOTAL_KEY, Acceptance, CheckResult, ColumnOut, @@ -46,6 +49,7 @@ from app.data.recipe_types import ( TableNote, UnionNotePart, derive_tables, + list_period_columns, prose_problems, render_prose, ) @@ -134,6 +138,13 @@ NOTE_TEXT = { "ignored_rows_union": "部分期的原表另有按配方忽略的行,内容见证据面板", "ignored_cells_union": "部分期的原表另有按配方忽略的单元格,内容见证据面板", "dim_values_vary": "各期的 {dim} 取值不完全相同:跨期比较前请先确认两期都有的取值", + # ---- 期 5:列表按期累积、穿插的小计。列表可以累积以后,空行之后的文字行也要有并集说法 + "rows_after_stop_union": "部分期的原表在空行之后另有未导入的文字行,内容见证据面板", + "subtotal_kept": "原表穿插的小计在 {table},不要与本表相加", + "subtotal_not_kept": "原表穿插的小计行未导入本表", + "list_subtotal_passed": "原表穿插的小计行的值,已按各组明细求和核对一致:不要与 {base} 相加,也不要彼此相加", + "list_subtotal_unverified": "原表穿插的小计行的值,按原表保存的值存储。本期部分小计未能核对:不要与 {base} 相加," + "也不要彼此相加", } #: 列说明的片段模板 @@ -148,6 +159,9 @@ COLUMN_TEXT = { "col_partial_periods": "部分期没有这一列的数据,为空值", "col_absent": "当前版本的数据中没有这一列,为空值", "col_retired": "这一列自某一期起不再导入,此后各期为空值", + # ---- 期 5:列表按期累积时由系统加的统计期起止两列(ListPeriod.synthesize)。不含数字(说明不许报数) + "period_start": "本期统计期的第一天,取自表内的统计期或人工录入的统计期,格式 YYYY-MM-DD", + "period_end": "本期统计期的最后一天,取自表内的统计期或人工录入的统计期,格式 YYYY-MM-DD", } #: 「逐日 / 逐行」「个别日期 / 个别行」:粒度只有一列日期时按日说,否则按行说 @@ -161,7 +175,7 @@ _K_REASON = {r: f"total_k_{r}" for r in ("uncached", "blank", "null_detail", "ti _T_REASON = {r: f"list_total_{r}" for r in ("uncached", "blank", "null_detail", "tiling", "hidden")} #: 允许出现「一致」的模板:只在对应的 K / T 通过时才会选用 CONSISTENT_KEYS = frozenset({"total_k_passed", "total_k_passed_g_unverifiable", "list_total_passed", - "list_total_passed_partial"}) + "list_total_passed_partial", "list_subtotal_passed"}) def _tok(kind: str, name: str) -> str: @@ -202,6 +216,8 @@ class _Writers: lists: list[ListBlock] = field(default_factory=list) #: 把合计行另存进本表的列表块 list_totals: list[ListBlock] = field(default_factory=list) + #: 期 5:把穿插的小计另存进本表的列表块 + list_subtotals: list[ListBlock] = field(default_factory=list) def add_sheet(self, sheet: SheetRecipe) -> None: if all(s is not sheet for s in self.sheets): @@ -242,6 +258,9 @@ def _writers(recipe: Recipe) -> dict[str, _Writers]: total = block.rows.total_row if total is not None and total.keep_as: w(total.keep_as, sheet).list_totals.append(block) + sub = block.rows.subtotals + if sub is not None and sub.keep_as: + w(sub.keep_as, sheet).list_subtotals.append(block) return out @@ -395,6 +414,15 @@ def _base_total_fragments(w: _Writers) -> list[NoteFragment]: b.rows.total_row is not None and not b.rows.total_row.keep_as for b in w.lists) if dropped: out.append(NoteFragment("total_not_kept", None, NOTE_TEXT["total_not_kept"])) + # 期 5:穿插的小计行不在本表里:另存在哪,或者没导入 + for b in w.lists: + sub = b.rows.subtotals + if sub is None: + continue + if sub.keep_as: + _add_unique(out, _frag("subtotal_kept", sub.keep_as, NOTE_TEXT["subtotal_kept"].format(table=_tab(sub.keep_as)))) + elif all(f.key != "subtotal_not_kept" for f in out): + out.append(NoteFragment("subtotal_not_kept", None, NOTE_TEXT["subtotal_not_kept"])) return out @@ -450,6 +478,13 @@ def _kept_total_fragments(ctx: _Ctx, table: str, w: _Writers) -> list[NoteFragme else: frag = _frag(key, table, NOTE_TEXT[key].format(base=_tab(block.table))) _add_unique(out, frag) + for block in w.list_subtotals: + # 期 5:另存的小计:T 通过才说「核对一致」,没核对上的(无法核对)照实说部分未能核对;不一致的整份拒收 + t = ctx.check(table, ctx.plan.t.get(f"{block.id}{SUBTOTAL_KEY}"), f"块「{block.id}」的小计行") + if t is None or _blocking(t): + continue + key = "list_subtotal_passed" if t.status == "passed" else "list_subtotal_unverified" + _add_unique(out, _frag(key, table, NOTE_TEXT[key].format(base=_tab(block.table)))) return out @@ -653,10 +688,17 @@ def _date_key(ctx: _Ctx, w: _Writers, table: str, col: ColumnOut) -> str | None: return None +def _period_column_key(col: ColumnOut) -> str | None: + """期 5:系统加的统计期起止两列(role=period)用哪条列说明。""" + if col.role != "period": + return None + return "period_start" if col.name == LIST_PERIOD_START else "period_end" if col.name == LIST_PERIOD_END else None + + def _null_eligible(ctx: _Ctx, w: _Writers, table: str, col: ColumnOut) -> bool: """这一列要不要考虑空值说明:值列;主键不会是空值;表内合计表的空值已由表说明(未能核对)交代。""" value_col = col.role in ("measure", "value") and col.type in ("INTEGER", "REAL") - is_total = bool(w.kept_from or w.list_totals) + is_total = bool(w.kept_from or w.list_totals or w.list_subtotals) return value_col and not is_total and col.name not in set(ctx.grains.get(table, [])) @@ -668,6 +710,8 @@ def _column_templates(ctx: _Ctx, table: str, null_counts: Mapping[str, Mapping[s frags: list[str] = [] if (date := _date_key(ctx, w, table, col)) is not None: frags.append(COLUMN_TEXT[date]) + if (period := _period_column_key(col)) is not None: + frags.append(COLUMN_TEXT[period]) if col.unit: frags.append(COLUMN_TEXT["unit"].format(unit=_tok("单位", col.unit))) has_nulls = counts is None or counts.get(col.name, 0) > 0 @@ -793,22 +837,27 @@ _FAMILY: dict[str, str] = { "hidden_excluded": "hidden_excluded", "hidden_included": "hidden_included", "ignored_columns": "ignored_columns", "ignored_rows": "ignored_rows", "ignored_cells": "ignored_cells", "rows_after_stop": "rows_after_stop", + "subtotal_kept": "subtotal_kept", "subtotal_not_kept": "subtotal_not_kept", + "list_subtotal_passed": "list_total", "list_subtotal_unverified": "list_total", "period_conflict_accepted": "period_conflict", "period_conflict_pending": "period_conflict", "period_human": "period_human", "outside_digits": "outside_digits", "note": "note", } #: 合并后的拼接顺序(与 _table_fragments 一致;累积特有的几句排在用户写的表说明之前) -_FAMILY_ORDER = ("grain", "sum_eq", "not_comparable", "hour_range", "const", "total_kept", "total_not_kept", "total_k", - "list_total", "total_overlap", "hidden_excluded", "hidden_included", "ignored_columns", +_FAMILY_ORDER = ("grain", "sum_eq", "not_comparable", "hour_range", "const", "total_kept", "total_not_kept", + "subtotal_kept", "subtotal_not_kept", "total_k", "list_total", "total_overlap", "hidden_excluded", "hidden_included", "ignored_columns", "ignored_rows", "ignored_cells", "rows_after_stop", "period_conflict", "period_human", "outside_digits", "union", "note") #: 与核对结论无关、只描述表结构的片段族:用目标配方那一期的;只在早期出现的 subject(退役的表、另存表)用 #: 最后一个含它的那一期的 -_STRUCTURAL = frozenset({"grain", "hour_range", "const", "total_kept", "total_not_kept", "total_overlap", "note"}) +_STRUCTURAL = frozenset({"grain", "hour_range", "const", "total_kept", "total_not_kept", "total_overlap", "note", + "subtotal_kept", "subtotal_not_kept"}) #: 「任一期出现就写」的片段族 → 并集里的说法 _ANY_UNION = {"hidden_excluded": "hidden_excluded_union", "hidden_included": "hidden_included_union", "ignored_columns": "ignored_columns_union", "ignored_rows": "ignored_rows_union", - "ignored_cells": "ignored_cells_union", "outside_digits": "outside_digits_union"} + "ignored_cells": "ignored_cells_union", "outside_digits": "outside_digits_union", + # 期 5:列表可以按期累积了,空行之后的文字行任一期有就写 + "rows_after_stop": "rows_after_stop_union"} _K_PASSED = frozenset({"total_k_passed", "total_k_passed_g_unverifiable"}) @@ -943,20 +992,26 @@ def _merge_group(table: str, family: str, subject: str | None, group: dict[int, if family == "period_human": key = "period_human_all" if len(present) == len(having) else "period_human_union" return NoteFragment(key, subject, NOTE_TEXT[key]) - # list_total、rows_after_stop 只有列表才有,而列表不能按期累积(P3-SPEC 2.2);不认识的片段族同样没有并集说法。 + # list_total 只在列表的合计行、小计行另存的表上,这种表不能按期累积(accumulate_blockers);不认识的片段族同样 + # 没有并集说法。 # 记 problems(不能保存),绝不沿用某一期的单期说法。消息给人看,不露片段族的键名 problems.append(f"表「{table}」:{_UNMERGEABLE.get(family, '有一段说明')}不能按期合并,无法生成说明") return None #: 没有并集说法的片段族 → 给人看的叫法(problems 里不露键名) -_UNMERGEABLE = {"list_total": "列表合计行的核对结论", "rows_after_stop": "列表空行之后的文字行"} +_UNMERGEABLE = {"list_total": "列表合计行、小计行的核对结论"} def _axis_column(recipe: Recipe, table: str) -> str | None: + """各期按哪一列区分(说明里「请按 {col} 筛选」):交叉表的日期轴;期 5 起列表按 period 绑定的日期列,或者 + 系统加的统计期起那一列。""" tables, _ = derive_tables(recipe) axes = [c.name for c in tables.get(table, []) if c.role == "axis"] - return axes[0] if len(axes) == 1 else None + if len(axes) == 1: + return axes[0] + spec = list_period_columns(recipe).get(table) + return spec[1] if spec else None def _union_table_fragments(table: str, having: list[int], parts: list[_UPart], ref: int, target: Recipe, @@ -1037,6 +1092,8 @@ def _union_column_templates(table: str, columns: list[ColumnOut], having: list[i frags.append(COLUMN_TEXT["date_year_from_period"]) elif "date" in dates or (not with_col and col.role == "axis"): frags.append(COLUMN_TEXT["date"]) + if (period := _period_column_key(col)) is not None: + frags.append(COLUMN_TEXT[period]) if target_cols is not None and col.name in target_cols: unit = target_cols[col.name].unit elif with_col: diff --git a/backend/app/data/recipe_select.py b/backend/app/data/recipe_select.py index daa7332c..34697950 100644 --- a/backend/app/data/recipe_select.py +++ b/backend/app/data/recipe_select.py @@ -427,6 +427,21 @@ def _bottom_box(ctx: _Ctx, plan: Any, first: int) -> bool: ctx.fail("selection_bottom_not_anchorable", f"框里只有表头和合计行(第 {last} 行),没有数据行:请把框扩大到数据所在的行") return False + # 期 5:框里夹在中间、第一列以合计字样开头、下面紧接着还有同一张表的明细的行(起草器 _scan_block 的同一条 + # 规则),照起草器提议成小计(_settle_subtotals);提议不成的照旧报错 + mid = [r for r in data[1:] if S._total_word(S._label_of(grid.get(r, c1))) is not None] + rest = [r for r in data if r not in mid] + if mid and rest and not any(grid.get(r, c) is not None and S._total_word(grid.get(r, c).value) is not None + for r in data for c in range(c1 + 1, c2 + 1)): + trial = S._ListPlan(header_rows=plan.header_rows, cols=plan.cols, headers=plan.headers, rows=rest, + subtotal_rows=list(mid), + subtotal_words=[S._total_word(S._label_of(grid.get(r, c1))) or "" for r in mid]) + types = S._types_of(grid, rest, plan.cols) + if all(S._more_rows(grid, r, trial, S._bottom(grid), types) for r in mid): + S._settle_subtotals(grid, trial) + if trial.subtotal_word is not None: + plan.subtotal_rows, plan.subtotal_word = trial.subtotal_rows, trial.subtotal_word + data = [r for r in data if r not in mid] if not _no_stray_totals(ctx, data, c1, c2): return False bottom = S._bottom(grid) diff --git a/backend/app/data/recipe_suggest.py b/backend/app/data/recipe_suggest.py index 53314072..9a176511 100644 --- a/backend/app/data/recipe_suggest.py +++ b/backend/app/data/recipe_suggest.py @@ -12,6 +12,11 @@ validate)和执行器干跑(注入的 dry_run),再由人逐条确认之 哪些是区域外文字、哪些格没有去处)直接用执行器的结论:同一件事只有执行器一套判法,这里自己定位只是没有 干跑时的近似。 +**期 5(下一轮 03 细案 E)。** 有 Excel 表格对象(插入 → 表格)的工作表优先按表格对象出列表块(_lists_from_tables: +表头、列范围、汇总行都是表格对象给的,锚点仍按表头文字);夹在明细中间、第一列以同一个合计字样开头的行提议成 +穿插的小计(_settle_subtotals),不再一律记起草失败;每个列表都问一道「每期怎么更新」(q_list_period),选按期 +累积时同时绑定每一行属于哪一期(按日期列核对,或加统计期起止两列)。 + **问题的 effects 怎么写。** 每个选项是一组 JSON Patch,路径按传进来的配方写;接口层从 answers_base 起、按 问题顺序重放全部回答(P2-SPEC 7.2)。因此: - 关系的两个选项都用整条 replace(「登记」写回完整的 sum_eq / not_comparable,「不登记」写 dismissed), @@ -63,7 +68,7 @@ from app.data.recipe_types import ( derive_tables, ) from app.data.tabular import DATE_HEADER_MIN, EXCEL_ERRORS -from app.data.xlsx_scan import WorkbookScan, cell_ref, col_letter, parse_ref +from app.data.xlsx_scan import TableScan, WorkbookScan, cell_ref, col_letter, parse_ref logger = logging.getLogger(__name__) @@ -612,8 +617,15 @@ class _ListPlan: #: 按数据行收进来并记起草失败 adjacent: list[tuple[int, list[int]]] = field(default_factory=list) #: 不在末尾第一列的合计字样:(行, 列)。第一列在中间或最上面、或者落在别的列上,都无法按合计行核对, - #: 记起草失败;不记就会把合计当明细导入,求和翻倍(H2) + #: 记起草失败;不记就会把合计当明细导入,求和翻倍(H2)。期 5:第一列里同一个字样的穿插行先记进 subtotal_rows, + #: 收完整块再判能不能提议成小计(_settle_subtotals);判不了的才回到这里 stray_totals: list[tuple[int, int]] = field(default_factory=list) + #: 期 5:穿插在明细中间、第一列以合计字样开头的行(不进 rows)和它们的字样;能提议成小计时 subtotal_word 非空 + subtotal_rows: list[int] = field(default_factory=list) + subtotal_words: list[str] = field(default_factory=list) + subtotal_word: str | None = None + #: 期 5:按 Excel 的表格对象起草的(xlsx_scan.TableScan 的名字):表名优先用它,卡片里写明 + table_object: str | None = None # 命名阶段填 id: str = "" table: str = "" @@ -808,13 +820,44 @@ def _scan_block(grid: Grid, plan: _ListPlan, bottom: int) -> None: if word is not None and plan.rows and not _more_rows(grid, r, plan, bottom, types): plan.total_row, plan.total_word = r, word break + others = [(r, c) for c, cell in zip(cols[1:], cells[1:]) + if cell is not None and _total_word(cell.value) is not None] + plan.stray_totals.extend(others) + if word is not None and plan.rows and not others: + # 期 5:夹在明细中间的合计字样,先当小计候选:不进数据行,收完整块再判(_settle_subtotals) + plan.subtotal_rows.append(r) + plan.subtotal_words.append(word) + r += 1 + continue if word is not None: plan.stray_totals.append((r, cols[0])) - plan.stray_totals.extend((r, c) for c, cell in zip(cols[1:], cells[1:]) - if cell is not None and _total_word(cell.value) is not None) plan.rows.append(r) acc.add(r) r += 1 + _settle_subtotals(grid, plan) + + +def _settle_subtotals(grid: Grid, plan: _ListPlan) -> None: + """期 5:夹在明细中间的合计字样能不能提议成小计(ListRows.subtotals,规则起草不再把它们记成失败)。 + + 能的条件:都在第一列、是同一个字样,每一行至少有一格数(没有数的小计核对不了)。末尾那一行和它们字样相同时 + (表以最后一组的小计结束,没有总计),它也是小计,不是合计行。不能的(字样不同、有一行没有数)照旧记进 + stray_totals(起草失败,交给人或 AI),这些行回到数据行里——和期 4 的结果一样。""" + if not plan.subtotal_rows: + return + words = set(plan.subtotal_words) + numeric = [c for c in plan.cols[1:] if any(_value_num(grid.get(r, c)) is not None for r in plan.rows)] + ok = len(words) == 1 and all(any(_value_num(grid.get(r, c)) is not None for c in numeric) + for r in plan.subtotal_rows) + if ok: + plan.subtotal_word = next(iter(words)) + if plan.total_row is not None and plan.total_word == plan.subtotal_word: + plan.subtotal_rows.append(plan.total_row) + plan.total_row = plan.total_word = None + return + plan.stray_totals.extend((r, plan.cols[0]) for r in plan.subtotal_rows) + plan.rows = sorted(set(plan.rows) | set(plan.subtotal_rows)) + plan.subtotal_rows, plan.subtotal_words = [], [] def _list_failures(sheet: str, grid: Grid, plan: _ListPlan) -> list[_Failure]: @@ -898,7 +941,8 @@ def _analyze_lists(grid: Grid) -> _Lists: _finish_block(grid, plan) out.failures.extend(_list_failures(grid.sheet, grid, plan)) out.blocks.append(plan) - end = max((p.total_row or (p.rows[-1] if p.rows else p.header_rows[-1])) for p in made) + end = max(max([p.total_row or 0, *p.subtotal_rows, p.rows[-1] if p.rows else p.header_rows[-1]]) + for p in made) while r_idx < len(rows) and rows[r_idx] <= end: r_idx += 1 return out @@ -945,7 +989,7 @@ def _finish_block(grid: Grid, plan: _ListPlan) -> None: break if cells: break - last = plan.total_row or rows[-1] + last = max([plan.total_row or 0, rows[-1], *plan.subtotal_rows]) for r in range(plan.header_rows[0], last + 1): for c in cols: if grid.get(r, c) is not None or r in plan.header_rows: @@ -975,15 +1019,77 @@ class _Sheet: hidden_cols: list[int] = field(default_factory=list) -def _analyze_sheet(grid: Grid, sid: str) -> _Sheet: +def _lists_from_tables(grid: Grid, objects: list[TableScan]) -> _Lists | None: + """期 5:按 Excel 的表格对象(插入 → 表格)起草列表块。表格对象明确给出了表头行、列范围、数据行和汇总行,比按 + 「像不像表头」猜可靠:数字格空着的行不会被当成表头,紧挨着的两张表也分得开。 + + 出来的块和按规则猜的一样只记文字:锚点仍按表头文字(不记坐标,下一期表格挪了位置照样认得出),汇总行按第一列 + 的文字映射成合计行(total_row),夹在中间的合计字样按小计提议(_settle_subtotals)。表格对象的 ref 写坏了、 + 没有表头行时返回 None,退回按规则猜。""" + out = _Lists(sheet=grid.sheet) + spans = sorted(((t.span(), t) for t in objects), key=lambda x: (x[0] or (0, 0, 0, 0))[:2]) + for span, t in spans: + if span is None or t.header_row_count < 1: + return None + r1, c1, r2, c2 = span + hrows = list(range(r1, r1 + t.header_row_count)) + cols = list(range(c1, c2 + 1)) + texts = _header_texts(grid, hrows, cols) + if set(texts) != set(cols): + blank = [col_letter(c) for c in cols if c not in texts] + msg = (f"工作表「{grid.sheet}」:表格「{t.name}」的表头有空着的列({'、'.join(blank)} 列)," + "按列表导入要每一列都有表头文字") + out.failures.append(_Failure(msg, msg)) + continue + plan = _ListPlan(header_rows=hrows, cols=cols, headers={c: texts[c] for c in cols}, table_object=t.name) + last_data = r2 - max(t.totals_row_count, 0) + rows = [r for r in grid.row_numbers() if hrows[-1] < r <= last_data and any(grid.get(r, c) is not None + for c in cols)] + prev = hrows[-1] + for r in rows: + if r > prev + 1: + plan.skip_blank = True + plan.skipped.extend(range(prev + 1, r)) + prev = r + word = _total_word(_label_of(grid.get(r, c1))) + if word is not None and plan.rows: + plan.subtotal_rows.append(r) + plan.subtotal_words.append(word) + continue + plan.rows.append(r) + if not plan.rows: + msg = f"工作表「{grid.sheet}」:表格「{t.name}」的表头下面没有数据行" + out.failures.append(_Failure(msg, msg)) + continue + if t.totals_row_count > 0: + tr = last_data + 1 + label = _label_of(grid.get(tr, c1)) + word = _total_word(label) or (label if label and len(label) <= 12 and not any( + ch.isdigit() for ch in label) else None) + if word is not None: + plan.total_row, plan.total_word = tr, word + else: + why = ("起草只读了表格前面的部分,没读到汇总行" if grid.truncated and tr > max(grid.row_numbers()) + else "汇总行的第一列没有可用作合计标签的文字(不能含数字、不超过 12 个字)") + msg = f"工作表「{grid.sheet}」:表格「{t.name}」的汇总行(第 {tr} 行)无法按合计行核对:{why}" + out.failures.append(_Failure(msg, msg)) + _settle_subtotals(grid, plan) + _finish_block(grid, plan) + out.failures.extend(_list_failures(grid.sheet, grid, plan)) + out.blocks.append(plan) + return out + + +def _analyze_sheet(grid: Grid, sid: str, objects: list[TableScan] | None = None) -> _Sheet: sh = _Sheet(sheet=grid.sheet, sid=sid) axis = _axis_row(grid) + from_objects = _lists_from_tables(grid, objects) if objects and axis is None else None if axis is not None: sh.cross = _analyze_crosstab(grid, *axis) sh.region = set(sh.cross.region) sh.failures.extend(sh.cross.failures) else: - sh.lists = _analyze_lists(grid) + sh.lists = from_objects if from_objects is not None else _analyze_lists(grid) sh.failures.extend(sh.lists.failures) for b in sh.lists.blocks: sh.region |= b.region @@ -1111,7 +1217,8 @@ def _measure_column(label: str, pos: int) -> tuple[str, str | None, str | None]: def _analyze(scan: WorkbookScan, grids: dict[str, Grid]) -> _Analysis: visible = _visible_grids(scan, grids) hidden = [s.name for s in scan.sheets if s.state != "visible"] - sheets = [_analyze_sheet(g, f"s{i + 1}") for i, g in enumerate(visible)] + sheets = [_analyze_sheet(g, f"s{i + 1}", [t for t in scan.tables if t.sheet == g.sheet]) + for i, g in enumerate(visible)] failures = [f for sh in sheets for f in sh.failures] tables_used: set[str] = set() @@ -1266,7 +1373,9 @@ def list_block(sheet: str, b: _ListPlan, *, block_id: str, tables_used: set[str] 合计表名撞了已用的名字时照起草器的老行为:不另建表定义,keep_as 仍写这个名字(交给静态校验报)。""" b.id = block_id if table is None: - title = b.title if b.title and not any(ch.isdigit() for ch in b.title) else None + # 期 5:按表格对象起草的,表名优先用表格对象的名字(不含数字时;Excel 默认的「表1」不用) + named = b.table_object if b.table_object and not any(ch.isdigit() for ch in b.table_object) else None + title = named or (b.title if b.title and not any(ch.isdigit() for ch in b.title) else None) want = _ENUM_PREFIX.sub("", P.split_unit_suffix(title)[0]) if title else sheet n_fallback = len(tables_used) + 1 b.table = _unique(to_sql_name(want, fallback=f"表{n_fallback}") if want else f"表{n_fallback}", tables_used) @@ -1293,6 +1402,11 @@ def list_block(sheet: str, b: _ListPlan, *, block_id: str, tables_used: set[str] total_spec = {"name": keep, "grain": ["合计项"], "kind": "reported_total", "units": num_units, "note": ""} b.keep_table = keep total = {"label_column": b.names[b.cols[0]], "pick": b.total_word, "keep_as": keep} + subtotals = None + if b.subtotal_word is not None: + # 期 5:穿插的小计只核对、不另存(另存的小计表不能按期累积,要另存由人在配方面板里打开) + subtotals = {"label_column": b.names[b.cols[0]], "pick": b.subtotal_word, "scope": "since_previous", + "group_column": None, "keep_as": None} grain = [b.names[b.grain_col]] if b.grain_col is not None else [] main = {"name": b.table, "grain": grain, "kind": "data", "units": dict(b.units), "note": ""} fill = bool(b.merged_fill) and all(r2 > r1 and c1 == c2 and b.types.get(c1) == "TEXT" @@ -1300,7 +1414,8 @@ def list_block(sheet: str, b: _ListPlan, *, block_id: str, tables_used: set[str] block = { "id": b.id, "layout": "list", "table": b.table, "header_rows": len(b.header_rows), "after_title": None, "columns": cols_out, "extra_columns": "reject", - "rows": {"blank_rows": "skip" if b.skip_blank else "stop", "total_row": total}, + "rows": {"blank_rows": "skip" if b.skip_blank else "stop", "total_row": total, + **({"subtotals": subtotals} if subtotals is not None else {})}, "values": {"placeholders": [{"text": e[0], "meaning": "无数据"} for e in b.placeholders.values()], "blank": "null", "text_number": "reject", "formula": "accept_cached"}, "merged_data": "fill" if fill else "reject", @@ -1797,6 +1912,8 @@ class _ListLoc: rows: list[int] total_row: int | None region: set[tuple[int, int]] = field(default_factory=set) + #: 期 5:穿插的小计行 + subtotal_rows: list[int] = field(default_factory=list) def _find_grid(name: str, grids: dict[str, Grid]) -> Grid | None: @@ -1943,19 +2060,26 @@ def _locate_list(block: ListBlock, grid: Grid, taken: set[int]) -> _ListLoc | No bottom = _bottom(grid) rr = r + 1 pick = P.match_key(block.rows.total_row.pick) if block.rows.total_row else None + sub_pick = P.match_key(block.rows.subtotals.pick) if block.rows.subtotals else None + subs: list[int] = [] while rr <= bottom: if all(grid.get(rr, c) is None for c in cols): if block.rows.blank_rows == "skip": rr += 1 continue break - if pick and P.match_key(_label_of(grid.get(rr, cols[0])) or "").startswith(pick): + label = P.match_key(_label_of(grid.get(rr, cols[0])) or "") + if pick and label.startswith(pick): total = rr break + if sub_pick and label.startswith(sub_pick): + subs.append(rr) + rr += 1 + continue rows.append(rr) rr += 1 - loc = _ListLoc(hrows, cols, rows, total) - for row in [*hrows, *rows, *([total] if total else [])]: + loc = _ListLoc(hrows, cols, rows, total, subtotal_rows=subs) + for row in [*hrows, *rows, *subs, *([total] if total else [])]: loc.region.update((row, c) for c in cols) return loc @@ -2130,6 +2254,9 @@ def questions_for(recipe: dict[str, Any], facts: DraftFacts, grids: dict[str, Gr if ext is not None and ext.sheets.skipped_hidden: for nm in _dedupe([item.get("sheet", "") for item in ext.sheets.skipped_hidden]): cards.append(_hidden_sheet_card(nm)) + # 期 5:列表每期怎么更新的问题排在其余问题之后、q_mode 之前:它只和按期累积有关,不该顶掉期 2 起排在前面的问题 + questions = ([q for q in questions if not q.id.startswith("q_list_period:")] + + [q for q in questions if q.id.startswith("q_list_period:")]) blockers = accumulate_blockers(parsed) cards.append(_mode_card(blockers)) if not blockers: @@ -2543,9 +2670,57 @@ def _list_cards(si: int, bi: int, sheet: Any, block: ListBlock, loc: _ListLoc | reason += f",原值另存表「{total.keep_as}」" if total.keep_as else ",只核对不另存" cells = _coords(name, [(loc.total_row, loc.cols[0])]) if loc is not None and loc.total_row else [] cards.append(Card(f"total_row:{block.id}", f"{where}是合计行,改作核对", reason, cells)) + subt = block.rows.subtotals + if subt is not None: + rows = loc.subtotal_rows if loc is not None else [] + where = _rows_text(rows) if rows else f"「{subt.pick}」开头的行" + scope = ("上一行小计之后的全部明细" if subt.scope == "since_previous" + else f"紧挨在它上面、「{subt.group_column}」相同的那几行明细") + reason = f"每行小计按列求和与{scope}核对,小计行不进数据" + reason += f",原值另存表「{subt.keep_as}」" if subt.keep_as else ",只核对不另存" + cells = _coords(name, [(r, loc.cols[0]) for r in rows]) if loc is not None else [] + cards.append(Card(f"subtotals:{block.id}", f"{where}是穿插的小计行,改作核对", reason, cells)) + q = _list_period_question(si, bi, sheet, block, recipe) + questions.append(q) + cards.append(Card(f"list_period:{block.id}", "这份明细每期怎么更新", + "每期替换:当前版本只含最新一期。按期累积:每期一份明细加进当前版本、可以跨期查询,要说明每一行" + "属于哪一期——选一个日期列,核对每行都在本期统计期内;明细里没有能代表统计期的日期列时," + "选「每行加上本期统计期的起止两列」", [], q.id)) return cards, questions +def _list_period_question(si: int, bi: int, sheet: Any, block: ListBlock, recipe: dict[str, Any]) -> Question: + """期 5:列表每期怎么更新、按期累积时每一行属于哪一期(ListBlock.period),一道题定:每期替换(不绑定);按期 + 累积并按某个日期列核对(每个日期列一个选项);按期累积并给每行加统计期起止两列。后两种同时把导入模式改成按期 + 累积,所在工作表没配统计期的一并加上统计期(回执里统计期取自表内,没有就请人录入)。 + + 为什么不分成「先绑定、再问 q_mode」两道题:问题集合按起草的草稿生成、回答时不变(recipe_imports.answer), + 列表的草稿还没绑定统计期,不符合按期累积的资格,q_mode 根本不会出现。几个列表各答各的:有的选累积、有的选替换 + 时,静态校验照实报出没绑定的那个列表不能累积。""" + base = ["sheets", si, "blocks", bi, "period"] + context = [] if sheet.context else _add_ops(recipe, ["sheets", si, "context"], [ + {"id": "统计期", "kind": "period", "parser": "cn_date_range", "prefer_prefix": None, "cross_check": "filename"}]) + mode = _add_ops(recipe, ["mode"], "accumulate") + # 「每期替换」排第一个:它就是草稿本来的样子(期 4 起各处取第一个选项的地方都不会因此改了配方) + options: list[dict[str, Any]] = [{"value": "none", "label": "每期替换", "needs_reason": False}] + effects: dict[str, list[dict[str, Any]]] = {"none": _add_ops(recipe, base, None)} + for c in block.columns: + if c.type != "DATE": + continue + value = f"column:{c.name}" + options.append({"value": value, "label": f"按期累积,按「{c.name}」核对每行都在本期统计期内", + "needs_reason": False}) + effects[value] = context + mode + _add_ops(recipe, base, {"column": c.name, "check": "within_context", + "synthesize": None}) + options.append({"value": "synthesize", "label": "按期累积,每行加上本期统计期的起止两列", "needs_reason": False}) + effects["synthesize"] = context + mode + _add_ops(recipe, base, {"column": None, "check": "within_context", + "synthesize": "统计期"}) + current = block.period + default = ("synthesize" if current is not None and current.synthesize else + f"column:{current.column}" if current is not None and current.column else "none") + return Question(f"q_list_period:{block.id}", f"列表「{block.id}」每期怎么更新", options, default, effects) + + # ========================================================================== # 期 3:按规则重新起草,并把名字对齐到现行配方(P3-SPEC 6.3) # ========================================================================== diff --git a/backend/app/data/recipe_types.py b/backend/app/data/recipe_types.py index 7643144f..215cd7f3 100644 --- a/backend/app/data/recipe_types.py +++ b/backend/app/data/recipe_types.py @@ -300,11 +300,32 @@ class TotalRow(_M): keep_as: str | None = None +class Subtotals(_M): + """穿插在明细中间的小计行:不作数据,按它所管的那一组明细逐列求和核对;keep_as 给了就另存一张表。""" + + #: 小计标签所在的列(列名) + label_column: str + #: 小计行的标签以这个词开头(match_key 比较)。取自实际格子文字的候选词,不能含数字 + pick: str = Field(min_length=1, max_length=12) + #: 一行小计管哪几行明细:since_previous 上一行小计(或表头)之后的全部明细;group_column 紧挨在它上面、 + #: group_column 这一列取值相同的那几行明细 + scope: Literal["since_previous", "group_column"] = "since_previous" + #: scope=group_column 时按哪一列分组(列名);since_previous 时为 None + group_column: str | None = None + #: 原表的小计另存成哪张表(kind=reported_total);None 只核对不另存 + keep_as: str | None = None + + class ListRows(_M): #: 数据中间的空行:stop 到此为止(默认);skip 跳过继续(回执记个数) blank_rows: Literal["stop", "skip"] = "stop" #: 合计行:到合计行为止,合计行按列求和核对 total_row: TotalRow | None = None + #: 期 5:穿插在明细中间的小计行(Subtotals)。默认 None = 期 4 的行为(像小计的行照常是数据或起草失败) + subtotals: Subtotals | None = None + #: 期 5:人确认过这张列表没有小计行:以合计字样开头的行就是数据(「累计充值」这类键值清单),执行器不再报 + #: subtotal_suspect,每期也就不用再勾那一项确认。和 subtotals 二选一(静态校验 subtotals_conflict) + no_subtotals: bool = False class ListValues(_M): @@ -317,6 +338,18 @@ class ListValues(_M): formula: Literal["accept_cached", "reject"] = "accept_cached" +class ListPeriod(_M): + """列表按期累积时,每一行属于哪一期,二选一:column 按表里的一个日期列,每行的日期都要落在本期的统计期内; + synthesize 每行加上本期统计期的起、止两列(明细里没有日期列,或者日期不代表它属于哪一期时)。""" + + #: 按这个日期列(ListColumn.name,类型 DATE)核对;和 synthesize 二选一 + column: str | None = None + #: 怎么核对:within_context 每行的日期都要落在本期统计期内(空着的也不行) + check: Literal["within_context"] = "within_context" + #: 加统计期起、止两列(LIST_PERIOD_START、LIST_PERIOD_END);和 column 二选一 + synthesize: Literal["统计期"] | None = None + + class ListBlock(_M): id: str = Field(min_length=1, max_length=32) layout: Literal["list"] @@ -333,6 +366,8 @@ class ListBlock(_M): merged_data: Literal["reject", "fill"] = "reject" #: 期 3:按表头忽略的列(IgnoreColumn),只针对这些列;其余多出的列照 extra_columns。默认空 = 期 2 的行为 ignore_columns: list[IgnoreColumn] = Field(default_factory=list, max_length=50) + #: 期 5:按期累积时每一行属于哪一期(ListPeriod)。默认 None = 期 4 的行为(列表只能每期替换) + period: ListPeriod | None = None Block = Annotated[Union[CrosstabBlock, ListBlock], Field(discriminator="layout")] @@ -441,9 +476,10 @@ class Problem: fix_args: dict[str, Any] | None = None -#: 修复按钮的九种(P3-SPEC 3.2 ①–⑨)。Problem.fix、FixProposal.kind、确认项 fix:<目标键> 的 kind 都取自这里 +#: 修复按钮的种类(P3-SPEC 3.2 ①–⑨;期 5 加 declare_subtotals:列表中间像小计的行改作小计核对)。Problem.fix、 +#: FixProposal.kind、确认项 fix:<目标键> 的 kind 都取自这里 FIX_KINDS = ("remove_label", "add_label", "edit_members", "rename_title", "declare_total", "ignore_cells", - "declare_placeholder", "declare_hidden", "rename_sheet") + "declare_placeholder", "declare_hidden", "rename_sheet", "declare_subtotals") @dataclass @@ -462,13 +498,14 @@ CheckStatus = Literal["passed", "mismatch", "unverifiable", "info"] @dataclass class CheckResult: - #: 稳定 id:K1… 合计核对、G1… 公式引用、T1… 列表合计、R1… 关系(沿用配方里的 id)、C1 统计期、C2 文件名、 - #: N1… 行数、P1… 主键;期 3 的并集整体核对 U1 行数、U2 主键、U3 统计期(物化时跑,都是结构类,P3-SPEC 2.7)。 + #: 稳定 id:K1… 合计核对、G1… 公式引用、T1… 列表合计(期 5 起也含列表中间的小计)、R1… 关系(沿用配方里的 + #: id)、C1 统计期、C2 文件名、N1… 行数、P1… 主键、D1… 列表的日期落在统计期内(期 5,ListPeriod.column); + #: 期 3 的并集整体核对 U1 行数、U2 主键、U3 统计期(物化时跑,都是结构类,P3-SPEC 2.7)。 #: F 编号留给系统发现的事实(Fact.id),两者不混用 id: str kind: Literal["derived_sum", "formula_refs", "column_sum", "relation_sum_eq", "relation_not_comparable", "context_agree", "filename_period", "row_count", "pk_unique", - "union_rows", "union_pk", "union_period"] + "union_rows", "union_pk", "union_period", "period_rows"] title: str status: CheckStatus #: 不通过时的类别:structure / data_quality / info @@ -759,9 +796,10 @@ class SheetsOut: #: hidden_excluded = hidden.rows=exclude 排除的隐藏行;blank_skipped = 列表 blank_rows=skip 跳过的空行; #: ignored_rows = 交叉表 ignore_rows 忽略的行;ignored_outside = ignore_outside 忽略了区域外数字格的行; #: after_stop = 列表停止之后没导入的文字行;total_not_kept = 参与核对、但不另存的合计行(交叉表 keep_as 为空的 -#: derived 段、列表 total_row.keep_as 为空的合计行) +#: derived 段、列表 total_row.keep_as 为空的合计行);期 5:subtotal = 列表中间的小计行(参与核对、不进数据, +#: 另存与否都记在这里:它们都不在本表里) ExcludedReason = Literal["hidden_excluded", "blank_skipped", "ignored_rows", "ignored_outside", "after_stop", - "total_not_kept"] + "total_not_kept", "subtotal"] @dataclass @@ -1031,6 +1069,15 @@ def prose_problems(template: str, *, known: dict[str, set[str]] | None = None) - #: 列表合计行另存成表时,合计项列的固定列名 LIST_TOTAL_DIM = "合计项" +#: 期 5:列表的小计行另存成表时,小计项列的固定列名 +LIST_SUBTOTAL_DIM = "小计项" +#: 期 5:列表按期累积、由系统给每行加统计期时(ListPeriod.synthesize)的两列:本期统计期的第一天、最后一天, +#: 写法 YYYY-MM-DD(同 DATE 列,存 TEXT) +LIST_PERIOD_START = "统计期起" +LIST_PERIOD_END = "统计期止" +#: 期 5:小计核对的键(核对编号 T、合计格 DerivedItem.segment)= 块 id 加这个后缀。块 id 在整份配方内唯一, +#: 同一块的合计行用块 id 本身,两者不相混 +SUBTOTAL_KEY = "·小计" _SQL_TYPE = {"INTEGER": "INTEGER", "REAL": "REAL", "TEXT": "TEXT", "DATE": "TEXT"} @@ -1107,16 +1154,42 @@ def derive_tables(recipe: Recipe) -> tuple[dict[str, list[ColumnOut]], list[Reci cols = [ColumnOut(c.name, _SQL_TYPE[c.type], header=c.header, role="measure" if c.type in ("INTEGER", "REAL") else "text") for c in block.columns] + if block.period is not None and block.period.synthesize: + # 期 5:每行加上本期统计期的起止(role=period:没有格子来历,取值来自统计期) + cols += [ColumnOut(LIST_PERIOD_START, "TEXT", role="period"), + ColumnOut(LIST_PERIOD_END, "TEXT", role="period")] put(block.table, cols, base) total = block.rows.total_row + numeric = [c for c in cols if c.type in ("INTEGER", "REAL")] if total is not None and total.keep_as: tcols = [ColumnOut(LIST_TOTAL_DIM, "TEXT", role="dim")] - tcols += [ColumnOut(c.name, c.type, header=c.header, role="value") - for c in cols if c.type in ("INTEGER", "REAL")] + tcols += [ColumnOut(c.name, c.type, header=c.header, role="value") for c in numeric] put(total.keep_as, tcols, f"{base}/rows/total_row") + sub = block.rows.subtotals + if sub is not None and sub.keep_as: + # 期 5:小计另存:小计项(标签的规范写法)、按组小计时的分组列、各数字列 + scols = [ColumnOut(LIST_SUBTOTAL_DIM, "TEXT", role="dim")] + if sub.scope == "group_column" and sub.group_column: + scols.append(ColumnOut(sub.group_column, "TEXT", role="dim")) + scols += [ColumnOut(c.name, c.type, header=c.header, role="value") for c in numeric] + put(sub.keep_as, scols, f"{base}/rows/subtotals") return out, problems +def list_period_columns(recipe: Recipe) -> dict[str, tuple[str, ...]]: + """期 5:列表写入的表 → 它按什么分期:("column", 日期列) 或 ("synthesize", 统计期起列, 统计期止列)。只列配了 + ListBlock.period 的列表;累积计划的 U3、单期的 D 核对、说明里「按哪一列筛选」共用这一份。""" + out: dict[str, tuple[str, ...]] = {} + for sheet in recipe.sheets: + for block in sheet.blocks: + if isinstance(block, ListBlock) and block.period is not None: + if block.period.synthesize: + out.setdefault(block.table, ("synthesize", LIST_PERIOD_START, LIST_PERIOD_END)) + elif block.period.column: + out.setdefault(block.table, ("column", block.period.column)) + return out + + # ========================================================================== # 期 3:按期累积的资格(静态校验、起草器、累积计划共用同一份,P3-SPEC 2.2) # ========================================================================== @@ -1130,10 +1203,13 @@ def accumulate_blockers(recipe: Recipe) -> list[RecipeProblem]: 1. 至少一个工作表配置了统计期上下文(SheetRecipe.context 非空),否则报 accumulate_needs_period(path=/mode); 2. derive_tables 推出的每张表都「按统计期分得开」,否则逐表报 accumulate_unkeyed(path=/tables/{ti},表没在 tables 里声明时指向第一个写入它的块),消息写明哪一条不满足: - - 只由交叉表块写入(measures、dimension 段,或者 derived 段的 keep_as)——列表写入的表一律不满足; - - 写入它的每个交叉表块,axis.checks 都含 covers_context(日期恰好落在统计期内); - - 这张表的 grain 含那个块的 axis.name。 - 两条都成立时,各期的日期集合互不相交(统计期不重叠由累积计划保证),主键里不需要另加统计期列。 + - 交叉表写入的(measures、dimension 段,或者 derived 段的 keep_as):写入它的每个交叉表块,axis.checks 都含 + covers_context(日期恰好落在统计期内),并且这张表的 grain 含那个块的 axis.name; + - 期 5:列表写入的明细表:写入它的每个列表块都配了 period(按日期列核对,或加统计期起止两列),并且所在的 + 工作表配了统计期。列表的合计行、小计行另存的表不能累积:各期的合计项、小计项相同,主键分不开各期; + - 同一张表不能既由交叉表又由列表写入。 + 都成立时,各期的行按统计期分得开(统计期不重叠由累积计划保证;列表明细跨期重复的同一条记录由并集的主键核对 + U2 拒收)。 """ problems: list[RecipeProblem] = [] if not any(sheet.context for sheet in recipe.sheets): @@ -1144,6 +1220,11 @@ def accumulate_blockers(recipe: Recipe) -> list[RecipeProblem]: writers: dict[str, list[CrosstabBlock | ListBlock]] = {} first_path: dict[str, str] = {} + #: 列表的合计行、小计行另存的表 → 写入它的块 + totals: dict[str, list[ListBlock]] = {} + #: 列表块 → 所在工作表有没有配统计期 + ctx_of: dict[int, bool] = {} + def _add(table: str, block: CrosstabBlock | ListBlock, path: str) -> None: got = writers.setdefault(table, []) if all(b is not block for b in got): @@ -1161,16 +1242,32 @@ def accumulate_blockers(recipe: Recipe) -> list[RecipeProblem]: _add(seg.keep_as.table, block, path) else: _add(block.table, block, path) - total = block.rows.total_row - if total is not None and total.keep_as: - _add(total.keep_as, block, path) + ctx_of[id(block)] = bool(sheet.context) + for kept in (block.rows.total_row.keep_as if block.rows.total_row else None, + block.rows.subtotals.keep_as if block.rows.subtotals else None): + if kept: + _add(kept, block, path) + totals.setdefault(kept, []).append(block) index = {t.name: ti for ti, t in enumerate(recipe.tables)} grains = {t.name: list(t.grain) for t in recipe.tables} for name in tables: blocks = writers.get(name, []) - if any(isinstance(b, ListBlock) for b in blocks): - reasons = ["列表形态的表目前只支持每期替换"] + lists = [b for b in blocks if isinstance(b, ListBlock)] + if lists and len(lists) != len(blocks): + reasons = ["同一张表既由交叉表又由列表写入,各期的行分不开"] + elif name in totals: + reasons = ["列表的合计行、小计行另存的表不能按期累积:各期的合计项、小计项相同,主键分不开各期" + "(可以改为只核对、不另存)"] + elif lists: + reasons = [] + unbound = [b.id for b in lists if b.period is None] + if unbound: + reasons.append("、".join(f"列表「{x}」" for x in unbound) + + "没有指定每一行属于哪一期(按日期列核对,或加统计期起止两列)") + no_ctx = [b.id for b in lists if b.period is not None and not ctx_of.get(id(b))] + if no_ctx: + reasons.append("、".join(f"列表「{x}」" for x in no_ctx) + "所在的工作表没有配置统计期") else: reasons = [] uncovered = [b.id for b in blocks if isinstance(b, CrosstabBlock) and "covers_context" not in b.axis.checks] diff --git a/backend/app/data/source_versions.py b/backend/app/data/source_versions.py index a3dcb73f..2e0dedfe 100644 --- a/backend/app/data/source_versions.py +++ b/backend/app/data/source_versions.py @@ -110,6 +110,16 @@ def load_manifest(artifact_id: str | None) -> dict[str, Any] | None: return doc if isinstance(doc, dict) else None +def _signer_verified(artifact_id: str | None) -> bool: + """导入清单里记下的「署名是否认证」;没有清单(简单导入)、读不出来的照实算未认证。""" + try: + doc = load_manifest(artifact_id) + except ValueError: + return False + signer = (doc or {}).get("signed_by") + return bool(signer.get("verified")) if isinstance(signer, dict) else False + + def _rows_of(report: dict[str, Any] | None) -> dict[str, int]: """构建回执里各表的行数(执行器回执、期 1 的解析回执都是 tables: [{name, rows}])。""" out: dict[str, int] = {} @@ -394,7 +404,7 @@ def columns_doc(report: dict[str, Any], periods: list[str]) -> dict[str, dict[st def manifest_doc(*, source_id: str, snapshot_id: str, action: str, recipe: dict[str, Any], union: dict[str, Any], parts: list[Part | dict[str, Any]], report: dict[str, Any], label_sets: list[Any], gaps: list[dict[str, str]], replaced: Any, removed: Any, - notes: SchemaNotes, signed_by: str | None) -> dict[str, Any]: + notes: SchemaNotes, signed_by: str | None, signer_verified: bool = False) -> dict[str, Any]: """快照清单(2.9)。parts 按统计期排序,与 report.part_rows 同序;本期写成 dict(导入记录还没落库)。""" part_rows = report.get("part_rows") or [] keys = [] @@ -422,7 +432,8 @@ def manifest_doc(*, source_id: str, snapshot_id: str, action: str, recipe: dict[ "checks": copy.deepcopy(report.get("checks") or []), "notes": {"templates": dict(notes.templates), "rendered": {t: {"comment": n.comment, "columns": dict(n.columns)} for t, n in notes.tables.items()}}, - "signed_by": {"name": signed_by, "verified": False}, + # 是否认证照实记(05-已知遗留 (c),同导入清单) + "signed_by": {"name": signed_by, "verified": bool(signer_verified and signed_by)}, "created_at": utcnow().isoformat(), } @@ -684,8 +695,10 @@ async def _current(session: AsyncSession, source: DataSource) -> _Current: mode=snap.mode or "replace", recipe=recipe) -def _revoked_record(reason: str, signed_by: str | None) -> dict[str, Any]: - return {"at": utcnow().isoformat(), "reason": reason, "signed_by": signed_by or None, "signed_by_verified": False} +def _revoked_record(reason: str, signed_by: str | None, signer_verified: bool = False) -> dict[str, Any]: + """作废接受的记录。是否认证照实记(05-已知遗留 (c)):开了登录、署名是登录的人时为真。""" + return {"at": utcnow().isoformat(), "reason": reason, "signed_by": signed_by or None, + "signed_by_verified": bool(signer_verified and signed_by)} async def rollback_target(session: AsyncSession, source: DataSource, import_id: str) -> SourceSnapshot | None: @@ -739,7 +752,8 @@ async def revoke_plan(session: AsyncSession, source: DataSource, imp: TableImpor async def _rebuild_without(session: AsyncSession, cur: _Current, imp: TableImport, *, expected_current: str | None, - kind: str, reason: str, signed_by: str | None, revoke: bool) -> tuple[str, bool]: + kind: str, reason: str, signed_by: str | None, revoke: bool, + signer_verified: bool = False) -> tuple[str, bool]: """从当前版本去掉一期、生成(或复用、复活)新快照(7.4)。配方和导入模式不变。返回 (快照 id, 是否复用)。 revoke=True 时同一个事务里写 table_imports.revoked(7.5):复用路径在调 activate_snapshot 之前改好(随它的提交 @@ -758,7 +772,7 @@ async def _rebuild_without(session: AsyncSession, cur: _Current, imp: TableImpor sid = table_versions.snapshot_id(source_id, keep, recipe_types.RECIPE_ENGINE_VER, mode="accumulate", recipe_sha=target_sha) imp_id = imp.id - revoked = _revoked_record(reason, signed_by) if revoke else None + revoked = _revoked_record(reason, signed_by, signer_verified) if revoke else None activation = {"kind": kind, "import_id": imp_id, "reason": reason, "signed_by": signed_by} existing = await session.get(SourceSnapshot, sid) @@ -871,7 +885,7 @@ async def _rebuild_without(session: AsyncSession, cur: _Current, imp: TableImpor "table_hashes": dict((info.union_report or report_json or {}).get("table_hashes") or {}), "union_ver": recipe_types.UNION_VER}, parts=parts, report=info.union_report or report_json or {}, label_sets=label_sets, gaps=gaps, - replaced=None, removed=removed, notes=notes, signed_by=signed_by) + replaced=None, removed=removed, notes=notes, signed_by=signed_by, signer_verified=signer_verified) cache["snapshot_manifest"] = await artifact_store.put_json( doc, kind="snapshot_manifest", meta={"source_id": source_id, "snapshot_id": info.snapshot_id}) return cache @@ -931,7 +945,7 @@ def _check_base(cur: _Current, expected_current: str | None) -> None: async def remove_period(session: AsyncSession, source: DataSource, import_id: str, *, expected_current: str | None, - reason: str, signed_by: str | None) -> dict[str, Any]: + reason: str, signed_by: str | None, signer_verified: bool = False) -> dict[str, Any]: """POST …/imports/{id}/remove(7.4)。""" imp = await session.get(TableImport, import_id) if imp is None or imp.source_id != source.id: @@ -947,13 +961,15 @@ async def remove_period(session: AsyncSession, source: DataSource, import_id: st if len(cur.imports) < 2: raise ImportRefused(409, "last_period", _LAST_PERIOD) sid, reused = await _rebuild_without(session, cur, imp, expected_current=expected_current, kind="remove_period", - reason=reason, signed_by=signed_by, revoke=False) + reason=reason, signed_by=signed_by, revoke=False, + signer_verified=signer_verified) return {"snapshot_id": sid, "removed_import_id": import_id, "reused": reused} async def revoke_acceptance(session: AsyncSession, source: DataSource, import_id: str, *, expected_current: str | None, expected_target: str | None, - ack_mask_lost: list[str] | None, reason: str, signed_by: str | None) -> dict[str, Any]: + ack_mask_lost: list[str] | None, reason: str, signed_by: str | None, + signer_verified: bool = False) -> dict[str, Any]: """POST …/imports/{id}/revoke-acceptance(7.5)。累积模式(至少两期)= 移除这一期;否则回滚到第 5 节定义的目标。""" imp = await session.get(TableImport, import_id) if imp is None or imp.source_id != source.id: @@ -968,7 +984,7 @@ async def revoke_acceptance(session: AsyncSession, source: DataSource, import_id if _accumulating(cur): sid, _reused = await _rebuild_without(session, cur, imp, expected_current=expected_current, kind="revoke_acceptance", reason=reason, signed_by=signed_by, - revoke=True) + revoke=True, signer_verified=signer_verified) return {"snapshot_id": sid, "action": "remove_period"} target = await rollback_target(session, source, imp.id) if target is None: @@ -976,7 +992,7 @@ async def revoke_acceptance(session: AsyncSession, source: DataSource, import_id if target.id != expected_target: raise ImportRefused(409, "revoke_target_changed", _TARGET_CHANGED) target_id = target.id - imp.revoked = _revoked_record(reason, signed_by) + imp.revoked = _revoked_record(reason, signed_by, signer_verified) try: await table_versions.activate_snapshot( session, source, target_id, expected_current=expected_current, kind="revoke_acceptance", reason=reason, @@ -1049,6 +1065,8 @@ async def import_records(session: AsyncSession, source: DataSource, imps: list[T # 按配方导入才有(简单导入为空):配方、统计期、署名,接受的条数 "recipe_id": imp.recipe_id, "period_start": imp.period_start, "period_end": imp.period_end, "signed_by": imp.signed_by, "overrides": len(imp.overrides or []), "waivers": len(imp.waivers or []), + # 署名是不是认证过的身份:按这一期导入清单里记下的(05-已知遗留 (c);旧清单写的是 False,照记下的说,不改写) + "signed_by_verified": _signer_verified(imp.manifest_artifact), # 期 3(7.6) "acceptances": _acceptance_rows(imp), "revoked": imp.revoked, "manifest_artifact": imp.manifest_artifact, "recipe_seq": seqs.get(imp.recipe_id) if imp.recipe_id else None, "in_current": in_current, diff --git a/backend/app/data/sqlcheck.py b/backend/app/data/sqlcheck.py index 2a300acd..c3f200f6 100644 --- a/backend/app/data/sqlcheck.py +++ b/backend/app/data/sqlcheck.py @@ -326,7 +326,7 @@ class _Table: 「唯一日期列」和这里认的时间列对得上)。""" meta = next((c for c in self.meta.get("columns") or [] if isinstance(c, Mapping) and str(c.get("name")).lower() == column.lower()), None) - return catalog.is_date_column(column, (meta or {}).get("type")) + return catalog.is_date_column(column, (meta or {}).get("type"), enum_values=(meta or {}).get("enum_values")) #: 一列解析到的位置:(表, 列的真名) @@ -1084,36 +1084,36 @@ def graph_checks(nodes: Iterable[Any], checkers: Mapping[str, SqlChecker], *, return out -async def load_checkers(session: Any, sources: Iterable[Any]) -> dict[str, SqlChecker]: +async def load_checkers(session: Any, sources: Iterable[Any], *, catalog_view: Any = None) -> dict[str, SqlChecker]: """给一批数据源(DataSource 行)各建一个检查器:{源名: SqlChecker}。没有目录的源不建。 - 一次查询读出这些源的全部目录。读不到就当没有目录(返回空 dict):检查是附加的,不能挡住自查和发布本身。 + 一次查询读出这些源的全部目录,读哪一份由 catalog_view 定(data/catalog_view.py:当前目录、目录快照、关;只用已核实 + 的项),不给是当前目录。读不到当前目录就当没有目录(返回空 dict):检查是附加的,不能挡住自查和发布本身;视图指定的 + 目录快照不在了照实报错。 """ - from sqlalchemy import select - - from app.db.models import CatalogNote + from app.data.catalog_view import CatalogSnapshotMissing, CatalogView, read_many rows = [s for s in sources if getattr(s, "id", None) and dialect_of(getattr(s, "kind", None))] if not rows: return {} try: - notes = (await session.execute( - select(CatalogNote.source_id, CatalogNote.table_name, CatalogNote.notes) - .where(CatalogNote.source_id.in_([r.id for r in rows])) - )).all() + entries = await read_many(session, CatalogView.parse(catalog_view), [r.id for r in rows]) + except CatalogSnapshotMissing: + raise except Exception: # noqa: BLE001 logger.exception("读取数据目录失败,本次不做基于目录的 SQL 检查") return {} by_source: dict[str, dict[str, Any]] = {} - for source_id, table, value in notes: - if isinstance(value, dict) and value: - by_source.setdefault(source_id, {})[table] = value + for source_id, tables in entries.items(): + for table, entry in tables.items(): + if isinstance(entry.notes, dict) and entry.notes: + by_source.setdefault(source_id, {})[table] = entry.notes return {r.name: SqlChecker(kind=r.kind, schema_cache=r.schema_cache, notes=by_source[r.id], source_name=r.name) for r in rows if r.id in by_source} async def load_checkers_by_name(session: Any, names: Iterable[str]) -> dict[str, SqlChecker]: - """按数据源名建检查器(只看启用着的源)。发布门禁用:图里写的是源名。""" + """按数据源名建检查器(只看启用着的源)。发布门禁用:图里写的是源名,读的是当前目录。""" from sqlalchemy import select from app.db.models import DataSource diff --git a/backend/app/data/table_versions.py b/backend/app/data/table_versions.py index 2407d0e3..3302344b 100644 --- a/backend/app/data/table_versions.py +++ b/backend/app/data/table_versions.py @@ -2130,7 +2130,7 @@ class RawPurge: async def purge_import_raw( - session: AsyncSession, imp: TableImport, *, reason: str, signed_by: str | None + session: AsyncSession, imp: TableImport, *, reason: str, signed_by: str | None, signer_verified: bool = False ) -> RawPurge: """清除一次导入的原件(提交)。按两元组解包得到 (是否删了文件, 一并标成已清除的其他导入记录), discarded_stagings 是一并放弃的暂存区。 @@ -2160,7 +2160,9 @@ async def purge_import_raw( now = utcnow() record = { "at": now.isoformat(), "reason": reason, - "signed_by": (signed_by or "").strip() or None, "signed_by_verified": False, + # 是否认证照实记(05-已知遗留 (c)):开了登录、署名是登录的人时为真 + "signed_by": (signed_by or "").strip() or None, + "signed_by_verified": bool(signer_verified and (signed_by or "").strip()), } imp.raw_state, imp.purged = "purged", record for other in others: diff --git a/backend/app/data/xlsx_cells.py b/backend/app/data/xlsx_cells.py index 28f137e0..69f93fad 100644 --- a/backend/app/data/xlsx_cells.py +++ b/backend/app/data/xlsx_cells.py @@ -24,7 +24,7 @@ from collections.abc import Iterator from typing import Any from xml.etree import ElementTree as ET -from app.data import tabular +from app.data import csv_grid, tabular from app.data.recipe_types import Grid, GridCell from app.data.xlsx_scan import SheetScan, UnsupportedTable, WorkbookScan, _broken, cell_ref, parse_ref @@ -70,6 +70,14 @@ class Workbooks: self.raw = raw self._values: Any = None self._formulas: Any = None + self._csv: Any = None + + def csv(self, delimiter: str | None = None) -> Any: + """CSV 原件解析好的记录(csv_grid.CsvTable),第一次要读时才解析,同一次执行里复用。delimiter 是扫描时定下的 + 分隔符(csv_grid.sheet_delimiter),不给时再嗅一次。""" + if self._csv is None: + self._csv = csv_grid.parse(self.raw, delimiter) + return self._csv def values(self) -> Any: if self._values is None: @@ -126,6 +134,11 @@ def sheet_rows(books: Workbooks, sheet: SheetScan, *, min_row: int, max_row: int 行数和每行的宽度都一样。文件内容写坏了(见 _READ_ERRORS)抛 UnsupportedTable,日志只记异常类型。 """ try: + if csv_grid.is_csv_sheet(sheet): + # CSV 适配出来的「工作表」:从解析好的记录里取,口径同下(只给非空格、整行为空的不给)。分隔符用扫描时定下的 + yield from csv_grid.rows(books.csv(csv_grid.sheet_delimiter(sheet)), min_row=min_row, max_row=max_row, + min_col=min_col, max_col=max_col) + return yield from _sheet_rows(books, sheet, min_row=min_row, max_row=max_row, min_col=min_col, max_col=max_col) except UnsupportedTable: raise diff --git a/backend/app/data/xlsx_scan.py b/backend/app/data/xlsx_scan.py index 4910c6ef..5acd20fd 100644 --- a/backend/app/data/xlsx_scan.py +++ b/backend/app/data/xlsx_scan.py @@ -216,6 +216,9 @@ class WorkbookScan: tables: list[TableScan] = field(default_factory=list) #: 定义名称 → 原文。工作表级的名称记作「工作表名!名称」,免得同名互相覆盖 defined_names: dict[str, str] = field(default_factory=dict) + #: 原件的格式:xlsx,或者 csv(csv_grid.scan 适配出来的一张「工作表」,读取层据 SheetScan.path 从 CSV 里读)。 + #: 老暂存区的扫描结果里没有这个键,按 xlsx 读 + format: str = "xlsx" def sheet(self, name: str) -> SheetScan | None: return next((s for s in self.sheets if s.name == name), None) @@ -1090,4 +1093,5 @@ def scan_from_json(data: dict[str, Any]) -> WorkbookScan: for t in tables: t.columns = list(t.columns) return WorkbookScan(sheets=sheets, full_calc_on_load=bool(data.get("full_calc_on_load")), tables=tables, - defined_names=dict(data.get("defined_names") or {})) + defined_names=dict(data.get("defined_names") or {}), + format=str(data.get("format") or "xlsx")) diff --git a/backend/app/db/base.py b/backend/app/db/base.py index 98e28bc1..40204684 100644 --- a/backend/app/db/base.py +++ b/backend/app/db/base.py @@ -239,6 +239,12 @@ _COLUMN_MIGRATIONS: list[tuple[str, str, str]] = [ ("apps", "scope", "VARCHAR(20) DEFAULT 'shared'"), ("apps", "owner", "VARCHAR(100)"), ("apps", "blueprint_id", "VARCHAR(32)"), + # 下一轮(docs/design/next-round/00-总方案.md X4、X9):运行为什么结束在 cancelled;定时的目标种类和 id; + # 定时触发建的那次评测。存量行:end_reason 为空,target_kind 是 workflow(target_id 由 scheduler 按 workflow_id 读) + ("runs", "end_reason", "VARCHAR(20)"), + ("schedules", "target_kind", "VARCHAR(20) DEFAULT 'workflow'"), + ("schedules", "target_id", "VARCHAR(32)"), + ("schedule_fires", "eval_run_id", "VARCHAR(32)"), ] #: 迁移补上的列要建的索引:(索引名, 表, 列或多列)。create_all 不碰已有的表,表上新列的索引也就不会建。 diff --git a/backend/app/db/models.py b/backend/app/db/models.py index 7e3ad023..6c11c6bf 100644 --- a/backend/app/db/models.py +++ b/backend/app/db/models.py @@ -654,6 +654,9 @@ class Run(Base, TimestampMixin): input: Mapped[dict[str, Any]] = mapped_column(default=dict) output: Mapped[dict[str, Any]] = mapped_column(default=dict) error: Mapped[str | None] = mapped_column(Text, default=None) + # 运行为什么结束在 cancelled(docs/design/next-round/00-总方案.md X9):rejected 人工驳回、stopped 手动停止。 + # 其余终态、加这一列之前的运行为空 + end_reason: Mapped[str | None] = mapped_column(String(20), default=None) usage: Mapped[dict[str, Any]] = mapped_column(default=dict) # tokens / cost / 各节点耗时 started_at: Mapped[datetime | None] = mapped_column(default=None) finished_at: Mapped[datetime | None] = mapped_column(default=None) @@ -975,6 +978,12 @@ class Schedule(Base, TimestampMixin): id: Mapped[str] = mapped_column(String(32), primary_key=True, default=new_id) name: Mapped[str] = mapped_column(String(200), default="") + # 定时的目标(next-round X4,定时只有一套):workflow 发起工作流运行;eval_suite 跑一次评测题集; + # source_maintenance 维护一个数据源(探查、剖析、取值观察)。加这一列之前的任务都是 workflow + target_kind: Mapped[str] = mapped_column(String(20), default="workflow") + # 目标的 id:workflow 时同 workflow_id;其余两种是题集 id、数据源 id + target_id: Mapped[str | None] = mapped_column(String(32), default=None, index=True) + # 非空(老表的约束,只加不改的迁移放宽不了):目标不是工作流时存空串,读写都经 scheduler/service 里的一处封装 workflow_id: Mapped[str] = mapped_column(String(32), index=True) # 保存时的工作流名。工作流还在时接口给它现在的名字,删掉之后才用这一份 workflow_name: Mapped[str] = mapped_column(String(200), default="") @@ -1020,6 +1029,8 @@ class ScheduleFire(Base): # started | skipped_overlap | skipped_misfire | failed_to_start status: Mapped[str] = mapped_column(String(30), default="started") run_id: Mapped[str | None] = mapped_column(String(32), default=None, index=True) + # 目标是评测题集时,这一次触发建的那次评测(target_kind=eval_suite,next-round X4) + eval_run_id: Mapped[str | None] = mapped_column(String(32), default=None) reason: Mapped[str | None] = mapped_column(Text, default=None) # 渲染后的输入,就是交给运行的那一份 inputs: Mapped[dict[str, Any]] = mapped_column(default=dict) diff --git a/backend/app/engine/approval.py b/backend/app/engine/approval.py index cb8beb21..5da191b8 100644 --- a/backend/app/engine/approval.py +++ b/backend/app/engine/approval.py @@ -39,6 +39,9 @@ class Decision: #: 审批卡上点的是「始终允许」:批准这次,并且这个工具以后改由门控把关(tools/trust.py)。 #: 只认明确的 True,而且只跟着批准走 always: bool = False + #: 这次驳回是待办超时、按「超时驳回」替人做的(process/tasks.py 的 tick),不是有人点了驳回。只跟着驳回走, + #: 节点据此在 NodeRejected 上记 decision=expired_reject + expired: bool = False def _word(value: Any) -> str: @@ -71,6 +74,7 @@ def read_decision(response: Any) -> Decision: code=code if isinstance(code, str) and code else None, value=response.get("value"), always=verdict is True and response.get("always") is True, + expired=verdict is False and response.get("expired") is True, ) verdict = _verdict(response) # 不是一个是/否词的字符串("拒绝,这个表是生产库")既算不上批准,本身又 diff --git a/backend/app/engine/compiler.py b/backend/app/engine/compiler.py index c9c115de..d0599d58 100644 --- a/backend/app/engine/compiler.py +++ b/backend/app/engine/compiler.py @@ -10,7 +10,7 @@ from langgraph.graph import END, START, StateGraph from app.core.errors import describe_exception, raw_detail from app.core.events import EventType -from app.engine.context import NodeContext, NodeError, RunContext +from app.engine.context import NodeContext, NodeError, NodeRejected, RunContext from app.engine.expressions import open_template_scope, template_scope from app.engine.nodes import ( challenge, control, dual_check, human, io, knowledge, llm, merge, metrics, multi, report, tools, @@ -144,6 +144,13 @@ def _wrap( raise except asyncio.CancelledError: raise + except NodeRejected as e: + # 人的决定(驳回、超时驳回),不是故障(总方案 X9):「失败重试」「出错时继续」都是给故障准备的。重试只会把 + # 同一件事再问一遍人;on_error=continue 会把驳回吞掉接着往下走,最后以 succeeded 结束,通知、定时推送、 + # 发布单都看不到这次驳回。所以不重试、不被容错接住;节点这一步照旧记一条 node.failed(时间线、实例进度 + # 按它收尾这一步),交给 runner 收成 cancelled + end_reason=rejected + last_error = e + break except Exception as e: # noqa: BLE001 # NodeError 以前在这里直接 raise,于是节点自己抛的错既不重试、 # 也不发 node.failed、更不被 on_error=continue 接住。而 NodeError @@ -168,8 +175,8 @@ def _wrap( ctx.emit(EventType.NODE_FAILED, error=message, duration_ms=elapsed, node_id=node.id, detail=_detail(last_error)) - # 容错模式:记下错误继续往下走,而不是让整张图挂掉 - if node.config.get("on_error") == "continue": + # 容错模式:记下错误继续往下走,而不是让整张图挂掉。驳回不算错误,不走这里(见上) + if node.config.get("on_error") == "continue" and not isinstance(last_error, NodeRejected): return { "nodes": {node.id: {"error": message, "failed": True}}, "trail": [{"node_id": node.id, "error": message, "ts": time.time()}], diff --git a/backend/app/engine/context.py b/backend/app/engine/context.py index ad197844..7cc915ea 100644 --- a/backend/app/engine/context.py +++ b/backend/app/engine/context.py @@ -106,3 +106,33 @@ class NodeError(RuntimeError): super().__init__(message) self.node_id = node_id self.retryable = retryable + + +#: 驳回发生在哪种关卡上(NodeRejected.kind):人工节点的三种模式、写入前的动作确认、代码执行审批、工具调用审批、双盲裁决 +REJECTION_KINDS = ("approve", "input", "edit", "action_confirm", "code", "tool", "dual_check") + + +class NodeRejected(NodeError): + """人的决定让运行到这里结束:驳回、超时驳回(docs/design/next-round/05-已知遗留.md (a))。 + + 这不是故障。以前六处驳回都抛普通的 NodeError,运行记成 failed:通知推「实例失败」,定时推「运行失败」,发布单把 + 驳回过的试运行说成「试运行失败了」,记录页还给「继续运行」——接着跑只会读到同一个驳回答复、再停一次。现在 runner + 单独接住它,收成 cancelled + end_reason=rejected,run.cancelled 带上在哪一步、谁、结论和备注,消费方都读这一个字段。 + + 是 NodeError 的子类:节点这一步照旧记 node.failed(时间线、实例进度按它收尾);但不重试、不被 on_error=continue + 接住(engine/compiler._execute:那两项是给故障准备的,驳回被吞掉运行就以 succeeded 结束了)。原话(str)照旧写给人看, + Run.error 和以前一字不差。kind 见 REJECTION_KINDS;decision 是 reject 或 expired_reject(同待办的结论)。""" + + def __init__(self, node_id: str, message: str, *, kind: str, decision: str = "reject", + actor: str | None = None, note: str = "") -> None: + super().__init__(node_id, message) + self.kind = kind + self.decision = decision + self.actor = actor + self.note = note + + @classmethod + def of(cls, ctx: Any, message: str, *, kind: str, decision: Any) -> NodeRejected: + """按节点上下文和读出来的审批结论(engine/approval.Decision)造一个:谁驳回的取 ctx.actor()。""" + return cls(ctx.node.id, message, kind=kind, decision="expired_reject" if decision.expired else "reject", + actor=ctx.actor(), note=decision.note or "") diff --git a/backend/app/engine/judge.py b/backend/app/engine/judge.py index 53c6dff4..51a73cb5 100644 --- a/backend/app/engine/judge.py +++ b/backend/app/engine/judge.py @@ -37,7 +37,6 @@ import re import time from collections.abc import Callable, Iterable from dataclasses import asdict, dataclass -from datetime import datetime from typing import Any from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, SystemMessage @@ -68,6 +67,7 @@ from app.engine.evidence import ( ) from app.engine.expressions import CellError, column_kind, locate_cell from app.engine.merge_query import MERGE_SOURCE +from app.engine import spend from app.engine.nodes.llm import _split_structured, _usage_of from app.engine.state import message_text from app.providers import catalog as pricing @@ -855,7 +855,7 @@ JUDGE_DEFAULTS: dict[str, Any] = { "click_max_cost_usd": 0.01, # 探索运行里每点一次「请模型判断」的金额上限 "daily_max_usd": 2.0, # 全局每日裁判金额上限(按本地日期) } -#: 每日计数存在设置表的这一行:{date: 本地日期, usd, calls, unpriced_calls}。不从设置接口进出 +#: 每日计数存在设置表的这一行:{date: 本地日期, usd, calls, unpriced_calls}。不从设置接口进出(账本见 engine/spend.py) SPEND_KEY = "judge_spend" _COUNTS = frozenset({"report_max_claims"}) _SECONDS = frozenset({"report_timeout_s"}) @@ -1024,61 +1024,20 @@ async def source_masks(catalog: dict[str, Any], *, loader: Callable[[str], Any] # 每日计数 # -------------------------------------------------------------------------- -_spend_lock = asyncio.Lock() - def _today() -> str: - """本地日期:每日上限按它滚动。""" - return datetime.now().astimezone().date().isoformat() + """裁判这本账的日历:本地日期(同 engine/spend._today)。单独留一个名字:测试换掉它来模拟裁判这边跨天。""" + return spend._today() -def _spend_of(row: Any) -> dict[str, Any]: - value = row.value if row is not None and isinstance(row.value, dict) else {} - today = _today() - if value.get("date") != today: - return {"date": today, "usd": 0.0, "calls": 0, "unpriced_calls": 0} - return {"date": today, "usd": float(value.get("usd") or 0.0), "calls": int(value.get("calls") or 0), - "unpriced_calls": int(value.get("unpriced_calls") or 0)} +#: 裁判这本每日账(engine/spend.py 的 Ledger:预留、找补、按本地日期滚动和核查、评测是同一套代码,各记各的键) +judge_spend = spend.Ledger(SPEND_KEY, today=lambda: _today()) +_LEDGER = judge_spend async def daily_spend() -> dict[str, Any]: """今天(本地日期)裁判已经花了多少:{date, usd, calls, unpriced_calls}。估不出金额的调用只计次数。""" - async with SessionLocal() as session: - return _spend_of(await session.get(Setting, SPEND_KEY)) - - -async def _charge(amount: float, *, limit: float | None, priced: bool) -> str | None: - """记一次调用、先按估算预留金额。加上这笔会超过每日上限就不记,返回 None;记下了返回记在哪天。""" - async with _spend_lock: - async with SessionLocal() as session: - row = await session.get(Setting, SPEND_KEY) - spend = _spend_of(row) - if limit is not None and spend["usd"] + amount > limit + 1e-12: - return None - spend["usd"] = round(spend["usd"] + amount, 8) - spend["calls"] += 1 - spend["unpriced_calls"] += 0 if priced else 1 - if row is None: - session.add(Setting(key=SPEND_KEY, value=spend)) - else: - row.value = spend - await session.commit() - return spend["date"] - - -async def _settle(date: str, delta: float) -> None: - """调用结束后按实际用量找补预留的差额。跨了天就不找补:预留记在前一天,今天从零算。""" - if not delta: - return - async with _spend_lock: - async with SessionLocal() as session: - row = await session.get(Setting, SPEND_KEY) - spend = _spend_of(row) - if row is None or spend["date"] != date: - return - spend["usd"] = max(0.0, round(spend["usd"] + delta, 8)) - row.value = spend - await session.commit() + return await _LEDGER.daily() # -------------------------------------------------------------------------- @@ -1318,7 +1277,7 @@ async def run_request(request: JudgeRequest, *, spec: ModelSpec, budget: Budget, pending = pending[:len(batch)] pending = pending[len(batch):] charged = cost or 0.0 - day = await _charge(charged, limit=budget.daily_max_usd if priced else None, priced=priced) + day = await _LEDGER.charge(charged, limit=budget.daily_max_usd if priced else None, priced=priced) if day is None: # 估算之后、记账之前别的裁判先花掉了今天剩下的钱 stop([*batch, *pending], "daily_max_usd") pending = [] @@ -1353,7 +1312,7 @@ async def run_request(request: JudgeRequest, *, spec: ModelSpec, budget: Budget, inp, out = sum(t["input_tokens"] for t in tokens), sum(t["output_tokens"] for t in tokens) # 回复没带用量(有的网关不给、调用超时)就按估算记:宁可多记,不能让每日上限漏掉 actual = (pricing.estimate_cost(model_id, inp, out) if inp or out else charged) if priced else 0.0 - await _settle(day, actual - charged) + await _LEDGER.settle(day, actual - charged) spent += actual usage["input_tokens"] += inp usage["output_tokens"] += out diff --git a/backend/app/engine/nodes/challenge.py b/backend/app/engine/nodes/challenge.py index 88753e2b..4b8f21ce 100644 --- a/backend/app/engine/nodes/challenge.py +++ b/backend/app/engine/nodes/challenge.py @@ -67,6 +67,7 @@ from app.core.errors import describe_exception from app.core.events import EventType from app.db.base import SessionLocal from app.db.models import DataSource +from app.data import catalog_view from app.engine import deadline, verify_agent from app.engine import judge as judging from app.engine.context import NodeContext, NodeError @@ -767,7 +768,8 @@ class _Review: spec=self.spec, system=CHALLENGE_SYSTEM, task=task_text, sources=live, output_schema=CHALLENGE_SCHEMA, max_queries=self.max_queries, timeout_s=allot.timeout_s, daily_max_usd=self.daily_max_usd, emit=deadline.relabel(ctx.emit, allot), run_id=ctx.run.run_id, - node_id=ctx.node.id, data_versions=run_versions(ctx.run), masked_values=known) + node_id=ctx.node.id, data_versions=run_versions(ctx.run), masked_values=known, + catalog_view=catalog_view.of_run(ctx.run)) outcome = await asyncio.to_thread(_masked_rationale, outcome, known, given=[cand.text], kinds=self.kinds, masks=self.masks) if outcome.error and not outcome.model: diff --git a/backend/app/engine/nodes/dual_check.py b/backend/app/engine/nodes/dual_check.py index 0152179c..70f31a7a 100644 --- a/backend/app/engine/nodes/dual_check.py +++ b/backend/app/engine/nodes/dual_check.py @@ -68,9 +68,10 @@ from app.core.errors import describe_exception from app.core.events import EventType from app.db.base import SessionLocal from app.db.models import DataSource +from app.data import catalog_view from app.engine import deadline, recompute from app.engine.approval import read_decision -from app.engine.context import NodeContext, NodeError +from app.engine.context import NodeContext, NodeError, NodeRejected from app.engine.dual_check_facts import preview, preview_joint, record_choices, upload_dates from app.engine.evidence import build_catalog, is_number, next_exec from app.engine.expressions import ExpressionError, _cell_call, _chain_root, cell_value, column_kind, numeric_text, \ @@ -346,9 +347,11 @@ def _skipped(ctx: NodeContext, reason: str) -> dict[str, Any]: class _Sources: - """一次复算里要读的查询快照、数据源和快照要遮的列:同一份只读一次、只算一次。""" + """一次复算里要读的查询快照、数据源和快照要遮的列:同一份只读一次、只算一次。view 是这次运行的目录视图 + (data/catalog_view.py):交给盲算的目录、比较两条 SQL 用的中文名都按它读,和运行时工具给模型看的是同一份。""" - def __init__(self) -> None: + def __init__(self, view: Any = None) -> None: + self.view = catalog_view.CatalogView.parse(view) self.snapshots: dict[str, tuple[Any, str | None]] = {} self.rows: dict[str, DataSource | None] = {} self.masks: dict[tuple[str, str | None], frozenset[str]] = {} @@ -782,14 +785,14 @@ async def _constant_columns(snapshot: Mapping[str, Any], sources: _Sources) -> f async def _catalog_text(sources: _Sources, name: str) -> str: """交给盲算 agent 的数据目录:表清单(表名、中文名、粒度)和有目录的表的业务说明(业务日期、有效记录条件、去重、 - 字段含义、码值、关联关系)。太长的截断,其余请它用 db_schema 查。""" + 字段含义、码值、关联关系)。太长的截断,其余请它用 db_schema 查。读哪一份由这次运行的目录视图定(sources.view), + 和运行时工具给模型看的是同一份。""" from app.data import catalog row = await sources.source(name) if row is None: return BLIND_NO_CATALOG_PROMPT - async with SessionLocal() as session: - entries = await catalog.read_catalog(session, row.id) + entries = await catalog_view.read(sources.view, row) cache = row.schema_cache if isinstance(row.schema_cache, dict) else {} tables = cache.get("tables") or {} notes = {table: entry.notes for table, entry in entries.items()} @@ -843,7 +846,7 @@ async def _blind(ctx: NodeContext, jobs: dict[str, dict[str, Any]], options: _Op return {}, None, False, usage async with SessionLocal() as session: spec = await verify_model_spec(session, ctx.config, settings=dict(settings)) - sources = _Sources() + sources = _Sources(catalog_view.of_run(ctx.run)) catalogs: dict[str, str] = {} for job in jobs.values(): if job["source"] not in catalogs: @@ -870,7 +873,7 @@ async def _blind(ctx: NodeContext, jobs: dict[str, dict[str, Any]], options: _Op spec=spec, system=BLIND_SYSTEM, task=_task_text(job, catalogs[job["source"]]), sources=[job["source"]], output_schema=BLIND_SCHEMA, max_queries=options.max_queries, timeout_s=allot.timeout_s, daily_max_usd=options.daily_max_usd, emit=deadline.relabel(ctx.emit, allot), run_id=ctx.run.run_id, - node_id=ctx.node.id, data_versions=run_versions(ctx.run)) + node_id=ctx.node.id, data_versions=run_versions(ctx.run), catalog_view=sources.view) keys = list(jobs) await one(keys[0], jobs[keys[0]]) @@ -1078,7 +1081,7 @@ async def _review(state: GraphState, ctx: NodeContext, checked: dict[str, Any], """ rows = [r for r in checked["rows"] if r["status"] == DIFFERENT] results = checked["results"] - sources = _Sources() + sources = _Sources(catalog_view.of_run(ctx.run)) infos: dict[str, dict[str, Any]] = {} metrics: list[dict[str, Any]] = [] summary: list[dict[str, Any]] = [] @@ -1179,15 +1182,15 @@ async def _source_info(sources: _Sources, name: str) -> dict[str, Any]: (uploaded:导入时日期一律存成文字)和它导入时认作日期的列(upload_dates,按当前版本的配方读,见 dual_check_facts.upload_dates)——业务日期据此认上传表格的日期列,认不出的原因照实说(_dated_problem)。两项都进 候选:审批卡的预告按发起审批时读的这一份;裁决后的真写按写入时的版本重读 upload_dates,和现读的表结构一起以写入时 - 为准(dual_check_facts.record_choices)。数据源没了就都不给,照样比原文。""" - from app.data import catalog + 为准(dual_check_facts.record_choices)。数据源没了就都不给,照样比原文。目录按这次运行的目录视图读(sources.view), + 比较两条 SQL 用到的中文名和运行时给模型看的是同一份。""" from app.data.sql_semantics import catalog_labels row = await sources.source(name) if row is None: return {"kind": None, "schema": None, "names": {}, "notes": {}, "uploaded": False, "upload_dates": {}} + entries = await catalog_view.read(sources.view, row) async with SessionLocal() as session: - entries = await catalog.read_catalog(session, row.id) dates = await upload_dates(session, row) notes = {table: entry.notes for table, entry in entries.items()} return {"kind": row.kind, "schema": row.schema_cache if isinstance(row.schema_cache, dict) else None, @@ -1396,7 +1399,7 @@ def _choices(ctx: NodeContext, response: Any, rows: list[dict[str, Any]]) -> tup 填写模式一样:这一步没有驳回的出口。""" decision = read_decision(response) if isinstance(response, dict) and decision.rejected: - raise NodeError(ctx.node.id, f"人工驳回:{decision.note or '未说明原因'}") + raise NodeRejected.of(ctx, f"人工驳回:{decision.note or '未说明原因'}", kind="dual_check", decision=decision) value = decision.value if isinstance(decision.value, dict) else response if isinstance(response, dict) else {} raw = value.get("choices") if isinstance(value.get("choices"), dict) else {} choices = {r["metric_id"]: raw[r["metric_id"]] for r in rows if raw.get(r["metric_id"]) in CHOICES} diff --git a/backend/app/engine/nodes/human.py b/backend/app/engine/nodes/human.py index 0c266bec..1e20e9b6 100644 --- a/backend/app/engine/nodes/human.py +++ b/backend/app/engine/nodes/human.py @@ -8,7 +8,7 @@ from typing import Any from app.core.events import EventType from app.db.base import SessionLocal from app.engine.approval import read_decision -from app.engine.context import NodeContext, NodeError +from app.engine.context import NodeContext, NodeError, NodeRejected from app.engine.replay import ask, once from app.engine.state import GraphState, merge_usage, message_text, template_context from app.providers import catalog @@ -52,7 +52,7 @@ async def run_human(state: GraphState, ctx: NodeContext) -> dict[str, Any]: "__decision__": "approved" if approved else "rejected", } if not approved and ctx.cfg("stop_on_reject", False): - raise NodeError(ctx.node.id, f"人工驳回:{result['note'] or '未说明原因'}") + raise NodeRejected.of(ctx, f"人工驳回:{result['note'] or '未说明原因'}", kind="approve", decision=decision) else: # edit / input # 驳回在这两种模式下以前是个摆设:__decision__ 硬编码成 approved, # approved 字段看都不看。界面上按钮在、toast 说"已驳回",运行却照常 @@ -65,7 +65,7 @@ async def run_human(state: GraphState, ctx: NodeContext) -> dict[str, Any]: # 只有结构体里明说了"不"才算驳回:这两种模式下裸字符串就是内容本身, # 用户在输入框里回一句"no"是在交稿,不是在驳回。 if isinstance(response, dict) and decision.rejected: - raise NodeError(ctx.node.id, f"人工驳回:{decision.note or '未说明原因'}") + raise NodeRejected.of(ctx, f"人工驳回:{decision.note or '未说明原因'}", kind=mode, decision=decision) note = decision.note if isinstance(response, dict) else "" # 裸字符串是内容,不是备注 result = {"value": decision.value, "note": note, "__decision__": "approved"} diff --git a/backend/app/engine/nodes/llm.py b/backend/app/engine/nodes/llm.py index 3c93e39f..bf737196 100644 --- a/backend/app/engine/nodes/llm.py +++ b/backend/app/engine/nodes/llm.py @@ -48,6 +48,7 @@ from app.providers.factory import ( get_chat_model, ) from app.engine.sql_problems import event_checks +from app.data import catalog_view from app.tools.datasource import QUERY_PREFIX, run_masks_results, run_reads_only, run_scope_fixed, run_versions from app.tools.registry import ( ToolArgsError, @@ -341,6 +342,8 @@ async def _resolve_tools(ctx: NodeContext, state: GraphState) -> list[BaseTool]: mask_results=run_masks_results(ctx.run), force_readonly=run_reads_only(ctx.run), fixed_scope=fixed, + # 读哪一份数据目录:发起时定下的(data/catalog_view.py),审批恢复、续跑都沿用 + catalog_view=catalog_view.of_run(ctx.run), # 数据源工具运行中途补固定版本时经它留一条事件(tools/datasource.py 的 _pin_late) emit=ctx.emit, ) diff --git a/backend/app/engine/nodes/multi.py b/backend/app/engine/nodes/multi.py index 9815407c..c83efada 100644 --- a/backend/app/engine/nodes/multi.py +++ b/backend/app/engine/nodes/multi.py @@ -28,6 +28,7 @@ from app.engine.toolcalls import ( from app.providers import catalog from app.providers.factory import ModelSpec, bind_tools_safely, get_chat_model from app.engine.sql_problems import event_checks +from app.data import catalog_view from app.tools.datasource import QUERY_PREFIX, run_masks_results, run_reads_only, run_scope_fixed, run_versions from app.tools.registry import ( ToolArgsError, @@ -317,6 +318,8 @@ async def run_supervisor(state: GraphState, ctx: NodeContext) -> dict[str, Any]: mask_results=run_masks_results(ctx.run), force_readonly=run_reads_only(ctx.run), fixed_scope=run_scope_fixed(ctx.run), + # 读哪一份数据目录:发起时定下的(data/catalog_view.py),审批恢复、续跑都沿用 + catalog_view=catalog_view.of_run(ctx.run), # 数据源工具运行中途补固定版本时经它留一条事件(tools/datasource.py 的 _pin_late) emit=ctx.emit, ) diff --git a/backend/app/engine/nodes/tools.py b/backend/app/engine/nodes/tools.py index 1b26352d..129abc99 100644 --- a/backend/app/engine/nodes/tools.py +++ b/backend/app/engine/nodes/tools.py @@ -11,7 +11,7 @@ from langgraph.func import task from app.core.artifact_store import canonical_json, content_hash from app.engine.approval import read_decision -from app.engine.context import NodeContext, NodeError +from app.engine.context import NodeContext, NodeError, NodeRejected from app.engine.evidence import ledger_enabled, next_exec, query_entry_fields, schema_ledger_entry from app.engine.replay import ask, once from app.engine.state import GraphState @@ -20,6 +20,7 @@ from app.sandbox.base import SandboxLimits from app.sandbox.manager import sandbox_manager from app.engine.sql_problems import event_checks from app.capabilities.registry import is_platform_query, snapshot_owned +from app.data import catalog_view from app.tools.datasource import (DATA_UNAVAILABLE, SCHEMA_PREFIX, run_masks_results, run_reads_only, run_scope_fixed, run_versions) from app.tools.registry import ( @@ -54,6 +55,8 @@ def _tool_ctx(ctx: NodeContext) -> ToolContext: mask_results=run_masks_results(ctx.run), force_readonly=run_reads_only(ctx.run), fixed_scope=fixed, + # 读哪一份数据目录:发起时定下的(data/catalog_view.py),审批恢复、续跑都沿用 + catalog_view=catalog_view.of_run(ctx.run), # 数据源工具运行中途补固定版本时经它留一条事件(tools/datasource.py 的 _pin_late) emit=ctx.emit, ) @@ -127,8 +130,8 @@ async def run_tool(state: GraphState, ctx: NodeContext) -> dict[str, Any]: await once(ctx, EventType.HUMAN_RESOLVED, tool=name, approved=decision.approved, note=decision.note, actor=ctx.actor(), **({"always": True} if always else {})) if not decision.approved: - raise NodeError(ctx.node.id, f"工具 {name} 的调用已被驳回" - + (f":{decision.note}" if decision.note else "")) + raise NodeRejected.of(ctx, f"工具 {name} 的调用已被驳回" + (f":{decision.note}" if decision.note else ""), + kind="tool", decision=decision) limit = limit_of(tool, name, args) ctx.emit(EventType.TOOL_START, tool=name, args=args, **limit_fields(limit)) @@ -299,10 +302,11 @@ async def _run_write(state: GraphState, ctx: NodeContext, name: str, args: dict[ await once(ctx, EventType.HUMAN_RESOLVED, tool=name, approved=decision.approved, note=decision.note, actor=ctx.actor(), payload_hash=confirmed) if not decision.approved: - # 驳回是人的决定,不是故障:运行到这里结束(状态记失败,没有可走的出口),说法写成「未确认(被驳回)」, - # 实例页按待办的结论另说(apps/resolve.instance_states) - raise NodeError(ctx.node.id, f"「{ctx.node.title}」未确认(被驳回),没有发送" - + (f":{decision.note}" if decision.note else "")) + # 驳回是人的决定,不是故障:运行到这里结束(没有可走的出口),收成 cancelled + end_reason=rejected + # (NodeRejected),说法写成「未确认(被驳回)」,实例页、通知按同一个结论说(apps/resolve.instance_state) + raise NodeRejected.of(ctx, f"「{ctx.node.title}」未确认(被驳回),没有发送" + + (f":{decision.note}" if decision.note else ""), + kind="action_confirm", decision=decision) if confirmed != prep.payload_hash: # 动作确认不接受改参数:要改在上游的表单里改。确认之后上游的数据变了,看过的就不是要发的,拒发 raise NodeError(ctx.node.id, f"「{name}」确认时看到的内容和现在要发送的不一致(内容指纹对不上)," @@ -380,8 +384,8 @@ async def run_code(state: GraphState, ctx: NodeContext) -> dict[str, Any]: await once(ctx, EventType.HUMAN_RESOLVED, approved=decision.approved, note=decision.note, actor=ctx.actor()) if not decision.approved: - raise NodeError(ctx.node.id, "代码执行已被驳回" - + (f":{decision.note}" if decision.note else "")) + raise NodeRejected.of(ctx, "代码执行已被驳回" + (f":{decision.note}" if decision.note else ""), + kind="code", decision=decision) # 隔离档位:strict 要硬件级(microVM),fast 要低延迟(Seatbelt/bwrap), # 留空跟随整机默认。要的那档不可用时会安静退回默认,不阻断运行。 diff --git a/backend/app/engine/origin.py b/backend/app/engine/origin.py index dd50ce0a..25081ff7 100644 --- a/backend/app/engine/origin.py +++ b/backend/app/engine/origin.py @@ -24,11 +24,34 @@ ORIGIN_KINDS = ( "api", # 直接调接口,没说是哪 "trial", # 蓝图的试运行(探索运行,docs/design/normal-user-apps.md N4) "personal_app", # 个人应用的刷新(探索运行,N7) + # 下一轮(docs/design/next-round/00-总方案.md X1):都只能由服务端写 + "eval", # 评测:作答和算期望的运行(探索运行) + "ledger_issue", # 问数据的答案按已治理口径正式出具(正式运行) + "reconcile", # 口径对账:这次的取数和口径在同一份数据上各算一次(探索运行) + "recall", # 口径召回:新旧两版口径在同一份数据上各算一次(探索运行) ) #: 客户端在 POST /api/runs 里能声明的来源 CLIENT_ORIGINS = ("chat", "canvas", "launcher", "api") +#: 系统派生的运行(docs/design/next-round/00-总方案.md X1):评测、口径对账、口径召回都是服务端为了量、为了比替人跑的, +#: 不是有人在用。它们是完整的运行(证据、封存都在、点得开),但不进使用统计——探索问题聚类、工具白名单衰减、数据目录的 +#: 使用次数和「被哪些运行用过」、审阅队列的使用权重、以后的账本统计——否则一跑评测,治理信号和目录的使用热度就失真了。 +#: 记录页默认也不列(筛选里能选,列表照实写有几次没列出)。ledger_issue 是替用户正式出具,照常计入 +SYSTEM_ORIGINS = ("eval", "reconcile", "recall") + + +def counted_runs() -> Any: + """SQL 条件:计入使用统计的运行(排除 SYSTEM_ORIGINS)。所有使用统计都只算它圈进来的运行,别处不另写一份过滤。 + + 来源按非空比较(coalesce,同 auth/access.hidden_runs):加来源这一列之前的老运行 origin_kind 为空,NOT IN 碰上 NULL + 会把它们一起筛掉;外连接没对上运行的行(问数据只回了一句、没有运行的那一轮)同理照常计入。""" + from sqlalchemy import func + + from app.db.models import Run + + return func.coalesce(Run.origin_kind, "").not_in(SYSTEM_ORIGINS) + def origin_note(kind: str, **data: Any) -> dict[str, Any]: """run.origin 事件的数据:kind 加上调用方给的字段(app_id、app_version、spec_hash、entry / view、hook、 diff --git a/backend/app/engine/runner.py b/backend/app/engine/runner.py index ef22bd1e..25c5e5e1 100644 --- a/backend/app/engine/runner.py +++ b/backend/app/engine/runner.py @@ -23,7 +23,7 @@ from app.db.base import SessionLocal from app.db.models import Approval, Run, RunEvent, Workflow from app.engine.approval import read_decision from app.engine.compiler import compile_graph, initial_state -from app.engine.context import NodeError, RunContext +from app.engine.context import NodeError, NodeRejected, RunContext from app.engine.deadline import segment from app.engine.replay import PROTOCOL, PROTOCOL_KEY, protocol_of from app.engine.schema import GraphSpec, NodeType, loop_steps, topology_of, validate_graph @@ -42,9 +42,18 @@ _EPHEMERAL = {EventType.LLM_TOKEN, EventType.LLM_THINKING_DELTA} RUN_OPTIONS_KEY = "run_options" -#: 接着跑被拒的机器码:正式运行带图(F2),有结果未知、还没核实的写入 +#: 接着跑被拒的机器码:正式运行带图(F2),有结果未知、还没核实的写入,被人驳回了的 FORMAL_GRAPH_LOCKED = "formal_graph_locked" ACTION_UNRESOLVED = "action_unresolved" +RUN_REJECTED = "run_rejected" + +#: 运行为什么结束在 cancelled(Run.end_reason,docs/design/next-round/00-总方案.md X9) +END_REJECTED = "rejected" +END_STOPPED = "stopped" +#: 被驳回的运行不能继续运行:从断点重跑会读到同一个驳回答复、再停在同一处 +REJECTED_CONTINUE = "被驳回的运行不能继续运行:改好内容后重新发起" +#: 服务重启时还在排队的运行改成失败时写的话(run.error 和补发的 run.failed 是同一句) +QUEUED_AT_RESTART = "服务重启时这次运行仍在排队,没有可恢复的进度,请重新发起运行" class ContinueRefused(ValueError): @@ -94,6 +103,9 @@ class RunManager: stranded = list((await session.execute( select(Run.id, Run.last_seq).where(Run.status == "running") )).all()) + queued = list((await session.execute( + select(Run.id, Run.last_seq).where(Run.status == "queued") + )).all()) await session.execute( update(Run) .where(Run.status == "running") @@ -102,7 +114,7 @@ class RunManager: await session.execute( update(Run) .where(Run.status == "queued") - .values(status="failed", error="服务重启时这次运行仍在排队,没有可恢复的进度,请重新发起运行") + .values(status="failed", error=QUEUED_AT_RESTART) ) await session.commit() # 进程被强杀时连 server_shutdown 日志都来不及发,事件流停在半路。补上这一条, @@ -113,6 +125,11 @@ class RunManager: "level": "warn", "code": "server_shutdown", "message": "服务上次未正常关闭,这次运行已中断;可从断点继续运行", }) + # 排队中的改成了失败,也补一条 run.failed:以前只改状态,时间线里没有失败原因这一行(05-已知遗留 (h))。 + # 不写结束时间、不封存:它从没开始执行,同以前(定时通知按「没有开始时间」认出它,scheduler/webhook.py) + for run_id, last_seq in queued: + bus.set_seq(run_id, last_seq or 0) + await self._emit(run_id, EventType.RUN_FAILED, data={"error": QUEUED_AT_RESTART}) async def shutdown(self) -> None: self._closing = True @@ -505,8 +522,13 @@ class RunManager: Approval.run_id == run_id, Approval.status.in_(["pending", "answered"]) ).limit(1) )).first() is not None + if run.status == "cancelled" and run.end_reason == END_REJECTED: + raise ContinueRefused(REJECTED_CONTINUE, code=RUN_REJECTED) if not (run.status == "failed" or (run.status == "interrupted" and not has_pending)): raise ValueError(_cannot_continue(run.status, has_pending)) + if run.status == "failed" and await _rejected_before(session, run): + # 加 end_reason 之前被驳回的运行记的是 failed(封存不改写):接着跑会读到同一个驳回答复、再停一次 + raise ContinueRefused(REJECTED_CONTINUE, code=RUN_REJECTED) if graph is not None and run.run_class == "formal": raise ContinueRefused( "正式运行固定在发布的版本上,不能修改配置后继续运行。如需修改,请在画布上修改并发布新版本后重新发起", @@ -613,7 +635,7 @@ class RunManager: # 或者另一个取消先到了,这里就什么都不做 claimed = await session.execute( update(Run).where(Run.id == run_id, Run.status == "interrupted") - .values(status="cancelled", finished_at=now) + .values(status="cancelled", end_reason=END_STOPPED, finished_at=now) ) if claimed.rowcount != 1: await session.rollback() @@ -631,7 +653,7 @@ class RunManager: bus.set_seq(run_id, await _seq_floor(session, run_id, run.last_seq)) # 和状态同一个事务提交,理由同 _finalize event = self._event(run_id, EventType.RUN_CANCELLED, data={ - "timing": timing, "actor": actor, + "reason": END_STOPPED, "timing": timing, "actor": actor, "message": f"放弃了这次运行,{closed} 条待审批一并关闭" if closed else "放弃了这次挂起的运行", }) @@ -684,6 +706,8 @@ class RunManager: #: 终态事件(run.failed / run.cancelled)等 _finalize 算完耗时再发,timing 才带得上 terminal: tuple[EventType, dict[str, Any]] | None = None error_node: str | None = None + #: 收成 cancelled 的原因(Run.end_reason):rejected 人的驳回,stopped 手动停止 + end_reason: str | None = None #: 结果未知、挂起等核实的动作(ActionPending),收尾之后发 action.unknown、安排核实 pending_action: str | None = None @@ -796,7 +820,9 @@ class RunManager: else: status = "cancelled" error = "用户取消" - terminal = (EventType.RUN_CANCELLED, {"actor": self._stopped_by.pop(run_id, None)}) + end_reason = END_STOPPED + terminal = (EventType.RUN_CANCELLED, {"reason": END_STOPPED, + "actor": self._stopped_by.pop(run_id, None)}) raise except ActionPending as e: # 写外部系统的结果未知(app/process/actions.py):不在节点里等,运行收成挂起、不封存,等核实之后 @@ -829,6 +855,12 @@ class RunManager: "(AGENTLAB_MAX_GRAPH_STEPS)" ) terminal = (EventType.RUN_FAILED, {"error": error}) + except NodeRejected as e: + # 人的决定(驳回、超时驳回),不是故障:收成 cancelled,同手动停止走一样的收尾(关掉并行支路上的审批、 + # 回收沙箱、关流、封存、不许继续运行)。原话照旧记在 error 上,在哪一步、谁、结论、备注进 run.cancelled + status, end_reason = "cancelled", END_REJECTED + error, error_node = str(e), e.node_id + terminal = (EventType.RUN_CANCELLED, rejected_data(e, spec)) except Exception as e: # noqa: BLE001 status = "failed" error, terminal = _failure(e, spec) @@ -840,7 +872,7 @@ class RunManager: self._finalizing.add(run_id) try: await self._finalize(run_id, status, error, final_state, elapsed, - terminal=terminal, error_node=error_node) + terminal=terminal, error_node=error_node, end_reason=end_reason) finally: # 收尾被打断(关停时的取消、库连接出错)也得把自己摘掉。以前写在 # _finalize 后面,一被打断就漏掉,留下的任务让下一次 shutdown 的 @@ -944,6 +976,7 @@ class RunManager: *, terminal: tuple[EventType, dict[str, Any]] | None = None, error_node: str | None = None, + end_reason: str | None = None, ) -> None: usage = dict(state.get("usage") or {}) output = dict(state.get("output") or {}) @@ -1000,6 +1033,7 @@ class RunManager: run.output = output run.usage = usage run.error_node_id = error_node + run.end_reason = end_reason if status != "interrupted": run.finished_at = now if status in ("failed", "cancelled"): @@ -1024,11 +1058,12 @@ class RunManager: # 关停时挂起的也要让在线的客户端收尾:进程马上就没了,不说一声, # 界面会一直以为它还在跑 await bus.close(run_id) - if app_id and status in ("succeeded", "failed"): - # 应用实例结束、失败:按应用订阅推送(app/notify/service.py)。放到后台,接收端慢不拖着收尾 + if app_id and (status in ("succeeded", "failed") or end_reason == END_REJECTED): + # 应用实例结束、失败:按应用订阅推送(app/notify/service.py)。放到后台,接收端慢不拖着收尾。被驳回的走 + # 「实例结束」(next-round X6):发起人要知道结果,订阅「实例失败」的人盯的是故障;手动停止的不推,同以前 from app.notify.service import notify_run, spawn - spawn(notify_run(run_id, "run.finished" if status == "succeeded" else "run.failed")) + spawn(notify_run(run_id, "run.failed" if status == "failed" else "run.finished")) async def _seal(self, run_id: str) -> None: """对到此为止的全部事件算清单哈希,记下封到了哪一条。""" @@ -1269,6 +1304,22 @@ def _trust_event(tool_trust: dict[str, str] | None) -> Any: return "formal" if tool_trust.get(FORMAL_MARK) else snapshot_levels(tool_trust) +def rejected_data(exc: NodeRejected, spec: GraphSpec) -> dict[str, Any]: + """被驳回的 run.cancelled 带什么:原因(rejected)、在哪一步(节点 id 和图上的名字)、哪种关卡、结论(驳回 / 超时 + 驳回)、谁、备注。进封存:谁在哪一步驳回、为什么,事后查得到。""" + node = spec.node_map().get(exc.node_id) + return {"reason": END_REJECTED, "node_id": exc.node_id, "label": node.title if node else exc.node_id, + "kind": exc.kind, "decision": exc.decision, "actor": exc.actor, "note": exc.note} + + +async def _rejected_before(session: Any, run: Run) -> bool: + """加 end_reason 之前被人驳回、记成 failed 的运行:失败的那一步上最近一条处理完的待办是驳回(和实例页认驳回的是 + 同一份判断,apps/resolve.legacy_rejected)。""" + from app.apps.resolve import legacy_rejected + + return await legacy_rejected(session, run) + + def _failure(exc: BaseException, spec: GraphSpec) -> tuple[str, tuple[EventType, dict[str, Any]]]: """失败的首行报错(给人看)和 run.failed 的负载(带上定位和原始异常)。""" if isinstance(exc, NodeError): diff --git a/backend/app/engine/spend.py b/backend/app/engine/spend.py index 0efa52a6..87510ba7 100644 --- a/backend/app/engine/spend.py +++ b/backend/app/engine/spend.py @@ -1,11 +1,18 @@ -"""核查花费账:双盲复算、反方审稿、审批质询里的模型调用,按本地日期记一本每日账。 +"""模型调用的每日花费账:一本账一个键,按本地日期记,先预留、后找补(docs/design/next-round/01-效果度量与回归评测.md 做法 17)。 -照结论句裁判的每日计数做(judge.py 的 SPEND_KEY、_charge、_settle):调用前按目录价估算、先预留,加上这笔会超过 -每日上限就不记;调用后按实际用量找补差额。账落在设置表里,按本地日期滚动。 +调用前按目录价估算、先预留,加上这笔会超过每日上限就不记;调用后按实际用量找补差额。账落在设置表里,按本地日期 +滚动,不从设置接口进出(api/settings.py 只给只读的「今天花了多少」)。 -**为什么和裁判分开记。** 核查是一个 agent 查库、来回好几轮,一次花的钱比裁判判一份报告多得多。记在同一本账上, -核查一多,当天的裁判就全因为「每日上限」判不了——报告的出具会因此降档,而降档的原因和这份报告毫无关系。两本账 -各管各的上限,互不挤占。 +**一套代码,几本账。** 以前核查(这个模块)和结论句裁判(judge.py 的 _charge、_settle)各写一份一模一样的读改写,评测 +要是再抄一份就是第三份。现在都是 Ledger(key) 的实例,各记各的键: + +- verify_spend:核查——双盲复算、反方审稿、审批质询里的模型调用; +- judge_spend:结论句裁判(实例在 engine/judge.py,键 judge_spend:日历跟着 judge._today,裁判的测试换它模拟跨天); +- eval_spend:评测(预留,第 3 波的评测执行接上;设置和接口在那时加)。 + +**为什么各记各的。** 核查是一个 agent 查库、来回好几轮,一次花的钱比裁判判一份报告多得多;评测一次几十道题。记在同一本 +账上,一边一多,另一边当天就全因为「每日上限」做不了——报告的出具会因此降档,而降档的原因和这份报告毫无关系。几本账 +各管各的上限,互不挤占。三本按同一个本地日期滚动(_today)。 目录里没有价格的模型估不出金额:只计次数(unpriced_calls),金额上限对它不起作用,由调用方照实说明。 """ @@ -13,27 +20,26 @@ from __future__ import annotations import asyncio import weakref +from collections.abc import Callable from datetime import datetime from typing import Any from app.db.base import SessionLocal from app.db.models import Setting -#: 每日计数存在设置表的这一行:{date: 本地日期, usd, calls, unpriced_calls}。不从设置接口进出(api/settings.py) -SPEND_KEY = "verify_spend" - -_locks: "weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, asyncio.Lock]" = weakref.WeakKeyDictionary() +#: 每本账的读改写锁:{事件循环: {键: 锁}}。asyncio.Lock 不能跨循环用(测试每个用例一个循环),按循环各一组;同一个键 +#: 同一把——几个核查并行时各自预留,不串行的话两边读到同一个余额,会一起越过上限 +_locks: "weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, dict[str, asyncio.Lock]]" = weakref.WeakKeyDictionary() -def _lock() -> asyncio.Lock: - """读改写这一行时持的锁,按事件循环各一把(asyncio.Lock 不能跨循环用)。 - - 几个核查并行时各自预留:不串行的话两边读到同一个余额,会一起越过上限。 - """ +def _lock(key: str) -> asyncio.Lock: loop = asyncio.get_running_loop() - lock = _locks.get(loop) + group = _locks.get(loop) + if group is None: + group = _locks[loop] = {} + lock = group.get(key) if lock is None: - lock = _locks[loop] = asyncio.Lock() + lock = group[key] = asyncio.Lock() return lock @@ -42,57 +48,90 @@ def _today() -> str: return datetime.now().astimezone().date().isoformat() -def _spend_of(row: Any) -> dict[str, Any]: - value = row.value if row is not None and isinstance(row.value, dict) else {} - today = _today() - if value.get("date") != today: - return {"date": today, "usd": 0.0, "calls": 0, "unpriced_calls": 0} - return {"date": today, "usd": float(value.get("usd") or 0.0), "calls": int(value.get("calls") or 0), - "unpriced_calls": int(value.get("unpriced_calls") or 0)} +class Ledger: + """一本每日账,存在设置表 key 这一行:{date: 本地日期, usd, calls, unpriced_calls}。 + + today 是这本账的日历(测试换掉它来模拟跨天);不给就按本模块的 _today,调用时才取,换掉 _today 对没给 today 的几本 + 账都生效。""" + + def __init__(self, key: str, *, today: Callable[[], str] | None = None) -> None: + self.key = key + self._clock = today + + def _date(self) -> str: + return self._clock() if self._clock is not None else _today() + + def _spend_of(self, row: Any) -> dict[str, Any]: + value = row.value if row is not None and isinstance(row.value, dict) else {} + today = self._date() + if value.get("date") != today: + return {"date": today, "usd": 0.0, "calls": 0, "unpriced_calls": 0} + return {"date": today, "usd": float(value.get("usd") or 0.0), "calls": int(value.get("calls") or 0), + "unpriced_calls": int(value.get("unpriced_calls") or 0)} + + async def daily(self) -> dict[str, Any]: + """今天(本地日期)这本账已经花了多少:{date, usd, calls, unpriced_calls}。估不出金额的调用只计次数。""" + async with SessionLocal() as session: + return self._spend_of(await session.get(Setting, self.key)) + + async def charge(self, amount: float, *, limit: float | None, priced: bool) -> str | None: + """记一次调用、先按估算预留金额。加上这笔会超过每日上限就不记,返回 None;记下了返回记在哪天(找补时要用)。 + + limit 为 None 表示不限。估不出金额的调用(priced=False)照记次数,金额记 0,调用方应传 limit=None: + 拿 0 去比上限,等于说它不花钱。 + """ + async with _lock(self.key): + async with SessionLocal() as session: + row = await session.get(Setting, self.key) + spend = self._spend_of(row) + if limit is not None and spend["usd"] + amount > limit + 1e-12: + return None + spend["usd"] = round(spend["usd"] + amount, 8) + spend["calls"] += 1 + spend["unpriced_calls"] += 0 if priced else 1 + if row is None: + session.add(Setting(key=self.key, value=spend)) + else: + row.value = spend + await session.commit() + return spend["date"] + + async def settle(self, date: str, delta: float) -> None: + """调用结束后按实际用量找补预留的差额。跨了天就不找补:预留记在前一天,今天从零算。""" + if not delta: + return + async with _lock(self.key): + async with SessionLocal() as session: + row = await session.get(Setting, self.key) + spend = self._spend_of(row) + if row is None or spend["date"] != date: + return + spend["usd"] = max(0.0, round(spend["usd"] + delta, 8)) + row.value = spend + await session.commit() + + +#: 核查、评测各一本(裁判那本在 engine/judge.py 的 judge_spend) +verify_spend = Ledger("verify_spend") +eval_spend = Ledger("eval_spend") + +#: 核查那本账的老名字(verify_agent、设置接口、已有测试从这里取) +SPEND_KEY = verify_spend.key async def daily_spend() -> dict[str, Any]: - """今天(本地日期)核查已经花了多少:{date, usd, calls, unpriced_calls}。估不出金额的调用只计次数。""" - async with SessionLocal() as session: - return _spend_of(await session.get(Setting, SPEND_KEY)) + """今天核查已经花了多少(verify_spend.daily)。""" + return await verify_spend.daily() async def charge(amount: float, *, limit: float | None, priced: bool) -> str | None: - """记一次调用、先按估算预留金额。加上这笔会超过每日上限就不记,返回 None;记下了返回记在哪天(找补时要用)。 - - limit 为 None 表示不限。估不出金额的调用(priced=False)照记次数,金额记 0,调用方应传 limit=None: - 拿 0 去比上限,等于说它不花钱。 - """ - async with _lock(): - async with SessionLocal() as session: - row = await session.get(Setting, SPEND_KEY) - spend = _spend_of(row) - if limit is not None and spend["usd"] + amount > limit + 1e-12: - return None - spend["usd"] = round(spend["usd"] + amount, 8) - spend["calls"] += 1 - spend["unpriced_calls"] += 0 if priced else 1 - if row is None: - session.add(Setting(key=SPEND_KEY, value=spend)) - else: - row.value = spend - await session.commit() - return spend["date"] + """核查记一次调用(verify_spend.charge)。""" + return await verify_spend.charge(amount, limit=limit, priced=priced) async def settle(date: str, delta: float) -> None: - """调用结束后按实际用量找补预留的差额。跨了天就不找补:预留记在前一天,今天从零算。""" - if not delta: - return - async with _lock(): - async with SessionLocal() as session: - row = await session.get(Setting, SPEND_KEY) - spend = _spend_of(row) - if row is None or spend["date"] != date: - return - spend["usd"] = max(0.0, round(spend["usd"] + delta, 8)) - row.value = spend - await session.commit() + """核查找补(verify_spend.settle)。""" + await verify_spend.settle(date, delta) -__all__ = ["SPEND_KEY", "charge", "daily_spend", "settle"] +__all__ = ["Ledger", "SPEND_KEY", "charge", "daily_spend", "eval_spend", "settle", "verify_spend"] diff --git a/backend/app/engine/verify_agent.py b/backend/app/engine/verify_agent.py index 2127610c..f44d5c03 100644 --- a/backend/app/engine/verify_agent.py +++ b/backend/app/engine/verify_agent.py @@ -758,6 +758,7 @@ async def run_verify_agent( node_id: str = "", data_versions: dict[str, Any] | Versions | None = Versions.UNSET, masked_values: Any = None, + catalog_view: Any = None, ) -> VerifyOutcome: """让一个只读 agent 查库,按 output_schema 交回有出处的结果。永远不抛。 @@ -775,6 +776,8 @@ async def run_verify_agent( - masked_values:调用方已经认出的遮罩值(engine/masking 的 _Secrets,审批质询、反方审稿收的报告快照里的值)。 查询工具从第一条查询起就按它们按值遮:agent 查到的备注、别的表里写着同一个号码时,模型看不到。随查询时遮掉的值 一起交回(VerifyOutcome.masked_values)。不改它。 + - catalog_view:db_schema 给 agent 看哪一份数据目录(data/catalog_view.py)。运行里的节点传这次运行的视图 + (catalog_view.of_run);不给是当前目录(质询接口)。 """ state = _State(started=time.monotonic(), seen=SeenValues( given=masked_values if masked_values is not None else _Secrets())) @@ -782,7 +785,7 @@ async def run_verify_agent( out = await _verify(state, spec=spec, system=system, task=task, sources=sources, output_schema=output_schema, max_queries=max_queries, timeout_s=timeout_s, max_cost_usd=max_cost_usd, daily_max_usd=daily_max_usd, max_steps=max_steps, emit=emit, - run_id=run_id, node_id=node_id, data_versions=data_versions) + run_id=run_id, node_id=node_id, data_versions=data_versions, catalog_view=catalog_view) except Exception as e: # noqa: BLE001 - 永远不抛:库写不进去、建工具出错这类意外也落在返回值里 logger.warning("核查出错", exc_info=True) out = state.outcome(error=f"核查出错({describe_exception(e)})") @@ -798,6 +801,7 @@ async def run_verify_agent( async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sources: Iterable[str], output_schema: dict[str, Any], max_queries: int | None, timeout_s: float | None, max_cost_usd: float | None, daily_max_usd: float | None | Default, max_steps: int | None, + catalog_view: Any = None, emit: Emit | None, run_id: str, node_id: str, data_versions: dict[str, Any] | Versions | None) -> VerifyOutcome: deadline = state.started + timeout_s if timeout_s is not None else None @@ -834,7 +838,7 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou # ---- 工具:只读的 db_schema / db_query,查询结果先遮再交给模型(遮掉的值并进 state.seen) wanted = list(dict.fromkeys(s for s in sources if isinstance(s, str) and s)) - ctx = ToolContext(run_id=run_id, node_id=node_id, data_versions=data_versions, emit=emit, + ctx = ToolContext(run_id=run_id, node_id=node_id, data_versions=data_versions, emit=emit, catalog_view=catalog_view, extra={MASKED_VALUES_KEY: state.seen}) async with SessionLocal() as session: tools = await build_datasource_tools([f"{p}{s}" for s in wanted for p in (SCHEMA_PREFIX, QUERY_PREFIX)], ctx, diff --git a/backend/app/look/tables.py b/backend/app/look/tables.py index 25fe1e5c..0d4e61d4 100644 --- a/backend/app/look/tables.py +++ b/backend/app/look/tables.py @@ -159,12 +159,16 @@ class TableKind: @classmethod async def load(cls, sources: list[Any], *, graph: Mapping[str, Any] | None = None, - picker: Picker | None = None) -> TableKind: - """读各源的数据目录。graph 是现有工作流(改图时的原图、追问时上一轮的图),概览里列出它查过的表。""" - from app.tools.datasource import _catalog_of + picker: Picker | None = None, catalog_view: Any = None) -> TableKind: + """读各源的数据目录。graph 是现有工作流(改图时的原图、追问时上一轮的图),概览里列出它查过的表。 + 读哪一份目录由 catalog_view 定(data/catalog_view.py:当前目录、目录快照、关;只用已核实的项),不给是当前目录。 + 查看结果里的目录和运行时 db_schema 给模型看的、自查用的是同一份——评测的「目录对照」量的才是整份贡献。""" + from app.data.catalog_view import CatalogView, read + + view = CatalogView.parse(catalog_view) seeds = graph_tables(graph, sources) - plans = [SourcePlan(source=s, entries=await _catalog_of(s) if introspect.table_names(s) else {}, + plans = [SourcePlan(source=s, entries=await read(view, s) if introspect.table_names(s) else {}, seeds=seeds.get(s.name, [])) for s in sources] return cls(plans, picker=picker) diff --git a/backend/app/main.py b/backend/app/main.py index 657c917a..045cb390 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -30,6 +30,7 @@ from app.core.config import settings from app.db.base import init_db from app.engine.runner import run_manager from app.seed import seed_defaults +from app.web_security import SecurityHeaders logging.basicConfig( level=logging.INFO, @@ -145,16 +146,20 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: async def _platform_startup() -> None: - """应用平台的启动整理(BE-1):登录设置读进缓存;自定义工具请求头、MCP env 和请求头里的存量明文加密一遍(幂等); - 存量的内网 MCP 服务改为绑定登记端点(只在升级后第一次启动时做);能力登记同步一次。后三样出错只记日志:最坏是登记 - 暂时不全,不该让服务起不来。""" + """应用平台的启动整理(BE-1):登录设置、对外访问地址读进缓存;自定义工具请求头、MCP env 和请求头里的存量明文加密 + 一遍(幂等);存量的内网 MCP 服务改为绑定登记端点(只在升级后第一次启动时做);能力登记同步一次。后三样出错只记 + 日志:最坏是登记暂时不全,不该让服务起不来。""" from app.auth import current_mode, load_settings from app.capabilities import registry as capability_registry from app.capabilities.secrets import seal_existing + from app.core import public_url from app.db.base import SessionLocal await load_settings(force=True) logger.info("登录方式:%s", current_mode()) + # 对外访问地址(core/public_url.py)同样读进缓存:中间件核对来源时不查库 + if address := await public_url.refresh(): + logger.info("对外访问地址:%s(%s)", address, "环境变量" if public_url.env_locked() else "设置") try: if sealed := await seal_existing(): logger.info("已加密 %d 处存量的明文密钥(自定义工具请求头、MCP 环境变量和请求头)", sealed) @@ -192,6 +197,8 @@ app.add_middleware( allow_methods=["*"], allow_headers=["*"], ) +# 安全响应头(app/web_security.py):最后加、在最外层,CORS 预检和登录检查自己回的 401 / 403 也带上 +app.add_middleware(SecurityHeaders) # 路由级权限(§3.12)。关登录时这些依赖全部放行;开了登录,中间件先挡掉没登录的,这里再按角色和运行挡 _BUILDER = [require_role("builder")] @@ -238,6 +245,7 @@ for router, guard in ( (identity_api.router, _ADMIN), # 选人用的名录(用户名、显示名、是否停用、组名):岗位绑定、应用授权由搭建者做,只读 (identity_api.directory_router, _BUILDER), + (identity_api.candidates_router, _USER), # 能力登记、端点:搭建者只读,改动在接口里要管理员 (capabilities_api.router, _BUILDER), # 普通用户建应用:个人应用的接口共用 /api/apps 前缀,要在 apps_api 之前登记;蓝图助手共用 /api/copilot 前缀, diff --git a/backend/app/notify/service.py b/backend/app/notify/service.py index a836574a..b2b1f218 100644 --- a/backend/app/notify/service.py +++ b/backend/app/notify/service.py @@ -7,15 +7,29 @@ 同一个渠道只推一次。待办改派、升级、催办都是「有一条待办要你处理」,订阅了 task.opened 的渠道一并收到。 消息里只放标题、状态、链接(P4,沿用定时任务 Webhook 的规矩):不放报告里的数字,待办的标题用节点在图上的名字 -(搭图时起的),不用按运行数据渲染出来的那一句。链接要设了对外访问地址(设置里的 notify)才有。 +(搭图时起的),不用按运行数据渲染出来的那一句。链接要有对外访问地址(环境变量或设置,core/public_url.py)才有。 推送放在后台(spawn):接收端挂住时超时加重试要几十秒,不能拖着开待办、运行收尾。测试用 drain 等它们推完。 + +事件只在这里登记(EVENTS,总方案 X6):每个事件一行,名字、界面上的叫法(前端 lib/terms.ts 的 NOTIFY_TEXT.eventLabel +逐字一致,test_notify_events 对照)、推给哪几条路(scopes:global 全局渠道自己订阅、user 相关的人自己的个人渠道、app 应用 +版本 spec.notify 的订阅)。_channels 只走登记了的路,没登记的事件推不出去;接口按 scopes 收订阅,设置页和应用的通知按 +GET /api/notify/events 列。scopes 是空的,是登记了、还没有推送的事件:不收订阅、界面上不列——勾了却收不到,就是「看着 +生效、其实没生效」(个人渠道勾了实例结束、失败却收不到,04-对外集成与触达 B1,就是这么错的)。本轮第 1 波把六个新事件 +一次登记好;实现推送的那一块写自己的 notify_*,只改自己那一行的 scopes,不再改这张表的别处、也不再改前端的叫法表:几块 +并行改同一处必然冲突。以前存下的订阅不按 scopes 重新核对(api/notify._validated 只核对新勾的)。 + +消息的结构,新事件一律照这个写(P4、D9):标题、状态、条数、链接,外加编号(#前 6 位)和搭建时起的名字(应用、工作流、 +节点在图上的名字,题集、数据源、访问密钥的名字)。条数说的是有几件事要处理(几个变化、通过几题、几条待办),可以放。 +不放:报告和答案里的数(金额、比例、差值、期望值),表名、列名、取值(码值、样例行),SQL,按运行数据渲染出来的句子。 +接收端是群机器人、外部系统,消息出了这个服务就不归遮罩管了。 """ from __future__ import annotations import asyncio import logging from collections.abc import Coroutine +from dataclasses import dataclass from typing import Any from sqlalchemy import select @@ -26,12 +40,60 @@ from app.notify.core import Message, Outcome, host_of logger = logging.getLogger(__name__) -#: 可以订阅的事件 -EVENTS = ("task.opened", "run.finished", "run.failed", "action.unknown", "review.done") -EVENT_LABEL = {"task.opened": "有待办派给我", "run.finished": "实例完成", "run.failed": "实例失败", - "action.unknown": "写入结果未知", - # 个人应用提交审核之后,搭建者发布了或退回了(docs/design/normal-user-apps.md N7) - "review.done": "我提交审核的应用有结果"} +@dataclass(frozen=True) +class Event: + """一个可订阅的通知事件(见模块说明「事件只在这里登记」)。""" + + #: 界面上的叫法:和前端 NOTIFY_TEXT.eventLabel 同一句 + label: str + #: 推给哪几条路(_channels):global 全局渠道自己订阅,user 相关的人自己的个人渠道,app 应用版本 spec.notify 的订阅。 + #: 空的:登记了、还没有推送 + scopes: tuple[str, ...] + + +_EVERY = ("global", "user", "app") +#: 渠道范围在说法里怎么叫(同前端 NOTIFY_TEXT.scopes) +SCOPE_TEXT = {"global": "全局", "user": "个人", "app": "应用"} + +#: 可以订阅的事件,只在这里登记 +EVENTS: dict[str, Event] = { + # 待办派出、改派、升级、催办(_AS):全局渠道、派给的人自己的个人渠道、应用的订阅 + "task.opened": Event("待办打开", _EVERY), + # 应用实例结束、失败:个人渠道是发起人自己的(04-对外集成与触达 B1) + "run.finished": Event("运行结束", _EVERY), + "run.failed": Event("运行失败", _EVERY), + # 写入外部系统的结果未知:个人渠道是核实待办派给的人 + "action.unknown": Event("动作结果未知", _EVERY), + # 个人应用提交审核之后,搭建者发布了或退回了(docs/design/normal-user-apps.md N7):发给提交人,不归哪个应用 + "review.done": Event("我提交审核的应用有结果", ("global", "user")), + # ---- 本轮登记、还没有推送(总方案 X6):实现推送的那一块只改自己这一行的 scopes,消息照模块说明的规矩 ---- + # 数据源有要处理的结构 / 取值变化(03 B3):只推 severity 为 action 的,按源汇总;源名、各级别条数、链接 + "source.changed": Event("数据源有变化要处理", ()), + # 口径升版后召回重算,数真变了(02 第 7 项):答案的主人有 recall 待办;口径名、版本、链接,不放新旧值和差值 + "ledger.recall": Event("口径升版,答案的数变了", ()), + # 定时评测有题从通过变成不通过 / 评测没跑完(01 第 15 项):题集名、通过数、退步题数、链接;不放期望值和答案里的数 + "eval.regressed": Event("评测有退步", ()), + "eval.failed": Event("评测失败", ()), + # 每天一条待办汇总(04 B3):只推个人渠道;条数、最早的时限、最多 5 条(节点在图上的名字)、去待办箱的链接 + "task.digest": Event("每天的待办汇总", ()), + # 访问密钥到期前 7 天、1 天各一次(04 B3):个人密钥推本人,服务账号的推全局渠道;密钥名、到期日、链接 + "key.expiring": Event("访问密钥快到期", ()), +} +EVENT_LABEL = {name: event.label for name, event in EVENTS.items()} + + +def subscribe_problem(event: str, scope: str) -> str | None: + """范围是 scope 的渠道能不能订阅 event:不能的回一句为什么(接口原样回 422),能的是 None。""" + spec = EVENTS.get(event) + if spec is None: + offered = "、".join(e.label for e in EVENTS.values() if scope in e.scopes) + return f"不认识的通知事件:{event}。可选:{offered}" + if not spec.scopes: + return f"「{spec.label}」的通知还没有推送,暂时不能订阅" + if scope not in spec.scopes: + return f"「{spec.label}」不推给范围是「{SCOPE_TEXT.get(scope, scope)}」的渠道" + return None + #: 这几种也是「有一条待办要你处理」,按 task.opened 的订阅推 _AS = {"task.reassigned": "task.opened", "task.escalated": "task.opened", "task.reminded": "task.opened"} _TASK_HEAD = {"task.opened": "新的待办", "task.reassigned": "待办已改派给你", "task.escalated": "待办已超时,升级处理", @@ -91,15 +153,19 @@ def generic_body(message: dict[str, Any], msg: Message) -> dict[str, Any]: async def _base_url(session: Any) -> str | None: - from app.api.settings import notify_settings + """拼链接用的对外访问地址:全站同一个(环境变量优先,其次设置),手里有会话就按库里的值对齐缓存。""" + from app.core import public_url - return (await notify_settings(session))["public_base_url"] + return await public_url.refresh(session) async def _channels(session: Any, event: str, run: Run | None, users: list[str]) -> list[NotifyChannel]: wanted = _AS.get(event, event) + if (registered := EVENTS.get(wanted)) is None: + raise ValueError(f"通知事件 {wanted} 没有登记:先在 notify/service.EVENTS 登记,再推送") + paths = registered.scopes found: dict[str, NotifyChannel] = {} - if run is not None and run.app_id and run.app_version is not None: + if "app" in paths and run is not None and run.app_id and run.app_version is not None: row = (await session.execute(select(AppVersion.spec, AppVersion.bindings).where( AppVersion.app_id == run.app_id, AppVersion.version == run.app_version))).first() spec, bindings = (row[0], row[1]) if row is not None else ({}, {}) @@ -115,11 +181,12 @@ async def _channels(session: Any, event: str, run: Run | None, users: list[str]) for row in (await session.execute(select(NotifyChannel).where( NotifyChannel.id.in_(subscribed), NotifyChannel.enabled.is_(True)))).scalars(): found[row.id] = row - for row in (await session.execute(select(NotifyChannel).where( - NotifyChannel.scope == "global", NotifyChannel.enabled.is_(True)))).scalars(): - if wanted in (row.events or []): - found.setdefault(row.id, row) - if users: + if "global" in paths: + for row in (await session.execute(select(NotifyChannel).where( + NotifyChannel.scope == "global", NotifyChannel.enabled.is_(True)))).scalars(): + if wanted in (row.events or []): + found.setdefault(row.id, row) + if "user" in paths and users: owners = list((await session.execute(select(User.id).where(User.username.in_(users)))).scalars()) if owners: for row in (await session.execute(select(NotifyChannel).where( @@ -201,22 +268,36 @@ async def notify_task(task_id: str, event: str, *, users: list[str]) -> int: async def notify_run(run_id: str, event: str) -> int: - """应用实例结束、失败(run.finished / run.failed)。只推应用发起的运行。""" + """应用实例结束、失败(run.finished / run.failed)。只推应用发起的运行。 + + 状态那一句取自实例页的说法(apps/resolve.instance_state),两边说同一句话:被驳回的(cancelled + end_reason=rejected) + 走「实例结束」、写「「付款」未确认(被驳回)」;走到「已退回」成果的写「已退回」,不再一律写「已完成」。ok 只在真的 + 办成了(code 是 succeeded)时为真。 + + 发起人是认证过的身份(run.origin.actor_verified)时,他自己订阅了这件事的个人渠道也收得到(「我发起的实例完成 / 失败」, + 04-对外集成与触达 B1)。以前这里给 _channels 的 users 是空的,个人渠道勾了这两项永远收不到。自报的署名不算:关登录 + 时谁都能填别人的名字。""" + from app.apps.resolve import _trails, instance_state + async with SessionLocal() as session: run = await session.get(Run, run_id) if run is None or not run.app_id: return 0 - channels = await _channels(session, event, run, []) + verified = bool((run.origin or {}).get("actor_verified")) and bool(run.started_by) + channels = await _channels(session, event, run, [run.started_by] if verified else []) if not channels: return 0 base = await _base_url(session) where = await _where(session, run) + state = instance_state(run, (await _trails(session, [run]))[run.id]) finished = event == "run.finished" - lines = [("状态", "已完成" if finished else "失败"), ("实例", f"#{run_id[:6]}")] - if not finished and run.error_node_id: + ok = state["code"] == "succeeded" + lines = [("状态", state["text"]), ("实例", f"#{run_id[:6]}")] + if not finished and run.error_node_id and state["code"] == "failed": lines.append(("失败节点", f"「{_label(run.graph, run.error_node_id)}」")) - message = {"event": event, "status": run.status, "ok": finished, - "title": f"{where or '应用'}的实例{'已完成' if finished else '失败'}", "lines": lines, + head = ("已完成" if ok else "已结束") if finished else ("已结束" if state["code"] == "cancelled" else "失败") + message = {"event": event, "status": run.status, "ok": ok, + "title": f"{where or '应用'}的实例{head}", "lines": lines, "links": [("查看实例", f"{base}/apps/{run.app_id}/i/{run_id}")] if base else [], "subject": {"run_id": run_id, "app_id": run.app_id}} return await _push(channels, message, subject_kind="run", subject_id=run_id, event=event, run_id=run_id, diff --git a/backend/app/plain.py b/backend/app/plain.py index 617afe92..6c38d3a4 100644 --- a/backend/app/plain.py +++ b/backend/app/plain.py @@ -181,6 +181,9 @@ PLAIN: dict[str, Plain] = { "plan.trial_failed": Plain( "最近一次试运行失败了", "用真数运行出了错,这样发布出去,每次出数都会是同一个错。", _Y, "回到试运行看出错的原因(可以交给助手修),修好后再用真数试一次。"), + "plan.trial_rejected": Plain( + "最近一次试运行被驳回、没有走完", "那是你(或审批人)的决定,不是出错;驳回之后的步骤没用真数看过。", _Y, + "要看完整结果,再试运行一次并批准;确认不需要的话可以直接发布。"), "plan.untrialed": Plain( "还没用真数试运行过", "没看过真数就发布,可能不是你要的结果。", _Y, "先试运行一次,看过结果再发布。"), "plan.trial_differs": Plain( diff --git a/backend/app/process/actions.py b/backend/app/process/actions.py index e75a269e..87ab7fca 100644 --- a/backend/app/process/actions.py +++ b/backend/app/process/actions.py @@ -572,17 +572,18 @@ async def _continue(run_id: str, actor: str | None) -> Any: return None -async def tick(now: datetime | None = None) -> int: - """调度循环每一拍:对登记了 lookup 的结果未知动作带退避地查询。返回查了几条。""" +async def tick(now: datetime | None = None) -> list[str]: + """调度循环每一拍:对登记了 lookup 的结果未知动作带退避地查询。交回这一拍查过的动作记录 id(同 tasks.tick)。""" now = now or utcnow() async with SessionLocal() as session: due = list((await session.execute(select(ActionRecord.id, ActionRecord.next_lookup_at).where( ActionRecord.state == "unknown", ActionRecord.next_lookup_at.is_not(None), ActionRecord.next_lookup_at <= now))).all()) - done = 0 + done: list[str] = [] for action_id, claimed_at in due: try: - done += await _lookup(action_id, claimed_at, now) + if await _lookup(action_id, claimed_at, now): + done.append(action_id) except Exception: # noqa: BLE001 - 一条出错不拖住这一拍的其他动作 logger.exception("动作 %s 的结果查询失败,下一拍再试", action_id) return done diff --git a/backend/app/process/tasks.py b/backend/app/process/tasks.py index 9f1dd240..818b4f76 100644 --- a/backend/app/process/tasks.py +++ b/backend/app/process/tasks.py @@ -406,6 +406,22 @@ def _check_hash(card: dict[str, Any], given: Any, *, approving: bool) -> None: status_code=422, code="payload_hash_mismatch") +async def _check_principals(session: AsyncSession, task: Task, principal: Any, values: dict[str, Any]) -> None: + """待办表单里的选人控件:开了登录时按候选池核对(同入口表单,auth/directory.principal_errors)。不属于应用的待办 + (试运行、画布)没有池,不核对。""" + from app.auth.directory import principal_errors + + if not task.app_id or getattr(principal, "mode", "off") == "off": + return + app = await session.get(App, task.app_id) + if app is None: + return + bindings = await session.scalar(select(AppVersion.bindings).where( + AppVersion.app_id == task.app_id, AppVersion.version == task.app_version)) if task.app_version else None + if errors := await principal_errors(session, app, bindings, task.form, values, principal.mode): + raise FormInvalid(errors) + + def _validated(form: dict[str, Any], values: dict[str, Any]) -> dict[str, Any]: from app.apps.forms import validate_form @@ -491,6 +507,8 @@ async def respond(task_id: str, principal: Any, *, decision: Any, values: dict[s rows = (await _assignments(session, [task.id])).get(task.id, []) mine = _pick(task, rows, principal) response, verdict = _engine_response(task, approval, decision, values, note, payload_hash) + if task.form and verdict in ("approve", "submit") and values: + await _check_principals(session, task, principal, values) response = await _bind_files(session, task, principal, response) now = utcnow() marked = await session.execute( @@ -761,8 +779,9 @@ async def _notify(task_id: str, event: str, users: list[str]) -> int: # -------------------------------------------------------------------------- -async def tick(now: datetime | None = None) -> int: - """调度循环每一拍:提醒、升级、超时驳回(永不自动批准)。返回处理的条数。试运行的待办不处理(只显示时限)。""" +async def tick(now: datetime | None = None) -> list[str]: + """调度循环每一拍:提醒、升级、超时驳回(永不自动批准)。交回这一拍处理过的待办 id(生效可查:办了哪几条, + 不只是几条;测试按自己造的那条断言,不受同一个库里别的待办影响)。试运行的待办不处理(只显示时限)。""" now = now or utcnow() action = func.json_extract(Task.on_timeout, "$.action") trial = select(Run.id).where(Run.origin_kind == TRIAL_ORIGIN) @@ -773,10 +792,11 @@ async def tick(now: datetime | None = None) -> int: and_(Task.due_at.is_not(None), Task.due_at <= now, action == "reject"), and_(Task.due_at.is_not(None), Task.due_at <= now, action == "escalate", Task.escalation_level == 0), )).order_by(Task.due_at))).scalars()) - done = 0 + done: list[str] = [] for task_id in due: try: - done += await _on_timeout(task_id, now) + if await _on_timeout(task_id, now): + done.append(task_id) except Exception: # noqa: BLE001 - 一条出错不拖住这一拍的其他待办 logger.exception("待办 %s 的超时处理失败,下一拍再试", task_id) return done @@ -838,7 +858,9 @@ async def _on_timeout(task_id: str, now: datetime) -> int: from app.engine.runner import run_manager try: - await run_manager.resume(run_id, {"approved": False, "note": EXPIRED_NOTE}, approval_id=approval_id) + # expired:节点据此把这次驳回记成超时驳回(engine/approval.Decision.expired → NodeRejected.decision) + await run_manager.resume(run_id, {"approved": False, "note": EXPIRED_NOTE, "expired": True}, + approval_id=approval_id) except (ValueError, KeyError): logger.warning("待办 %s 超时驳回时运行已无法恢复", task_id) async with SessionLocal() as session: diff --git a/backend/app/providers/budget.py b/backend/app/providers/budget.py index 3d52402a..78b31f31 100644 --- a/backend/app/providers/budget.py +++ b/backend/app/providers/budget.py @@ -18,6 +18,7 @@ engine/nodes/llm.py(cite_fields 的抽取)、data/recipe_ai.py 各有一份 from __future__ import annotations from collections.abc import Awaitable, Callable, Hashable +from contextlib import aclosing from typing import Any #: 服务方嫌额度太大时报错里会出现的字样:超过模型的输出上限(DeepSeek chat 系只到 8K), @@ -167,19 +168,24 @@ class BudgetFallback: return await (await self._current()).ainvoke(messages, *args, **kwargs) async def astream(self, messages: Any, *args: Any, **kwargs: Any): + # 里面那条流用 aclosing 包着:调用方读到一半不读了(助手读到 done、look 就关流),关这一层时里面那条也当场关掉、 + # 模型那一次调用当场收尾(LangChain 的回调这时把已经收到的用量交出来,providers/usage.py)。不包的话里面那条要等 + # 垃圾回收才关,收尾落在调用方算完账之后 used = await self._current() started = False try: - async for chunk in used.astream(messages, *args, **kwargs): - started = True - yield chunk + async with aclosing(used.astream(messages, *args, **kwargs)) as stream: + async for chunk in stream: + started = True + yield chunk return except Exception as e: if started or not self._can_fall_back(used) or not self._switch.rejected(e): raise self._switch.fall_back() - async for chunk in (await self._current()).astream(messages, *args, **kwargs): - yield chunk + async with aclosing((await self._current()).astream(messages, *args, **kwargs)) as stream: + async for chunk in stream: + yield chunk def _derived(self, make: Callable[[Any], Any]) -> BudgetFallback: """对大小两个模型做同一件事(绑工具、要结构化输出),包成共用开关的一层。小的还没建时,建好再做。""" diff --git a/backend/app/providers/usage.py b/backend/app/providers/usage.py new file mode 100644 index 00000000..37563e6b --- /dev/null +++ b/backend/app/providers/usage.py @@ -0,0 +1,204 @@ +"""模型用量计量:助手建图(含挑表、接着写、自查修正)和复核这两段花了多少(细案 01 做法 3)。 + +**为什么要。** 运行按 llm.end 事件记用量,问数据一轮里另外两块——助手建图、复核——一直不记:这一轮的花费是低报的, +评测也算不出按角色的成本。在每个调用点手写累计(生成、挑表、退回小额度重来、复核各一处)迟早漏掉一处,所以用 LangChain +的回调在模型这一层统一截:和 langchain_core.callbacks.get_usage_metadata_callback 同一个机制(一个登记过的上下文变量, +模型每次调用时把它挂成回调),按模型累计 input / output tokens 和调用次数。 + +**为什么不直接用 get_usage_metadata_callback。** 它是一个 with 块,块里设上下文变量、出块时按 token 复原。生成过程是一个 +async generator:with 块跨着 yield 的话,变量在两次 yield 之间留在调用方的上下文里——调用方这时去发起运行,运行的任务 +会继承它,运行里的调用就被重复计一遍;调用方中途不迭代了、在别的任务里收尾,复原还会报「token 不属于这个上下文」。 +这里只在每一步 __anext__ 期间设上变量、交出操作之前复原(metered):生成过程里起的任务(心跳那一层每一步一个任务) +在创建时拷走上下文,照样计到;调用方自己的代码一行都不在计量里。 + +**金额。** 按 providers/catalog.estimate_cost 算,和运行的 llm.end 一个口径。估不出的调用记 unpriced_calls:模型不在价格 +目录里,或者用量没报全而目录价又不是 0。估不出的不当成 0 元,调用方据此写「≥」或「金额无法估算」。用量没报全有两种: +回复没带用量(有的网关流式不给);读到一半就不读了——生成过程读到 done、look、ask 那一行就关掉模型的流(后面的闲话不要、 +也不再为它付钱),用量在流的末尾才报全(Anthropic 的输出 token 在最后一条,OpenAI 兼容系开了 stream_usage 也在最后), +关掉的那一次只拿得到已经报了的部分。这一次照样计次数、计已报的 token,记成估不出,不悄悄少算。 + +模型名按发起调用时的参数取(invocation_params 的 model / model_name,就是接入里配的模型 id,价格目录按它查),取不到再 +看回复里的 model_name。 +""" +from __future__ import annotations + +import threading +from collections.abc import AsyncIterator, Iterator +from contextlib import contextmanager +from contextvars import ContextVar +from typing import Any +from uuid import UUID + +from langchain_core.callbacks import BaseCallbackHandler +from langchain_core.tracers.context import register_configure_hook + +#: 取不到模型名的调用记在这个名下 +UNKNOWN_MODEL = "unknown" + + +class Meter(BaseCallbackHandler): + """一次计量:按模型累计用量。线程安全(同步模型的回调可能在线程里来)。""" + + #: 同步模型在线程里调用时也要回调 + run_inline = True + + def __init__(self) -> None: + super().__init__() + self._lock = threading.Lock() + self._pending: dict[UUID, str] = {} + self._models: dict[str, dict[str, int]] = {} + + # ------------------------------------------------------------------ 回调 + + def _started(self, run_id: UUID, serialized: dict[str, Any] | None, kwargs: dict[str, Any]) -> None: + params = kwargs.get("invocation_params") or {} + model = params.get("model") or params.get("model_name") or params.get("model_id") + if not model and isinstance(serialized, dict): + model = (serialized.get("kwargs") or {}).get("model") or (serialized.get("kwargs") or {}).get("model_name") + with self._lock: + self._pending[run_id] = str(model) if model else "" + + def on_chat_model_start(self, serialized: dict[str, Any], messages: Any, *, run_id: UUID, + **kwargs: Any) -> None: + self._started(run_id, serialized, kwargs) + + def on_llm_start(self, serialized: dict[str, Any], prompts: Any, *, run_id: UUID, **kwargs: Any) -> None: + self._started(run_id, serialized, kwargs) + + def on_llm_end(self, response: Any, *, run_id: UUID, **kwargs: Any) -> None: + message = None + try: + message = response.generations[0][0].message + except (AttributeError, IndexError, TypeError): + pass + usage = getattr(message, "usage_metadata", None) or None + meta = getattr(message, "response_metadata", None) or {} + with self._lock: + model = self._pending.pop(run_id, "") or str(meta.get("model_name") or meta.get("model") or "") \ + or UNKNOWN_MODEL + row = self._models.setdefault(model, {"input_tokens": 0, "output_tokens": 0, "calls": 0, "unreported": 0}) + row["calls"] += 1 + if usage: + row["input_tokens"] += int(usage.get("input_tokens") or 0) + row["output_tokens"] += int(usage.get("output_tokens") or 0) + else: + row["unreported"] += 1 + + def on_llm_error(self, error: BaseException, *, run_id: UUID, **kwargs: Any) -> None: + """流读到一半停了(生成过程读到 done / look / ask 就关流,或者连接断了):LangChain 照样交来已经收到的那几段。 + 有收到东西的照计一次、记已报的 token,算用量没报全;一段都没收到的(请求刚发出去就被拒了)不计。""" + from langchain_core.outputs import ChatGenerationChunk + + response = kwargs.get("response") + partial = None + try: + first = response.generations[0][0] if response is not None and response.generations[0] else None + partial = first.message if isinstance(first, ChatGenerationChunk) else None + except (AttributeError, IndexError, TypeError): + partial = None + with self._lock: + model = self._pending.pop(run_id, "") + if partial is None: + return + meta = getattr(partial, "response_metadata", None) or {} + model = model or str(meta.get("model_name") or meta.get("model") or "") or UNKNOWN_MODEL + row = self._models.setdefault(model, {"input_tokens": 0, "output_tokens": 0, "calls": 0, "unreported": 0}) + row["calls"] += 1 + row["unreported"] += 1 + reported = getattr(partial, "usage_metadata", None) or {} + row["input_tokens"] += int(reported.get("input_tokens") or 0) + row["output_tokens"] += int(reported.get("output_tokens") or 0) + + # ------------------------------------------------------------------ 合并、汇总 + + def merge(self, other: Meter) -> None: + """把另一次计量并进来(嵌套的计量出块时并进外层)。""" + with other._lock: + rows = {k: dict(v) for k, v in other._models.items()} + with self._lock: + for model, row in rows.items(): + mine = self._models.setdefault(model, dict.fromkeys(row, 0)) + for key, value in row.items(): + mine[key] = mine.get(key, 0) + value + + def summary(self) -> dict[str, Any]: + """{by_model: {模型: {input_tokens, output_tokens, calls, cost_usd, unpriced_calls}}, input_tokens, output_tokens, + calls, cost_usd, unpriced_calls}。cost_usd 只是估得出的那部分。""" + from app.providers.catalog import estimate_cost, find_pricing + + with self._lock: + rows = {k: dict(v) for k, v in self._models.items()} + by_model: dict[str, dict[str, Any]] = {} + for model, row in sorted(rows.items()): + pricing = find_pricing(model) + free = pricing is not None and not any(pricing.values()) + if pricing is None: + unpriced = row["calls"] + else: + unpriced = 0 if free else row["unreported"] + by_model[model] = {"input_tokens": row["input_tokens"], "output_tokens": row["output_tokens"], + "calls": row["calls"], + "cost_usd": round(estimate_cost(model, row["input_tokens"], row["output_tokens"]), 6), + "unpriced_calls": unpriced} + total = {key: sum(r[key] for r in by_model.values()) + for key in ("input_tokens", "output_tokens", "calls", "unpriced_calls")} + return {"by_model": by_model, **total, "cost_usd": round(sum(r["cost_usd"] for r in by_model.values()), 6)} + + +#: 计量挂在这个上下文变量上(inheritable:模型调用时 LangChain 按它把 Meter 挂成回调,子任务拷走上下文时一并带走) +_CURRENT: ContextVar[Meter | None] = ContextVar("agentlab_usage_meter", default=None) +register_configure_hook(_CURRENT, inheritable=True) + + +def meter() -> Meter: + """新的一次计量。用 metering(一段代码)或 metered(一个生成过程)把它挂上。""" + return Meter() + + +def current() -> Meter | None: + return _CURRENT.get() + + +@contextmanager +def metering(m: Meter) -> Iterator[Meter]: + """这一段代码里的模型调用计进 m。只给不跨 yield 的一段用(复核那一次调用);生成过程用 metered。 + + 嵌套的:出块时并进外层的那一次,外层照样看得到。复原用 set 而不是按 token reset:在别的任务里收尾也不报错。""" + outer = _CURRENT.get() + _CURRENT.set(m) + try: + yield m + finally: + _CURRENT.set(outer) + if outer is not None and outer is not m: + outer.merge(m) + + +async def metered(source: AsyncIterator[Any], m: Meter) -> AsyncIterator[Any]: + """迭代一个生成过程,它的模型调用计进 m:每一步 __anext__ 期间挂上,交出操作之前复原(见模块开头)。 + + 开始时外层已经有一次计量的(评测把一整题的助手用量记成一份):结束时把 m 并进去,同 metering。""" + enclosing = _CURRENT.get() + iterator = source.__aiter__() + try: + while True: + outer = _CURRENT.get() + _CURRENT.set(m) + try: + item = await iterator.__anext__() + except StopAsyncIteration: + return + finally: + _CURRENT.set(outer) + yield item + finally: + if enclosing is not None and enclosing is not m: + enclosing.merge(m) + + +def empty() -> dict[str, Any]: + """一次调用都没有时的汇总(同 Meter().summary())。""" + return Meter().summary() + + +__all__ = ["Meter", "UNKNOWN_MODEL", "current", "empty", "meter", "metered", "metering"] diff --git a/backend/app/report_export/build.py b/backend/app/report_export/build.py index df2ccb88..1832abba 100644 --- a/backend/app/report_export/build.py +++ b/backend/app/report_export/build.py @@ -320,7 +320,8 @@ def build_report( ) -> dict[str, Any]: """页面里的数据(report-data)。 - - run:运行记录的几样(id、workflow_id、workflow_name、run_class、status、started_at、finished_at) + - run:运行记录的几样(id、workflow_id、workflow_name、run_class、status、end_reason、started_at、finished_at); + end_reason 分开同是 cancelled 的被驳回和手动停止(05-已知遗留 (a)) - report_node / doc_artifact / doc_hash_ok:选中的报告节点、它的文档工件、取回时哈希核对的结果 - doc:报告文档(正文的服务端渲染另走 render_body,这里不给:数据块只装点开数字要用的东西) - seal:{_sealed 的 seal 字段里的几样}(sealed、ok、manifest_hash、manifest_seq) @@ -334,6 +335,7 @@ def build_report( "meta": { "run_id": run.get("id"), "workflow_id": run.get("workflow_id"), "workflow_name": display_name(run), "run_class": run.get("run_class"), "status": run.get("status"), + "end_reason": run.get("end_reason"), "started_at": _iso(run.get("started_at")), "finished_at": _iso(run.get("finished_at")), "report": {"node_id": report_node, "label": (nodes.get(report_node) or {}).get("label") or report_node}, "doc_artifact": doc_artifact, "doc_hash_ok": doc_hash_ok, diff --git a/backend/app/report_export/runtime.js b/backend/app/report_export/runtime.js index a66b748c..e0f7eeef 100644 --- a/backend/app/report_export/runtime.js +++ b/backend/app/report_export/runtime.js @@ -25,7 +25,7 @@ metaRun: '运行 ID', metaClass: '运行类别', metaStatus: '运行状态', metaReport: '报告节点', metaStarted: '开始', metaFinished: '结束', metaExported: '导出时间', classLabel: { formal: '正式运行', exploratory: '探索运行' }, - statusLabel: { succeeded: '已完成', failed: '已失败', cancelled: '已取消', running: '进行中', waiting: '等待审批' }, + statusLabel: { succeeded: '已完成', failed: '已失败', cancelled: '已取消', rejected: '被驳回', running: '进行中', waiting: '等待审批' }, docHashBad: '这份报告文档与哈希不一致(疑似被修改),正文仅供参考,其中的口径和出处未经核对。', sealNone: '这次运行导出时尚未封存:下列出处来自导出那一刻的系统记录,未经封存核对。', sealOk: (hash) => `运行已封存,清单哈希 ${hash}。要向系统外的人证明文件未被修改,请导出「可验证报告」(记录页 · 凭证详情),拿封存哈希去系统对照。`, @@ -147,7 +147,8 @@ $('meta').replaceChildren(...[...[ [T.metaRun, h('code', {}, String(meta.run_id || ''))], [T.metaClass, label(T.classLabel, meta.run_class)], - [T.metaStatus, label(T.statusLabel, meta.status)], + // 被驳回(cancelled + end_reason=rejected)照运行记录写「被驳回」,不写成手动停止的「已取消」 + [T.metaStatus, label(T.statusLabel, meta.status === 'cancelled' && meta.end_reason === 'rejected' ? 'rejected' : meta.status)], [T.metaReport, String(meta.report?.label || meta.report?.node_id || NONE)], [T.metaStarted, when(meta.started_at)], [T.metaFinished, when(meta.finished_at)], diff --git a/backend/app/scheduler/loop.py b/backend/app/scheduler/loop.py index 243f5f97..ee6373c2 100644 --- a/backend/app/scheduler/loop.py +++ b/backend/app/scheduler/loop.py @@ -130,7 +130,9 @@ async def process_due(now: datetime | None = None) -> None: for name, step in (("待办的时限处理", tasks.tick), ("写入结果的查询", actions.tick), ("没有提交的上传文件清理", files_store.cleanup_tick)): try: - await step(now) + done = await step(now) + if isinstance(done, list) and done: + logger.info("%s:这一拍处理了 %d 条", name, len(done)) except Exception: # noqa: BLE001 logger.exception("%s这一拍出错,下一拍照常进行", name) diff --git a/backend/app/scheduler/service.py b/backend/app/scheduler/service.py index 14c25443..87d0ff18 100644 --- a/backend/app/scheduler/service.py +++ b/backend/app/scheduler/service.py @@ -3,6 +3,11 @@ 调度循环(loop.py)到点调 fire_once,接口的「立即运行」也调它:重叠跳过、渲染输入、记触发、发起运行这一套 只有一份。发起本身走 engine/launch.py 的 launch_run,和 HTTP 发起是同一套规则——正式运行只能从当前发布版本 发起,这一条不会在定时任务这边被绕过。 + +**定时的目标**(docs/design/next-round/00-总方案.md X4:定时只有一套):一个定时任务指向一个目标——工作流、评测题集 +(第 3 波)或数据源维护(第 3 波)。认领、补跑、重叠跳过、触发记录、通知都只有这一套,fire_once 按目标种类分派。 +老表的 workflow_id 是非空约束,只加不改的迁移放宽不了:目标不是工作流时它存空串。这条规矩只在 target_of / +target_columns 这一处,别处读写目标都经它们,不直接碰这两列。 """ from __future__ import annotations @@ -64,6 +69,33 @@ INTERRUPTED_REASON = "服务在发起运行前停止,这次触发没有发起 GENERIC_REASON = "发起运行时出错,详细信息见服务日志" +#: 定时的目标种类(Schedule.target_kind),顺序即以后界面上的顺序。这一波 fire_once 只会发起 workflow +TARGET_KINDS = ("workflow", "eval_suite", "source_maintenance") +TARGET_LABEL = {"workflow": "工作流", "eval_suite": "评测题集", "source_maintenance": "数据源维护"} +#: 目标种类这个版本还发起不了(库里有别的版本建的任务、手改过库):照实记「未能发起」,不当成工作流去跑 +TARGET_UNSUPPORTED = "这个定时任务的目标是「{kind}」,当前版本还不能按时发起这类目标,这次触发没有发起" + + +def target_of(schedule: Any) -> tuple[str, str]: + """一个定时任务的目标:(种类, id)。加列之前建的任务 target_kind 缺省 workflow、target_id 为空,目标就是 workflow_id。 + + workflow 时 id 以 workflow_id 为准(两列由 target_columns 一起写,对得上;老任务只有这一列)。""" + kind = getattr(schedule, "target_kind", None) or "workflow" + if kind == "workflow": + return kind, schedule.workflow_id or getattr(schedule, "target_id", None) or "" + return kind, getattr(schedule, "target_id", None) or "" + + +def target_columns(kind: str, target_id: str) -> dict[str, str]: + """写目标的几列:{target_kind, target_id, workflow_id}。workflow 时 target_id 同 workflow_id;别的种类 workflow_id 存空串 + (老表的非空约束)。种类不认识、id 为空抛 ValueError。""" + if kind not in TARGET_KINDS: + raise ValueError(f"不认识的定时目标种类:{kind}") + if not target_id: + raise ValueError("定时任务的目标 id 不能为空") + return {"target_kind": kind, "target_id": target_id, "workflow_id": target_id if kind == "workflow" else ""} + + class TemplateError(ValueError): """输入模板写错了。str() 是给人看的中文说明,指出是哪个输入。""" @@ -282,46 +314,61 @@ async def fire_once(schedule_id: str, fire_at: datetime, kind: str, *, actor: st """处理一次触发,返回记下的触发记录。同一 (任务, fire_at, kind) 已经有记录(两拍撞在一起、立即运行连点两下) 时不再发起,返回已有的那一条;任务不存在返回 None。 - 上一次运行还没结束就记 skipped_overlap;输入渲染不了、launch_run 拒绝或出错记 failed_to_start(拒绝写原话, - 其他异常写通用说明、详细信息进日志);发起成功记 started 和运行 id。 + 按目标种类分派(target_of):这一波只有工作流(_fire_workflow),别的种类记 failed_to_start、写明原因,不发起。 """ async with launch_lock(): async with SessionLocal() as session: schedule = await session.get(Schedule, schedule_id) if schedule is None: return None - name, workflow_id, run_class = schedule.name, schedule.workflow_id, schedule.run_class + target_kind, target_id = target_of(schedule) + name, run_class = schedule.name, schedule.run_class tz, cron_expr, template = schedule.timezone, schedule.cron, schedule.input_template - try: - inputs = render_inputs(check_template(template), template_values(fire_at, tz, cron_expr)) - except (TemplateError, cron.CronError, cron.TimezoneError) as e: - # 存进库之后才坏掉的(手改过库、时区库升级删了时区):照实说是哪一项,不发起 - return await _record(schedule_id, fire_at, kind, "failed_to_start", f"无法生成这次运行的输入:{e}", {}) - busy = await _active_run(schedule_id) - if busy is not None: - return await _record(schedule_id, fire_at, kind, "skipped_overlap", overlap_reason(*busy), inputs) - fire = await _record(schedule_id, fire_at, kind, "failed_to_start", INTERRUPTED_REASON, inputs, fresh=True) - if fire is None: - return await _existing(schedule_id, fire_at, kind) - from app.engine.launch import LaunchRefused, launch_run + if target_kind != "workflow": + reason = TARGET_UNSUPPORTED.format(kind=TARGET_LABEL.get(target_kind, target_kind)) + return await _record(schedule_id, fire_at, kind, "failed_to_start", reason, {}) + return await _fire_workflow(schedule_id, fire_at, kind, actor=actor, name=name, workflow_id=target_id, + run_class=run_class, tz=tz, cron_expr=cron_expr, template=template) - _LAUNCHING.add(fire.id) + +async def _fire_workflow(schedule_id: str, fire_at: datetime, kind: str, *, actor: str | None, name: str, + workflow_id: str, run_class: str, tz: str, cron_expr: str, template: Any, + ) -> ScheduleFire | None: + """目标是工作流的一次触发(持着 launch_lock)。 + + 上一次运行还没结束就记 skipped_overlap;输入渲染不了、launch_run 拒绝或出错记 failed_to_start(拒绝写原话, + 其他异常写通用说明、详细信息进日志);发起成功记 started 和运行 id。 + """ + try: + inputs = render_inputs(check_template(template), template_values(fire_at, tz, cron_expr)) + except (TemplateError, cron.CronError, cron.TimezoneError) as e: + # 存进库之后才坏掉的(手改过库、时区库升级删了时区):照实说是哪一项,不发起 + return await _record(schedule_id, fire_at, kind, "failed_to_start", f"无法生成这次运行的输入:{e}", {}) + busy = await _active_run(schedule_id) + if busy is not None: + return await _record(schedule_id, fire_at, kind, "skipped_overlap", overlap_reason(*busy), inputs) + fire = await _record(schedule_id, fire_at, kind, "failed_to_start", INTERRUPTED_REASON, inputs, fresh=True) + if fire is None: + return await _existing(schedule_id, fire_at, kind) + from app.engine.launch import LaunchRefused, launch_run + + _LAUNCHING.add(fire.id) + try: try: - try: - async with SessionLocal() as session: - run = await launch_run(session, workflow_id=workflow_id, run_class=run_class, - input_payload=inputs, actor=actor or actor_label(name), - schedule_id=schedule_id, origin_kind="schedule", - origin={"schedule_id": schedule_id, "schedule": name, - "fire_id": fire.id, "fire_kind": kind}) - except LaunchRefused as e: - return await _finish(fire.id, status="failed_to_start", reason=e.message) - except Exception: # noqa: BLE001 - 发起失败不能让调度循环停下;详细信息进日志,记录上写通用说明 - logger.exception("定时任务「%s」(%s)发起运行失败", name, schedule_id) - return await _finish(fire.id, status="failed_to_start", reason=GENERIC_REASON) - return await _finish(fire.id, status="started", reason=None, run_id=run.id) - finally: - _LAUNCHING.discard(fire.id) + async with SessionLocal() as session: + run = await launch_run(session, workflow_id=workflow_id, run_class=run_class, + input_payload=inputs, actor=actor or actor_label(name), + schedule_id=schedule_id, origin_kind="schedule", + origin={"schedule_id": schedule_id, "schedule": name, + "fire_id": fire.id, "fire_kind": kind}) + except LaunchRefused as e: + return await _finish(fire.id, status="failed_to_start", reason=e.message) + except Exception: # noqa: BLE001 - 发起失败不能让调度循环停下;详细信息进日志,记录上写通用说明 + logger.exception("定时任务「%s」(%s)发起运行失败", name, schedule_id) + return await _finish(fire.id, status="failed_to_start", reason=GENERIC_REASON) + return await _finish(fire.id, status="started", reason=None, run_id=run.id) + finally: + _LAUNCHING.discard(fire.id) async def record_skip(schedule_id: str, fire_at: datetime, reason: str) -> ScheduleFire | None: diff --git a/backend/app/scheduler/webhook.py b/backend/app/scheduler/webhook.py index 4a19e1c6..23ea36c6 100644 --- a/backend/app/scheduler/webhook.py +++ b/backend/app/scheduler/webhook.py @@ -5,7 +5,8 @@ 再导出,调用方和测试不用改;_transport、RETRY_DELAYS 仍是这个模块自己的(测试照旧替换它们),send 每次调用时读。 通知里只有任务名、状态、出具档位、缺口条数、运行链接、封存哈希(界面上叫清单哈希)前 12 位和起止时间,失败时 -另写失败节点的名字和下一步(不许诺能继续运行);运行成功结束后和同一任务上一次成功的运行的对比(auto_compare)只写 +另写失败节点的名字和下一步(不许诺能继续运行);被人驳回的那一期按「运行结束」推「未出具(被驳回)」、写驳回的那一步, +手动停止的不推;运行成功结束后和同一任务上一次成功的运行的对比(auto_compare)只写 各类指标的个数和对比页链接。不放报告里的具体数字,也不放失败时的原始报错——报错里可能带着查询到的数据,Webhook 推去的群不一定有权看。 """ @@ -71,6 +72,11 @@ NEXT_AS_USUAL = "下一次触发会照常发起" #: 从没开始执行、任务又不在排期上(停用了):原因刚写过,记录页上也没有更多可看的,下一步是重新发起。「立即运行」 #: 是定时任务页上那个按钮的名字,停用的任务也能点 RERUN_ACTION = "如需重新发起,请在 AgentLab 的「定时任务」中点击「立即运行」" +#: 被人驳回的那一期(cancelled + end_reason=rejected,next-round X9):是人的决定,不是故障。按「运行结束」推(订阅 +#: 「运行失败」的人盯的是故障),标题、状态照实写被驳回;驳回的备注不写(可能带着数),驳回的是哪一步写节点名 +REJECTED_HEADLINE = "未出具(被驳回)" +REJECTED_STATUS = "被驳回" +REJECTED_ACTION = "驳回的说明请在 AgentLab 的「记录」中查看" # -------------------------------------------------------------------------- @@ -110,12 +116,18 @@ class Notice: #: 挂起的原因是写外部系统的结果未知(等核实,核实后自动接着跑),不是服务重启。只在 suspended 时有意义 action_pending: bool = False error_node: str | None = None + #: 运行结束在 cancelled 的原因(Run.end_reason):rejected 时按「未出具(被驳回)」说,error_node 是驳回的那一步 + end_reason: str | None = None next_fire: bool = False compare: dict[str, Any] | None = None compare_url: str | None = None fmt: str | None = None signed: bool = False + @property + def rejected(self) -> bool: + return self.status == "cancelled" and self.end_reason == "rejected" + @property def never_started(self) -> bool: """失败的运行从没开始执行(没有开始时间):服务重启时还在排队、被改成失败的就是这种,没有断点可续。""" @@ -142,6 +154,8 @@ def message(notice: Notice) -> tuple[str, list[tuple[str, str]], str | None]: headline = "未能发起运行" elif notice.suspended: headline = "运行已挂起" + elif notice.rejected: + headline = REJECTED_HEADLINE else: headline = _HEADLINE.get(notice.event, notice.event) title = f"定时任务「{notice.schedule_name}」{headline}" @@ -163,6 +177,10 @@ def message(notice: Notice) -> tuple[str, list[tuple[str, str]], str | None]: lines += [("原因", ACTION_PENDING_REASON), ("处理", ACTION_PENDING_ACTION)] elif notice.suspended: lines += [("原因", SUSPENDED_REASON), ("处理", SUSPENDED_ACTION)] + elif notice.rejected: + if notice.error_node: + lines.append(("驳回节点", f"「{notice.error_node}」")) + lines.append(("处理", f"{REJECTED_ACTION};{NEXT_AS_USUAL}" if notice.next_fire else REJECTED_ACTION)) elif notice.status == "failed": # 不写原始报错(可能带着查询到的数据),写失败节点的名字和下一步 if notice.error_node: @@ -219,6 +237,8 @@ def _status_label(notice: Notice) -> str: if notice.pending_approvals: return "等待审批" return "已挂起,等待核实" if notice.action_pending else "已挂起,可继续运行" + if notice.rejected: + return REJECTED_STATUS return _STATUS_LABEL.get(notice.status or "", notice.status or "") @@ -239,7 +259,9 @@ def _links(notice: Notice) -> list[tuple[str, str]]: def _message(notice: Notice) -> core.Message: title, lines, _ = message(notice) - return core.Message(title=title, lines=lines, links=_links(notice), ok=notice.event in ("finished", "test")) + # 被驳回的那一期按 finished 推,但没出具:不画成成功的颜色 + return core.Message(title=title, lines=lines, links=_links(notice), + ok=notice.event in ("finished", "test") and not notice.rejected) def _text(notice: Notice) -> str: @@ -332,10 +354,17 @@ async def _record(schedule_id: str, fire_id: str | None, run_id: str | None, eve #: 运行到了这些状态才可能要推送 _WATCHED = ("succeeded", "failed", "cancelled", "interrupted") -#: 结束的运行 → 事件。cancelled 是有人手动停的,不推。interrupted 分停在审批和挂起两种,见 _state_of +#: 结束的运行 → 事件。cancelled 是有人手动停的,不推;被驳回的(end_reason=rejected)按 finished 推,见 _event_of。 +#: interrupted 分停在审批和挂起两种,见 _state_of _EVENT_OF = {"succeeded": "finished", "failed": "failed", "cancelled": None} +def _event_of(run: Run) -> str | None: + if run.status == "cancelled" and run.end_reason == "rejected": + return "finished" + return _EVENT_OF[run.status] + + @dataclass class _Job: fmt: str @@ -395,9 +424,10 @@ async def notify_due(now: datetime | None = None) -> int: .order_by(ScheduleFire.created_at) )).all() if rows: - from app.api.settings import notify_settings + # 链接里的地址:全站同一个对外访问地址(环境变量优先,其次设置,core/public_url.py) + from app.core import public_url - base_url = (await notify_settings(session))["public_base_url"] + base_url = await public_url.refresh(session) for fire, run in rows: seen = await _state_of(session, fire, run, now) if seen.state is None or seen.state == fire.notified_status: @@ -469,7 +499,7 @@ async def _state_of(session: Any, fire: ScheduleFire, run: Run | None, now: date since = run.finished_at or run.updated_at or fire.created_at if not sealed and (now - service.as_utc(since)).total_seconds() < SEAL_WAIT_S: return _Seen(None) - return _Seen(run.status, _EVENT_OF[run.status], sealed=sealed) + return _Seen(run.status, _event_of(run), sealed=sealed) if run.status != "interrupted": return _Seen(None) counts = dict((await session.execute( @@ -516,7 +546,9 @@ async def _notice(session: Any, schedule: Schedule, fire: ScheduleFire, run: Run pending_approvals=pending or None, suspended=run is not None and run.status == "interrupted" and not pending, action_pending=seen.action_pending, - error_node=_failed_node(run) if run is not None and run.status == "failed" else None, + error_node=_failed_node(run) if run is not None and (run.status == "failed" or run.end_reason == "rejected") + else None, + end_reason=run.end_reason if run is not None else None, next_fire=bool(schedule.enabled and schedule.next_run_at is not None), ) diff --git a/backend/app/tools/datasource.py b/backend/app/tools/datasource.py index 53e11ecf..255be841 100644 --- a/backend/app/tools/datasource.py +++ b/backend/app/tools/datasource.py @@ -277,7 +277,7 @@ async def build_datasource_tools( except table_versions.SnapshotMissing as e: tools.extend(_unavailable_tool(row, prefix, str(e)) for prefix in prefixes) continue - entries = await _catalog_of(row) + entries = await _catalog_of(row, ctx.catalog_view) for prefix in prefixes: if prefix == QUERY_PREFIX: tools.append(_make_query_tool(source, ctx, fixed=fixed, catalog=entries, force_readonly=force_readonly, @@ -287,20 +287,17 @@ async def build_datasource_tools( return tools -async def _catalog_of(row: DataSource) -> dict[str, data_catalog.CatalogEntry]: +async def _catalog_of(row: Any, view: Any = None) -> dict[str, data_catalog.CatalogEntry]: """这个源的数据目录,建工具时读一次:db_schema 给模型看的和查询时冻结进表结构快照的是同一版。 - 读不到就当没有目录,工具照常可用——目录是附加说明,不能挡住查询。用单独的会话读:出错要回滚, - 而回滚会让调用方会话里已经加载的对象(row)全部过期,之后再读属性就是异步环境里的懒加载。 + 读哪一份由目录视图定(view,data/catalog_view.py:当前目录、目录快照、关;只用已核实的项),不给就是当前目录。 + 读不到就当没有目录,工具照常可用——目录是附加说明,不能挡住查询;视图指定的目录快照不在了照实报错(不悄悄换成当前 + 目录)。用单独的会话读:出错要回滚,而回滚会让调用方会话里已经加载的对象(row)全部过期,之后再读属性就是异步环境里 + 的懒加载。 """ - from app.db.base import SessionLocal + from app.data import catalog_view - try: - async with SessionLocal() as own: - return await data_catalog.read_catalog(own, row.id) - except Exception: # noqa: BLE001 - logger.exception("读取数据源 %s 的数据目录失败,本次不附目录", row.name) - return {} + return await catalog_view.read(catalog_view.CatalogView.parse(view), row) # -------------------------------------------------------------------------- @@ -753,6 +750,12 @@ async def _store_schema(source: Any, ctx: ToolContext, *, if frozen := data_catalog.frozen_catalog(catalog or {}, tables): content[CATALOG_SNAPSHOT_KEY] = frozen meta["catalog_tables"] = len(frozen) + # 这次运行读的不是缺省的当前目录(评测的对照:关、目录快照、只用已核实的项):记在引用的 meta 里,证据面板写得出 + # 「本次运行目录:关」。不进内容哈希,缺省视图不记,快照和以前一字不差 + from app.data.catalog_view import CatalogView, describe + + if (view := describe(CatalogView.parse(getattr(ctx, "catalog_view", None)))) is not None: + meta["catalog_view"] = view try: return await put_json(content, kind=SCHEMA_SNAPSHOT, run_id=ctx.run_id or "", node_id=ctx.node_id or "", meta=meta) diff --git a/backend/app/tools/registry.py b/backend/app/tools/registry.py index c1215267..c80f0ed4 100644 --- a/backend/app/tools/registry.py +++ b/backend/app/tools/registry.py @@ -49,6 +49,9 @@ class ToolContext: #: 个人应用的刷新、个人蓝图的试运行)。记忆、知识库的内容不经遮罩:kb_search 的 collection 参数(调用工具节点写的、 #: 参数名纠错改出来的、图级 defaults 补上的、模型挑的)一律拒,只能检索 collection 这一个 fixed_scope: bool = False + #: 读哪一份数据目录(data/catalog_view.CatalogView):运行里由节点从 Run.options 取来(catalog_view.of_run),db_schema + #: 给模型看的、查询冻结进表结构快照的、查询时的 SQL 检查都按它。None 是当前目录(缺省,行为同以前) + catalog_view: Any = None extra: dict[str, Any] = field(default_factory=dict) diff --git a/backend/app/web.py b/backend/app/web.py index d2c36605..30602b95 100644 --- a/backend/app/web.py +++ b/backend/app/web.py @@ -9,7 +9,9 @@ - /api 下不存在的路径照旧是 JSON 404。排除 /api 写在路由的路径规则里,不在处理函数里判断—— 否则 POST /api/不存在 会因为「路径对上了、方法不对」变成 405; - 只注册 GET / HEAD 的 HTTP 路由,WebSocket 不经过这里; -- index.html 不缓存,发版后刷新即生效;assets/ 下的文件名带内容哈希,长缓存。 +- index.html 不缓存,发版后刷新即生效;assets/ 下的文件名带内容哈希,长缓存; +- index.html 另带严格 script-src 的上报策略(Content-Security-Policy-Report-Only):启动时按构建产物里的内联脚本算 + sha256(web_security.script_policy),只上报、不拦。其余安全头由 web_security 的中间件统一补。 """ from __future__ import annotations @@ -20,6 +22,8 @@ from fastapi import FastAPI, HTTPException from fastapi.responses import FileResponse from starlette.convertors import Convertor, register_url_convertor +from app.web_security import REPORT_ONLY_HEADER, script_policy + _NO_CACHE = "no-cache" _IMMUTABLE = "public, max-age=31536000, immutable" @@ -72,7 +76,12 @@ def mount_web(app: FastAPI, dist: Path) -> None: "先构建前端(cd frontend && pnpm build),或者去掉这个设置" ) + # 哈希按启动这一刻的 index.html 算:换了构建产物要重启服务,哈希才跟着换(只上报不拦,对不上也只多几条上报) + page_headers = {"Cache-Control": _NO_CACHE, REPORT_ONLY_HEADER: script_policy(index.read_text(encoding="utf-8"))} + def respond(target: Path, *, immutable: bool = False) -> FileResponse: + if target in (index, index.resolve()): + return FileResponse(target, media_type=_media_type(target), headers=page_headers) return FileResponse( target, media_type=_media_type(target), diff --git a/backend/app/web_security.py b/backend/app/web_security.py new file mode 100644 index 00000000..a7ae8646 --- /dev/null +++ b/backend/app/web_security.py @@ -0,0 +1,90 @@ +"""安全响应头(04 细案 C1):页面不能被别的站点套进 iframe 诱导点击,浏览器不按内容猜类型,链接不把完整地址带给别的站点。 + +以前服务端一个安全头都不发:任何站点都能把 AgentLab 嵌进 iframe。同站的门户页面(SameSite=Lax 的会话 cookie 会随请求 +带上)和关登录的部署,都有被点击劫持的可能;也没有一条「允许谁嵌」的白名单可配。 + +纯 ASGI 中间件(同 auth/middleware.py,main.py 里加在最外层):在 http.response.start 上补头,WebSocket 不经过它。 +所有 HTTP 响应(页面、/api、登录检查自己回的 401 / 403、CORS 预检)都补: +- X-Content-Type-Options: nosniff; +- Referrer-Policy: strict-origin-when-cross-origin(分享页 /s 用 no-referrer:分享的 key 在地址的 # 后面,本来不随 + Referer 走,这里再收一道); +- Content-Security-Policy: frame-ancestors。缺省只有 'self';AGENTLAB_FRAME_ANCESTORS 列出允许嵌整站的同站门户;分享页 + /s 只认 AGENTLAB_EMBED_ORIGINS(分享链接第 3 波才加,路由在那时挂,头的规则先在这里定好); +- 只有 frame-ancestors 是 'self' 时另加 X-Frame-Options: SAMEORIGIN,给不认 CSP 的老浏览器。XFO 写不了白名单,有白名单 + 时只发 CSP:两个都发的话,只认 XFO 的老浏览器会把白名单里的门户也拦掉(认 CSP 的浏览器以 frame-ancestors 为准)。 +接口自己设过的同名头(文件下载的 nosniff)不覆盖、不重复。 + +严格的 script-src 先只上报、不拦(Content-Security-Policy-Report-Only):页面里有一段首帧前定主题的内联脚本,强制执行 +的话写错一个哈希页面就会先闪一帧深色,构建会不会产出别的内联内容也没核实。所以 mount_web(app/web.py)启动时给构建产物 +index.html 里的内联脚本算 sha256(script_policy),只在页面这一个响应上发;观察一个版本没有误报,再改成强制执行。没配 +上报地址:违规只在浏览器的控制台里看得到。接口返回的 HTML(导出的报告、可验证报告)文件里自带 meta CSP,不发它,也不动。 + +HSTS 交给终结 https 的反向代理:这里不知道对外是不是 https、要不要连子域名一起管,发错了浏览器会按 max-age 记很久, +撤不回来。 +""" +from __future__ import annotations + +import base64 +import hashlib +import re +from typing import Any + +#: 分享页的路径(/s#<分享 key>):只认 /s 这一段,/settings、/studio 不算 +SHARE_PATH = "/s" +_SELF = "'self'" +_DEFAULT_REFERRER = b"strict-origin-when-cross-origin" +_SHARE_REFERRER = b"no-referrer" +#: 页面里不带 src 的 ', report.text, re.S) + assert m and json.loads(m.group(1))["meta"]["end_reason"] == "rejected" + + async def test_a_broken_seal_is_refused(client, monkeypatch): row = await sealed_run(monkeypatch) async with SessionLocal() as session: diff --git a/backend/tests/test_bundle_build.py b/backend/tests/test_bundle_build.py index 7dd58ded..4172ade7 100644 --- a/backend/tests/test_bundle_build.py +++ b/backend/tests/test_bundle_build.py @@ -340,6 +340,11 @@ def test_meta_names_the_seal_and_the_run(): assert meta["manifest_seq"] == 15 and meta["manifest_hash"] == "ab" * 32 and meta["legacy"] is False assert meta["started_at"] == RUN["started_at"] and meta["finished_at"] == RUN["finished_at"] assert meta["exported_at"] == "2026-10-05T08:00:00+00:00" + # 同是 cancelled 的被驳回和手动停止靠 end_reason 分开;没有的(成功、失败、老运行)是 None + assert meta["end_reason"] is None + rejected = build_bundle({**RUN, "status": "cancelled", "end_reason": "rejected"}, sample_events(), manifest_seq=15, + manifest_hash="ab" * 32, graph=GRAPH, read=STORE.get, exported_at="x") + assert (rejected["meta"]["status"], rejected["meta"]["end_reason"]) == ("cancelled", "rejected") # 出具档位取封存范围内最后一条出具判定事件,带上序号,浏览器拿已核对的事件对照 assert meta["tier"] == {"value": "formal", "seq": 13} assert bundle["nodes"]["write"] == {"label": "写周报", "type": "report"} @@ -463,7 +468,7 @@ def test_the_sample_pages_for_the_browser_check_are_sealed_for_real(): rendered = pages() got = {name: json.loads(data_block(page)) for name, page in rendered.items()} - assert set(got) == {"base", "two-reports", "metric-only", "legacy", "failed", "continued", "degraded", + assert set(got) == {"base", "two-reports", "metric-only", "legacy", "failed", "rejected", "continued", "degraded", "missing-doc", "missing-card", "nan", "stray", "masked-card", "masked-old-doc", "masked-source", "formats", "formats-unit", "prose"} for name, bundle in got.items(): @@ -504,6 +509,10 @@ def test_the_sample_pages_for_the_browser_check_are_sealed_for_real(): assert [s["text"] for s in segments(only)] and not [s for s in segments(only) if s.get("cite", {}).get("kind") == "cell"] failed = got["failed"] assert failed["meta"]["status"] == "failed" and failed["meta"]["tier"] is None + rejected = got["rejected"] + assert (rejected["meta"]["status"], rejected["meta"]["end_reason"]) == ("cancelled", "rejected") + assert max(rejected["events"], key=lambda e: e["seq"])["type"] == "run.cancelled" + assert re.search(r'id="seal-failed"[^>]*\shidden', rendered["rejected"]), "被驳回不是失败:不提重新封存" assert max(failed["events"], key=lambda e: e["seq"])["type"] == "run.failed" # 先失败、经继续运行跑到成功的早期运行:范围里有那条运行失败,之后运行又继续了,最后一条不是结束事件 continued = got["continued"] diff --git a/backend/tests/test_bundle_rejected.py b/backend/tests/test_bundle_rejected.py new file mode 100644 index 00000000..6a3e41d1 --- /dev/null +++ b/backend/tests/test_bundle_rejected.py @@ -0,0 +1,79 @@ +"""可验证报告里被驳回的运行写「被驳回」,不写成手动停止的「已取消」(05-已知遗留 (a)、总方案 X9)。 + +人的驳回让运行结束在 cancelled + end_reason=rejected,手动停止也是 cancelled(end_reason=stopped)。封存里二者是同一种 +结束事件(运行取消),分开它们的原因在那条事件的内容里,内容不随文件附带。所以: + +- 运行结果照运行记录(元信息的 end_reason)写「被驳回」,核对的仍是「最后一条结束类事件是运行取消」这一半,旁边写明 + 被驳回取自运行记录,不把它说成已核对出来的; +- 手动停止、老运行(没有 end_reason)照旧「已取消」; +- 文件里写着被驳回、最后一条事件却是运行结束的,照旧不通过,点名文件里写的「被驳回」。 + +用 node 直接载入 verifier.js(同 test_bundle_shown.py),只看运行信息(⑥)这一块。数据全是虚构的。 +""" +from __future__ import annotations + +import copy +import json +import shutil +import subprocess +from pathlib import Path +from typing import Any + +import pytest + +import app.bundle +from tests.bundle_samples import RUN, Sample, WEEKLY, build, bundles + +NODE = shutil.which("node") +VERIFIER = Path(app.bundle.__file__).parent / "verifier.js" + +pytestmark = pytest.mark.skipif(NODE is None, reason="没有 node,无法执行页面上的核对脚本") + +#: 载入 verifier.js(node 里没有 document,它把核对用的纯函数交给 module.exports),交回运行信息每一项 +HARNESS = r""" +const fs = require('fs') +const { createHash } = require('crypto') +const input = JSON.parse(fs.readFileSync(0, 'utf8')) +const mod = { exports: {} } +new Function('module', fs.readFileSync(process.argv[1], 'utf8'))(mod) +const digest = async (bytes) => createHash('sha256').update(bytes).digest('hex') +mod.exports.verify(input.data, digest, input.data.meta.manifest_hash).then((out) => { + process.stdout.write(JSON.stringify(out.meta.map((m) => ({ field: m.field, shown: m.shown, claim: m.claim, + result: m.result, reason: m.reason || '', mark: m.mark ?? null })))) +}, (e) => { console.error((e && e.stack) || String(e)); process.exit(1) }) +""" + + +def status_of(bundle: dict[str, Any]) -> dict[str, Any]: + done = subprocess.run([NODE, "-e", HARNESS, str(VERIFIER)], input=json.dumps({"data": bundle}), + capture_output=True, text=True, timeout=120) + assert done.returncode == 0, done.stderr[-2000:] + return next(m for m in json.loads(done.stdout) if m["field"] == "status") + + +def ended(end_reason: str | None) -> dict[str, Any]: + """同一串事件(最后一条是运行取消),运行记录分别写被驳回、手动停止、没写(老运行)。""" + s = Sample() + events, bound = s.events({"write": s.doc(WEEKLY, "write")}, ending="rejected") + return build(s, events, bound, run={**RUN, "status": "cancelled", "end_reason": end_reason}) + + +def test_a_rejected_run_reads_rejected_and_says_which_half_is_checked(): + got = status_of(bundles()["rejected"]) + assert got["result"] == "pass" and got["shown"] == "被驳回" and got["claim"] == "被驳回", got + assert "运行取消" in got["reason"] and "取自运行记录" in got["reason"], got["reason"] + # 旁边那一句不说成全核对过了 + assert got["mark"] == "已核对运行取消,被驳回取自运行记录", got + + +@pytest.mark.parametrize("end_reason", ["stopped", None]) +def test_a_stopped_or_old_cancelled_run_still_reads_cancelled(end_reason): + got = status_of(ended(end_reason)) + assert (got["result"], got["shown"], got["claim"], got["mark"]) == ("pass", "已取消", "已取消", None), got + + +def test_a_file_claiming_rejected_over_a_finished_run_fails_and_names_the_claim(): + forged = copy.deepcopy(bundles()["base"]) + forged["meta"].update(status="cancelled", end_reason="rejected") + got = status_of(forged) + assert got["result"] == "fail" and got["shown"] == "已完成" and "被驳回" in got["reason"], got diff --git a/backend/tests/test_catalog_draft_facts.py b/backend/tests/test_catalog_draft_facts.py index 37f3705b..c65eff02 100644 --- a/backend/tests/test_catalog_draft_facts.py +++ b/backend/tests/test_catalog_draft_facts.py @@ -402,7 +402,7 @@ async def test_a_big_table_is_checked_on_its_first_rows(scenic): found = await catalog_profile.check_valid_filter(source, "visits", source.schema_cache["tables"]["visits"], "status = 1") assert found.sampled and found.rows == 100 - assert found.note.startswith("剖析核对:按前 100 行统计,") + assert found.note.startswith("剖析核对:按存储顺序取前 100 行统计,") async def test_unchecked_filters_are_kept_with_the_reason(scenic, tmp_path): diff --git a/backend/tests/test_catalog_patch.py b/backend/tests/test_catalog_patch.py index e9c1a49d..15fb7c1b 100644 --- a/backend/tests/test_catalog_patch.py +++ b/backend/tests/test_catalog_patch.py @@ -268,7 +268,8 @@ async def _stream(monkeypatch, model, source, base_graph=None): async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: async with client.stream("POST", "/api/copilot/generate-stream", json=body) as r: async for line in r.aiter_lines(): - if line.startswith("data: "): + # 末尾那条 usage 是这一轮助手的用量(answer_ops),不是生成的操作 + if line.startswith("data: ") and not line.startswith('data: {"op": "usage"'): events.append(json.loads(line[6:])) return events diff --git a/backend/tests/test_catalog_profile.py b/backend/tests/test_catalog_profile.py index fee64c5f..0a1ef009 100644 --- a/backend/tests/test_catalog_profile.py +++ b/backend/tests/test_catalog_profile.py @@ -23,7 +23,7 @@ from httpx import ASGITransport, AsyncClient from sqlalchemy import event from sqlalchemy.engine import Engine -from app.data import catalog_profile, catalog_profile_jobs +from app.data import catalog, catalog_profile, catalog_profile_jobs from app.data import engine as data_engine from app.data.catalog_profile import PROFILE_OPTION, ProfileSettings from app.db.base import SessionLocal @@ -240,7 +240,7 @@ async def test_profile_verifies_name_relations_and_proposes_codes_and_business_d assert rel["to_table"] == parent assert (rel["source"], rel["status"]) == ("profile", "verified") assert rel["coverage"] == 1.0 and rel["cardinality"] == "many_to_one" - assert "数据剖析" in rel["note"] and "抽样" in rel["note"] + assert "数据剖析" in rel["note"] and "子表取了" in rel["note"] and "抽样" not in rel["note"] # 子表一侧数过 COUNT 和 COUNT(DISTINCT)、确有重复:基数是用数据核实的(A6,一对多关联的检查据此给 error) assert rel["cardinality_checked"] is True # A7:说明里不写日期,时间看这一项的 updated_at(界面按本地时间显示) @@ -318,7 +318,7 @@ async def test_wrong_name_relation_stays_proposed_with_coverage_in_note(client, rel = _relation((await _notes(client, sid, "parking_records"))["notes"], "lot_id") assert rel["status"] == "proposed" and rel["source"] == "profile" assert rel["coverage"] == 0.15 - assert "抽样覆盖率 15%" in rel["note"] and "可能不是这条关系" in rel["note"] and "20 个" in rel["note"] + assert "覆盖率 15%" in rel["note"] and "可能不是这条关系" in rel["note"] and "20 个" in rel["note"] finding = _table(body, "parking_records")["findings"][0] assert finding["status"] == "proposed" and finding["coverage"] == 0.15 # 两个日期类列:不提议业务日期,取值范围只在剖析结果里 @@ -327,6 +327,39 @@ async def test_wrong_name_relation_stays_proposed_with_coverage_in_note(client, assert "business_date" not in (await _notes(client, sid, "parking_records"))["notes"] +async def test_text_keys_that_cannot_be_integers_count_as_unmatched(client, make_source, tmp_path): + """模型起草的关系:orders.customer_ref(文字)指向 customers.id(整数主键),可取值大多是散客代号。不是整数写法的 + 文字永远对不上整数主键,算作没对上、留在分母里——不能从分母里拿掉,把 1 个对上的键说成覆盖率 100% 升为有确证。""" + path = str(tmp_path / "walkin.db") + db = sqlite3.connect(path) + try: + db.executescript("CREATE TABLE customers (id INTEGER PRIMARY KEY, name TEXT NOT NULL);" + "CREATE TABLE orders (id INTEGER PRIMARY KEY, customer_ref TEXT, amount INTEGER);") + db.executemany("INSERT INTO customers VALUES (?, ?)", [(i, f"会员{i}") for i in (1, 2, 3)]) + refs = ["WALKIN"] * 90 + ["1"] * 10 + [f"GUEST{i}" for i in range(30)] + db.executemany("INSERT INTO orders VALUES (?, ?, ?)", [(i, ref, 10) for i, ref in enumerate(refs, 1)]) + db.commit() + finally: + db.close() + sid = await make_source(path, options=_on()) + rel_id = catalog.relation_id("orders", ["customer_ref"], "customers", ["id"]) + async with SessionLocal() as session: + await catalog.write_entry(session, sid, "orders", {"relations": [ + {"id": rel_id, "columns": ["customer_ref"], "to_table": "customers", "to_columns": ["id"], + "cardinality": "many_to_one", "coverage": None, "source": "llm", "status": "proposed"}]}, + if_version=0, actor=None) + + body = await _profile(client, sid, ["orders"]) + rel = _relation((await _notes(client, sid, "orders"))["notes"], "customer_ref") + assert (rel["source"], rel["status"]) == ("profile", "proposed") + assert rel["coverage"] == round(1 / 32, 4) + assert "覆盖率 3.1%" in rel["note"] and "可能不是这条关系" in rel["note"] + assert "子表取了 32 个不同键值,父表对上 1 个" in rel["note"] + assert "31 个不是整数写法" in rel["note"] and "未计入" not in rel["note"] + finding = _table(body, "orders")["findings"][0] + assert (finding["status"], finding["sample"], finding["matched"]) == ("proposed", 32, 1) + + async def test_confirmed_relation_only_gets_coverage_and_cardinality(client, make_source, scenic_db): sid = await make_source(scenic_db, options=_on(), draft=["visits"]) detail = await _notes(client, sid, "visits") diff --git a/backend/tests/test_catalog_profile_units.py b/backend/tests/test_catalog_profile_units.py index 4fa1472f..22933754 100644 --- a/backend/tests/test_catalog_profile_units.py +++ b/backend/tests/test_catalog_profile_units.py @@ -40,7 +40,7 @@ def test_settings_default_off_and_parse_leniently(): ({"table_max_s": 9}, "每张表时长上限"), ({"table_max_s": 601}, "每张表时长上限"), ({"query_timeout_s": 61}, "单条查询时限"), - ({"sample_size": 5}, "抽样键值数"), + ({"sample_size": 5}, "核对键值数"), ({"max_scan_rows": -1}, "整表统计行数上限"), ({"enabled": "true"}, "开启数据剖析"), ({"table_max_queries": True}, "每张表最多查询数"), @@ -143,8 +143,16 @@ def test_profile_sql_passes_readonly_guard_on_every_dialect(kind): d.unique_check_sql(tbl, col), d.value_counts_sql(tbl, col, 21, scan_cap=None), d.value_counts_sql(tbl, col, 21, scan_cap=100_000), - d.min_max_sql(tbl, [col]), + d.min_max_sql(tbl, [(col, False)]), + d.min_max_sql(tbl, [(col, True)]), + d.day_volume_sql(tbl, col, "2026-08-01"), + d.window_value_counts_sql(tbl, col, col, "2026-08-01", 21), ] + sampled = d.sample_value_counts_sql(tbl, col, 1, 21) + if kind in ("postgresql", "oracle"): + sqls.append(sampled) + else: + assert sampled is None # MySQL、SQLite 没有块采样 for sql in sqls: assert guard.check(sql, readonly=True, dialect=kind) == sql, sql diff --git a/backend/tests/test_catalog_semantics.py b/backend/tests/test_catalog_semantics.py index 25eb9aad..d15e0332 100644 --- a/backend/tests/test_catalog_semantics.py +++ b/backend/tests/test_catalog_semantics.py @@ -14,6 +14,7 @@ """ from __future__ import annotations +import re import sqlite3 import pytest @@ -365,7 +366,7 @@ async def test_profile_marks_codes_complete_only_when_the_whole_table_was_read(c capped = await make_source(scenic_db, options=_on(max_scan_rows=0, sample_size=10), draft=["visits"]) await _profile(client, capped, ["visits"]) codes = (await _notes(client, capped, "visits"))["notes"]["columns"]["status"]["codes"] - assert "按前 10 行统计" in codes["note"] and "complete" not in codes + assert "按存储顺序取前 10 行统计" in codes["note"] and "complete" not in codes # ========================================================================== @@ -408,6 +409,95 @@ async def test_rerun_on_another_day_with_same_data_keeps_the_version(client, mak assert codes["note"].startswith("数据剖析:") and "2026-10-03" not in codes["note"] +#: 升级前剖析写的说法 → 现在的说法(下一轮 C1:大表看的是按存储顺序的前 N 行,不是抽样)。测试里反过来用:把新写法 +#: 改回旧写法,模拟升级前写进目录的备注 +_WORDING = (("覆盖率 ", "抽样覆盖率 ", ",可能不是这条关系"), ("子表取了 ", "子表抽样 ", None), + ("(按存储顺序只看前 ", "(只看前 ", None), ("按存储顺序取前 ", "按前 ", None)) + + +def _legacy(note: str) -> str: + out = note + for new, old, tail in _WORDING: + if tail: + out = re.sub(re.escape(new) + r"(\S+?)" + re.escape(tail), lambda m: old + m.group(1) + tail, out) + else: + out = out.replace(new, old) + return out + + +def _profile_notes(notes: dict) -> list[dict]: + """目录里剖析写了备注的项(关系、码值、业务日期)。""" + found = [r for r in notes.get("relations") or [] if catalog.is_profile_note(r.get("note"))] + for column in (notes.get("columns") or {}).values(): + found += [item for item in column.values() if isinstance(item, dict) + and catalog.is_profile_note(item.get("note"))] + bdate = notes.get("business_date") + return found + ([bdate] if isinstance(bdate, dict) and catalog.is_profile_note(bdate.get("note")) else []) + + +def test_profile_notes_in_the_old_wording_are_the_same_note(): + """改过的四处说法:旧写法和新写法一字之外都一样才算同一句;数变了、或者不是剖析写的备注,照旧算不同。""" + old = "数据剖析:抽样覆盖率 15%,可能不是这条关系;子表抽样 20 个不同键值(只看前 100 行),父表对上 3 个;被指向列是主键。" + new = "数据剖析:覆盖率 15%,可能不是这条关系;子表取了 20 个不同键值(按存储顺序只看前 100 行),父表对上 3 个;被指向列是主键。" + assert catalog.same_profile_note(old, new) and catalog.same_profile_note(new, old) + assert catalog.same_profile_note(old.replace("数据剖析:", "数据剖析(2026-10-01):"), new) + assert not catalog.same_profile_note(old, new.replace("对上 3 个", "对上 4 个")) + codes_old = "数据剖析(2026-10-01):按前 10 行统计,10 行非空值共 2 个取值:1(90%)、0(10%)。" + codes_new = "数据剖析:按存储顺序取前 10 行统计,10 行非空值共 2 个取值:1(90%)、0(10%)。" + assert catalog.same_profile_note(codes_old, codes_new) + assert not catalog.same_profile_note(codes_old, codes_new.replace("前 10 行", "前 20 行")) + # 起草写的有效行过滤备注:只认接在后面的剖析核对那句 + filter_old = "依据:status 为 1 才是有效记录。剖析核对:按前 100,000 行统计,排除 3% 的行" + filter_new = "依据:status 为 1 才是有效记录。剖析核对:按存储顺序取前 100,000 行统计,排除 3% 的行" + assert catalog.same_profile_note(filter_old, filter_new) + assert not catalog.same_profile_note(filter_old, filter_new.replace("3%", "4%")) + # 不是剖析写的备注:「抽样覆盖率」是人写的原话,不换 + assert not catalog.same_profile_note("人工备注:抽样覆盖率 15%,可能不是这条关系", + "人工备注:覆盖率 15%,可能不是这条关系") + assert not catalog.same_profile_note(new, "覆盖率 15%,可能不是这条关系") + + +async def test_rerun_after_upgrade_keeps_the_version_when_only_the_wording_changed(client, make_source, tmp_path): + """升级前写进目录的剖析备注用的是旧说法(「抽样覆盖率」「子表抽样」「(只看前 N 行)」「按前 N 行统计」)。数据 + 没变、升级后第一次重新剖析:结论一字不差、只是措辞改了,不算变化——否则每张有剖析结论的表都升一个版本,发布时 + 记下的目录版本对不上,之后每次正式运行都提醒「目录有变化」。""" + path = scenic.build(tmp_path / "wording.db") + db = sqlite3.connect(path) + try: + db.execute("CREATE TABLE lots (id INTEGER PRIMARY KEY, name TEXT NOT NULL)") + db.executemany("INSERT INTO lots VALUES (?, ?)", [(i, f"{i} 号停车区") for i in (1, 2, 3)]) + db.executemany("INSERT INTO parking_records VALUES (?, ?, ?, ?, ?, ?)", + [(i, 1 + i % 20, None, "2026-08-01 09:00:00", None, 10.0) for i in range(1, 101)]) + db.commit() + finally: + db.close() + tables = ["visits", "parking_records"] + whole = await make_source(path, options=_on(), draft=tables) + capped = await make_source(path, options=_on(max_scan_rows=0, sample_size=10), draft=tables) + seen: set[str] = set() + for sid in (whole, capped): + await _profile(client, sid, tables) + for table in tables: + detail = await _notes(client, sid, table) + notes = detail["notes"] + for item in _profile_notes(notes): + old = _legacy(item["note"]) + seen |= {o for _, o, _ in _WORDING if o in old} + item["note"] = old + async with SessionLocal() as session: + await catalog.write_entry(session, sid, table, notes, if_version=detail["version"], actor=None) + assert seen == {o for _, o, _ in _WORDING}, "四处旧说法都写进了目录" + + for sid in (whole, capped): + before = {t: await _notes(client, sid, t) for t in tables} + body = await _profile(client, sid, tables) + for table in tables: + done = _table(body, table) + assert (done["added"], done["updated"], done["removed"]) == (0, 0, 0), (table, done) + assert done["version"] == before[table]["version"] + assert (await _notes(client, sid, table))["notes"] == before[table]["notes"] + + def test_merge_ignores_a_profile_note_that_only_changed_its_date(): old = {"columns": {"status": {"codes": catalog.make_item( {"1": "", "0": ""}, "profile", "proposed", note="数据剖析(2026-10-01):统计全表,1500 行非空值共 2 个取值。", diff --git a/backend/tests/test_catalog_view.py b/backend/tests/test_catalog_view.py new file mode 100644 index 00000000..3edd71d2 --- /dev/null +++ b/backend/tests/test_catalog_view.py @@ -0,0 +1,270 @@ +"""目录视图接缝(data/catalog_view.py;细案 01 做法 5,总方案 X7)。 + +目录在四处被读:运行时的 db_schema / db_query(目录同时冻结进表结构快照)、助手查看的表详情、自查的 SQL 检查器、双盲复算。 +这里守:四种视图(当前目录、只用已核实和已确认的项、关、目录快照)在四处给出的是同一份;缺省是当前目录、行为不变;运行 +从 Run.options 读视图(随发起定下,审批恢复、续跑沿用),冻结快照的 meta 记下不是缺省的视图;lint:这几个消费模块不许 +绕过视图直接 read_catalog、直接查 CatalogNote。 + +库、表、列、目录内容都是虚构的。 +""" +from __future__ import annotations + +import asyncio +import re +import sqlite3 +import uuid +from pathlib import Path + +import pytest +from sqlalchemy import select + +from app.core.artifact_store import put_json +from app.data import catalog, catalog_view +from app.data.catalog import make_item +from app.data.catalog_view import CatalogSnapshotMissing, CatalogView, CatalogViewInvalid +from app.data.introspect import introspect +from app.db.base import SessionLocal +from app.db.models import Artifact, DataSource, Run +from app.engine.runner import run_manager +from tests.fixtures.sources import drop_source + +APP = Path(__file__).resolve().parents[1] / "app" + +#: 已确认的一项、推断的一项、驳回的一项:三种视图各留哪些一眼看得出 +NOTES = { + "label": make_item("门店订单", "human"), # confirmed + "grain": make_item("每笔订单一行", "llm"), # proposed + "description": make_item("旧说明", "llm", "rejected"), # rejected + "columns": {"amount": {"label": make_item("实收金额", "human")}, + "status": {"label": make_item("订单状态", "llm")}}, +} +SNAPSHOT_NOTES = {"label": make_item("快照里的订单", "human")} + +VIEWS = { + "live": CatalogView(), + "reviewed": CatalogView(statuses="reviewed"), + "off": CatalogView(mode="off"), +} + + +@pytest.fixture +async def shop(tmp_path): + path = tmp_path / "view_shop.db" + db = sqlite3.connect(path) + db.executescript("CREATE TABLE orders (id INTEGER PRIMARY KEY, amount REAL NOT NULL, status INTEGER);" + "INSERT INTO orders VALUES (1, 30.0, 1), (2, 20.0, 9);") + db.commit() + db.close() + async with SessionLocal() as session: + row = DataSource(name=f"viewshop_{uuid.uuid4().hex[:6]}", kind="sqlite", database=str(path)) + session.add(row) + await session.commit() + row.schema_cache = await introspect(row) + await session.commit() + await session.refresh(row) + await catalog.write_entry(session, row.id, "orders", NOTES, if_version=0, actor=None) + session.expunge(row) + yield row + await drop_source(row.id) + + +async def _snapshot_view(source) -> CatalogView: + sid = await put_json({"sources": {source.id: {"tables": {"orders": {"version": 7, "notes": SNAPSHOT_NOTES}}}}}, + kind=catalog_view.SNAPSHOT_KIND) + return CatalogView(mode="snapshot", snapshot=sid) + + +def _says(text: str) -> dict[str, bool]: + return {word: word in text for word in ("门店订单", "每笔订单一行", "旧说明", "实收金额", "订单状态", "快照里的订单")} + + +EXPECT = { + "live": {"门店订单": True, "每笔订单一行": True, "旧说明": False, "实收金额": True, "订单状态": True, "快照里的订单": False}, + "reviewed": {"门店订单": True, "每笔订单一行": False, "旧说明": False, "实收金额": True, "订单状态": False, + "快照里的订单": False}, + "off": dict.fromkeys(("门店订单", "每笔订单一行", "旧说明", "实收金额", "订单状态", "快照里的订单"), False), + "snapshot": {"门店订单": False, "每笔订单一行": False, "旧说明": False, "实收金额": False, "订单状态": False, + "快照里的订单": True}, +} + + +async def _all_views(source) -> dict[str, CatalogView]: + return {**VIEWS, "snapshot": await _snapshot_view(source)} + + +# -------------------------------------------------------------------------- +# 视图本身 +# -------------------------------------------------------------------------- + + +def test_parse_and_defaults(): + assert CatalogView.parse(None) is catalog_view.LIVE and catalog_view.LIVE.is_default + assert CatalogView.parse({"mode": "off"}) == CatalogView(mode="off") + assert catalog_view.describe(None) is None and catalog_view.describe(CatalogView()) is None + assert catalog_view.describe(CatalogView(mode="off")) == {"mode": "off", "statuses": "all"} + assert catalog_view.of_options({"catalog_view": {"statuses": "reviewed"}}) == CatalogView(statuses="reviewed") + for bad in ({"mode": "nope"}, {"statuses": "some"}, {"mode": "snapshot"}, {"mode": "live", "snapshot": "x"}, + {"extra": 1}, "off"): + with pytest.raises(CatalogViewInvalid): + CatalogView.parse(bad) + + +async def test_a_missing_snapshot_is_an_error_not_the_live_catalog(shop): + with pytest.raises(CatalogSnapshotMissing): + await catalog_view.read(CatalogView(mode="snapshot", snapshot="0" * 64), shop) + + +# -------------------------------------------------------------------------- +# 四处一致 +# -------------------------------------------------------------------------- + + +async def test_runtime_db_schema_follows_the_view(shop): + from app.tools.datasource import SCHEMA_PREFIX, build_datasource_tools + from app.tools.registry import ToolContext, Versions + + for name, view in (await _all_views(shop)).items(): + ctx = ToolContext(run_id="", node_id="", data_versions=Versions.CURRENT, catalog_view=view) + async with SessionLocal() as session: + [tool] = await build_datasource_tools([f"{SCHEMA_PREFIX}{shop.name}"], ctx, session) + assert _says(await tool.coroutine(table="orders")) == EXPECT[name], name + + +async def test_look_table_detail_follows_the_view(shop): + from app.look.service import LookService + from app.look.tables import TableKind + + for name, view in (await _all_views(shop)).items(): + looks = LookService([await TableKind.load([shop], catalog_view=view)]) + got = await looks.look({"op": "look", "need": "订单", "targets": [{"kind": "table", "name": "orders"}]}) + assert _says(got.text) == EXPECT[name], name + + +async def test_sql_checkers_follow_the_view(shop): + from app.data.sqlcheck import load_checkers + + for name, view in (await _all_views(shop)).items(): + async with SessionLocal() as session: + checkers = await load_checkers(session, [shop], catalog_view=view) + if name == "off": + assert checkers == {} + continue + notes = checkers[shop.name]._raw["orders"] + assert _says(str(notes)) | {"旧说明": False} == EXPECT[name] | {"旧说明": False}, name + if name == "reviewed": + # 驳回的项原样留着交给检查器(它自己按驳回处理),推断的去掉 + assert notes["description"]["status"] == "rejected" and "grain" not in notes + + +async def test_dual_check_blind_catalog_follows_the_view(shop): + from app.engine.nodes.dual_check import BLIND_NO_CATALOG_PROMPT, _catalog_text, _source_info, _Sources + + for name, view in (await _all_views(shop)).items(): + sources = _Sources(view) + text = await _catalog_text(sources, shop.name) + assert _says(text) == EXPECT[name], name + info = await _source_info(sources, shop.name) + assert ("orders" in info["notes"]) == (name != "off"), name + if name == "off": + # 关目录:表清单(表结构里的事实)照给,目录的说明一句都没有 + assert "orders" in text and "数据目录(" not in text and BLIND_NO_CATALOG_PROMPT not in text + + +async def test_the_default_is_the_live_catalog_everywhere(shop): + """不给视图:和以前一样读当前目录。""" + from app.engine.nodes.dual_check import _catalog_text, _Sources + from app.look.service import LookService + from app.look.tables import TableKind + from app.tools.datasource import _catalog_of + + assert _says(str({t: e.notes for t, e in (await _catalog_of(shop)).items()}))["门店订单"] + looks = LookService([await TableKind.load([shop])]) + got = await looks.look({"op": "look", "need": "订单", "targets": [{"kind": "table", "name": "orders"}]}) + assert _says(got.text) == EXPECT["live"] + assert _says(await _catalog_text(_Sources(), shop.name)) == EXPECT["live"] + + +# -------------------------------------------------------------------------- +# 运行:从 Run.options 读视图,冻结快照记下 +# -------------------------------------------------------------------------- + + +def _graph(source: str) -> dict: + def node(nid, ntype, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}} + + return {"nodes": [node("start", "input"), + node("q", "tool", tool=f"db_query__{source}", args={"sql": "SELECT SUM(amount) AS 金额 FROM orders"}, + assign_to="rows"), + node("out", "output", fields=[{"name": "rows", "value": "{{ vars.rows }}"}])], + "edges": [{"source": "start", "target": "q"}, {"source": "q", "target": "out"}]} + + +async def _frozen(options: dict | None, source) -> tuple[dict, dict]: + from app.core.artifact_store import load + from app.engine.evidence import SCHEMA_SNAPSHOT + from app.engine.launch import launch_run + + async with SessionLocal() as session: + run = await launch_run(session, graph=_graph(source.name), input_payload={}, actor="测试", + origin_kind="canvas", options=options) + run_id = run.id + for _ in range(200): + async with SessionLocal() as session: + status = (await session.get(Run, run_id)).status + if status in ("succeeded", "failed", "cancelled"): + break + await asyncio.sleep(0.05) + assert status == "succeeded", status + async with SessionLocal() as session: + ref = (await session.execute(select(Artifact).where( + Artifact.run_id == run_id, Artifact.kind == SCHEMA_SNAPSHOT))).scalars().first() + assert ref is not None + return load(ref.id), ref.meta or {} + + +async def test_a_run_reads_its_view_from_options_and_the_freeze_says_so(shop): + await run_manager.setup() + try: + content, meta = await _frozen(None, shop) + assert "orders" in content.get("catalog", {}), "缺省:当前目录照旧冻结" + assert "catalog_view" not in meta, "缺省视图不记,meta 和以前一样" + + content, meta = await _frozen({"catalog_view": {"mode": "off"}}, shop) + assert "catalog" not in content, "关目录:快照里没有目录" + assert meta["catalog_view"] == {"mode": "off", "statuses": "all"} + + content, meta = await _frozen({"catalog_view": {"statuses": "reviewed"}}, shop) + frozen = content["catalog"]["orders"]["notes"] + assert "grain" not in frozen and frozen["label"]["value"] == "门店订单" + assert meta["catalog_view"] == {"mode": "live", "statuses": "reviewed"} + finally: + await run_manager.shutdown() + + +def test_of_run_reads_the_run_options(): + from types import SimpleNamespace + + from app.engine.runner import RUN_OPTIONS_KEY + + run = SimpleNamespace(extra={RUN_OPTIONS_KEY: {"catalog_view": {"mode": "off"}}}) + assert catalog_view.of_run(run) == CatalogView(mode="off") + assert catalog_view.of_run(SimpleNamespace(extra={})) == catalog_view.LIVE + + +# -------------------------------------------------------------------------- +# lint:消费模块只经视图读目录 +# -------------------------------------------------------------------------- + +#: 给模型、给自查读目录的模块(四处消费 + 它们的帮手)。要读目录一律经 data/catalog_view.py +CONSUMERS = ("tools/datasource.py", "look/tables.py", "data/sqlcheck.py", "engine/nodes/dual_check.py", + "engine/verify_agent.py", "api/copilot_context.py") +FORBIDDEN = re.compile(r"\bread_catalogs?\s*\(|\bread_entry\s*\(|\bCatalogNote\b") + + +@pytest.mark.parametrize("module", CONSUMERS) +def test_consumers_never_read_the_catalog_directly(module): + text = (APP / module).read_text(encoding="utf-8") + code = "\n".join(line for line in text.splitlines() if not line.lstrip().startswith("#")) + hits = [m.group(0) for m in FORBIDDEN.finditer(code)] + assert not hits, f"{module} 绕过了目录视图:{hits}" diff --git a/backend/tests/test_chat_turn_policy.py b/backend/tests/test_chat_turn_policy.py new file mode 100644 index 00000000..2f3bd8d1 --- /dev/null +++ b/backend/tests/test_chat_turn_policy.py @@ -0,0 +1,202 @@ +"""问数据一轮的「决定」(app/chat/turn.py;细案 01 做法 1)。 + +要不要运行、输入怎么填、要不要带着原图重来、重来时说什么,从问数据页下沉到后端:final 带 next,复核带 retry_instruction, +问数据页照它们走;老服务端没给这两个字段时,问数据页退回它自己那份同样的规则。这里守: + +- 各条决定的分支(next_step、fill_input、retry_after_review、retry_instruction 的两种措辞); +- 前后端对照:frontend/src/store/chat.ts 里留着的那份常量(MAX_ATTEMPTS、fillInput 认问题字段的正则、两种重来的措辞) + 和这里逐字一致——写法同 test_frontend_type_unions.py,两边一改就对不上; +- /copilot/review 按这一轮试过几次给 retry_instruction:可以重来时是那句话,不重来是 null。 + +问题、字段名都是虚构的。 +""" +from __future__ import annotations + +import re +import uuid +from pathlib import Path + +import pytest +from httpx import ASGITransport, AsyncClient + +from app.chat import turn +from app.main import app + +CHAT_TS = Path(__file__).resolve().parents[2] / "frontend" / "src" / "store" / "chat.ts" + + +def _final(*, issues=(), autorun=True, **extra) -> dict: + return {"op": "final", "graph": {"nodes": [], "edges": []}, "issues": list(issues), "autorun": autorun, **extra} + + +ERROR = {"level": "error", "message": "图是空的"} +WARNING = {"level": "warning", "message": "多余的 {{ }}"} + + +# -------------------------------------------------------------------------- +# next_step +# -------------------------------------------------------------------------- + + +def test_next_step_branches(): + assert turn.next_step(_final()) == "run" + assert turn.next_step(_final(issues=[WARNING])) == "run", "warning 不挡运行" + assert turn.next_step(_final(autorun=False)) == "ready", "模型说不运行:只建不跑" + assert turn.next_step(_final(autorun=False, asked={"question": "哪个月?", "options": []})) == "ready" + assert turn.next_step(_final(issues=[ERROR])) == "fail", "自查过还有 error:判失败,不再整轮重来" + assert turn.next_step(_final(issues=[ERROR]), self_checked=False) == "retry", "老服务端没自查:重来一次" + assert turn.next_step(_final(issues=[ERROR], autorun=False)) == "fail", "没过检查比只建不跑优先" + assert turn.next_step({"op": "reply", "text": "已经查过了"}) == "reply" + assert turn.next_step({"op": "error", "message": "x"}) == "fail" + assert turn.next_step(None) == "asked", "流结束时只有反问" + assert set(turn.NEXT_STEPS) == {"run", "ready", "fail"} + + +# -------------------------------------------------------------------------- +# fill_input +# -------------------------------------------------------------------------- + + +def _graph(*fields: dict) -> dict: + return {"nodes": [{"id": "start", "type": "input", "data": {"label": "输入", "config": {"fields": list(fields)}}}, + {"id": "out", "type": "output", "data": {"label": "成果", "config": {}}}], "edges": []} + + +def test_fill_input_rules(): + assert turn.fill_input(_graph(), "上周销售额", "问:…") == {"question": "上周销售额", "history": "问:…"} + assert turn.fill_input({"nodes": []}, "上周销售额") == {"question": "上周销售额", "history": ""} + # 名字像「问题」的优先,其余字段填默认值(没有就空串) + got = turn.fill_input(_graph({"name": "region", "default": "华东"}, {"name": "topic"}, {"name": "limit"}), "退款率") + assert got == {"history": "", "region": "华东", "topic": "退款率", "limit": ""} + got = turn.fill_input(_graph({"name": "门店"}, {"name": "需求说明"}), "退款率") + assert got["需求说明"] == "退款率" and got["门店"] == "" + # 没有像问题的:第一个必填的;再没有:第一个 + got = turn.fill_input(_graph({"name": "a"}, {"name": "b", "required": True}), "退款率") + assert got["b"] == "退款率" and got["a"] == "" + got = turn.fill_input(_graph({"name": "a"}, {"name": "b"}), "退款率") + assert got["a"] == "退款率" + # 大小写不敏感 + assert turn.fill_input(_graph({"name": "UserQuery"}), "退款率")["UserQuery"] == "退款率" + + +# -------------------------------------------------------------------------- +# 复核之后:要不要重来、说什么 +# -------------------------------------------------------------------------- + + +BROKEN = {"retry": True, "severity": "broken", "note": "结论不可用", + "signals": [{"kind": "step_limit", "detail": "agent 用满了 8 步"}, {"kind": "tool_error", "detail": "表不存在"}]} + + +def test_retry_after_review(): + assert turn.retry_after_review(BROKEN, 0) is True + assert turn.retry_after_review(BROKEN, 1) is False, "一轮最多自动再试一次" + assert turn.retry_after_review({**BROKEN, "severity": "degraded"}, 0) is False + assert turn.retry_after_review({**BROKEN, "retry": False}, 0) is False + assert turn.retry_after_review(None, 0) is False + assert turn.review_reason(BROKEN) == "agent 用满了 8 步;表不存在" + assert turn.review_reason({"signals": [], "note": "结论不可用"}) == "结论不可用" + + +def test_retry_instruction_two_wordings(): + with_base = turn.retry_instruction("上周销售额", "表不存在", True) + assert with_base.startswith("这个工作流上一次运行没有成功:表不存在。\n\n请针对这个原因改它") + assert with_base.endswith("改完继续回答原来的问题:上周销售额") + scratch = turn.retry_instruction("上周销售额", "表不存在", False) + assert scratch == "上周销售额\n\n(上一次没有成功:表不存在。请针对这个原因调整,重新给出完整的工作流。)" + + +# -------------------------------------------------------------------------- +# 前后端对照 +# -------------------------------------------------------------------------- + + +def _ts() -> str: + return CHAT_TS.read_text(encoding="utf-8") + + +def test_max_attempts_matches_the_frontend(): + found = re.search(r"const MAX_ATTEMPTS = (\d+)", _ts()) + assert found and int(found.group(1)) == turn.MAX_ATTEMPTS + + +def test_the_question_field_pattern_matches_the_frontend(): + found = re.search(r"/(question\|[^/]+)/i\.test", _ts()) + assert found, "chat.ts 的 fillInput 里找不到认问题字段的正则" + assert found.group(1) == turn.QUESTION_FIELD.pattern and turn.QUESTION_FIELD.flags & re.IGNORECASE + + +def _template(source: str) -> str: + """一段 TS 里由 + 连起来的模板字符串 → 一个 Python format 模板(${x} → {x},\\n → 换行)。""" + pieces = re.findall(r"`([^`]*)`", source) + text = "".join(pieces).replace("\\n", "\n") + return re.sub(r"\$\{(\w+)\}", r"{\1}", text) + + +def test_the_retry_wordings_match_the_frontend(): + body = re.search(r"function retryInstruction\([^)]*\): string \{(.*?)\n\}", _ts(), re.S) + assert body, "chat.ts 里找不到 retryInstruction" + with_base, scratch = re.split(r"\n\s*: ", body.group(1).split("hasBase", 1)[1].split("?", 1)[1], maxsplit=1) + assert _template(with_base) == turn.RETRY_WITH_BASE + assert _template(scratch) == turn.RETRY_FROM_SCRATCH + + +# -------------------------------------------------------------------------- +# /copilot/review 给 retry_instruction +# -------------------------------------------------------------------------- + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +async def _run(output: dict, graph: dict, code: str) -> str: + from app.core.events import EventType + from app.db.base import SessionLocal + from app.db.models import Run, RunEvent + + async with SessionLocal() as session: + run = Run(id=uuid.uuid4().hex, status="succeeded", output=output, graph=graph) + session.add(run) + await session.flush() + session.add(RunEvent(run_id=run.id, seq=1, type=EventType.LOG, node_id="a", ts=0.0, + data={"level": "warn", "code": code, "message": "agent 用满了 8 步还没给出结论"})) + await session.commit() + return run.id + + +async def test_review_says_whether_and_how_to_retry(client, monkeypatch): + import app.api.copilot as copilot + from app.providers.factory import ProviderNotConfigured + + async def unconfigured(*_a, **_k): + raise ProviderNotConfigured("没有可用的 provider") + + # 没配模型:规则层照样判得出不可用、该重来 + monkeypatch.setattr(copilot, "get_chat_model", unconfigured) + graph = _graph({"name": "question"}) + run_id = await _run({"answer": "大概 3 家"}, graph, "step_limit") + body = (await client.post("/api/copilot/review", json={"run_id": run_id, "question": "有几家门店"})).json() + assert body["severity"] == "broken" and body["retry"] is True + assert body["retry_instruction"] == turn.retry_instruction("有几家门店", turn.review_reason(body), True) + # 这一轮已经重来过一次:不再给 + body = (await client.post("/api/copilot/review", + json={"run_id": run_id, "question": "有几家门店", "attempt": 1})).json() + assert body["retry_instruction"] is None + # 运行的图是空的:措辞换成从零再建 + bare = await _run({"answer": "大概 3 家"}, {}, "step_limit") + body = (await client.post("/api/copilot/review", json={"run_id": bare, "question": "有几家门店"})).json() + assert body["retry_instruction"].startswith("有几家门店\n\n(上一次没有成功:") + # 干净的运行:不重来 + from app.db.base import SessionLocal + from app.db.models import Run + + async with SessionLocal() as session: + clean = Run(status="succeeded", output={"answer": "3 家"}, graph=graph) + session.add(clean) + await session.commit() + clean_id = clean.id + body = (await client.post("/api/copilot/review", json={"run_id": clean_id, "question": "有几家门店"})).json() + assert body["verdict"] == "ok" and body["retry_instruction"] is None diff --git a/backend/tests/test_conversation_status.py b/backend/tests/test_conversation_status.py index 1a9fe345..1f2a9ce1 100644 --- a/backend/tests/test_conversation_status.py +++ b/backend/tests/test_conversation_status.py @@ -43,9 +43,9 @@ async def _turn(client, conv_id: str, question: str = "订单数多少", **patch return turn -async def _run(status: str, *, pending: bool = False) -> str: +async def _run(status: str, *, pending: bool = False, end_reason: str | None = None) -> str: async with SessionLocal() as session: - run = Run(workflow_name="状态测试", status=status) + run = Run(workflow_name="状态测试", status=status, end_reason=end_reason) session.add(run) await session.flush() if pending: @@ -162,6 +162,39 @@ async def test_the_list_says_where_each_conversation_last_stopped(client): assert got == cases +async def test_a_rejected_turn_is_listed_as_rejected_not_cancelled(client): + """被审批人驳回结束的一轮(运行 cancelled + end_reason=rejected,05-已知遗留 (a)):列表说「被驳回」,不和手动停止混成 + cancelled——发问的人不一定知道被驳回了,以前驳回记成失败,列表标的是「失败」。认法两条:前端落库时记下的 + meta.rejected,和页面关掉、轮次还停在 running 时运行的 end_reason。手动停止照旧是 cancelled。""" + cases: dict[str, tuple[str, str | None]] = {} + + conv = await _conversation(client, "驳回:前端落库了") + run = await _run("cancelled", end_reason="rejected") + await _turn(client, conv, run_id=run, status="error", error="已取消", + meta={"v": 1, "runId": run, "outcome": "cancelled", "rejected": {"sub": "张工 驳回了「付款」"}}) + cases[conv] = ("rejected", run) + + conv = await _conversation(client, "驳回:页面关了") + run = await _run("cancelled", end_reason="rejected") + await _turn(client, conv, run_id=run) + cases[conv] = ("rejected", run) + + conv = await _conversation(client, "手动停止:页面关了") + run = await _run("cancelled", end_reason="stopped") + await _turn(client, conv, run_id=run) + cases[conv] = ("cancelled", run) + + conv = await _conversation(client, "手动停止:前端落库了") + run = await _run("cancelled", end_reason="stopped") + await _turn(client, conv, run_id=run, status="error", error="已取消", meta={"v": 1, "runId": run, "outcome": "cancelled"}) + cases[conv] = ("cancelled", run) + + listed = await _listed(client) + assert {cid: (listed[cid]["last_status"], listed[cid]["last_run_id"]) for cid in cases} == cases + detail = (await client.get(f"/api/conversations/{next(iter(cases))}")).json() + assert detail["last_status"] == "rejected" + + async def test_only_the_last_turn_counts(client): conv = await _conversation(client, "后来好了") await _turn(client, conv, status="error", error="查询超时") diff --git a/backend/tests/test_conversations.py b/backend/tests/test_conversations.py index df0d1fb9..55f03d55 100644 --- a/backend/tests/test_conversations.py +++ b/backend/tests/test_conversations.py @@ -215,7 +215,8 @@ async def _stream_graph(client, monkeypatch, conv_id: str, lines: list[str]) -> }) as r: last = {} async for line in r.aiter_lines(): - if line.startswith("data: "): + # 末尾那条 usage 是这一轮助手的用量(answer_ops),不是生成的操作 + if line.startswith("data: ") and not line.startswith('data: {"op": "usage"'): last = json.loads(line[6:]) return last diff --git a/backend/tests/test_copilot_answer_ops.py b/backend/tests/test_copilot_answer_ops.py new file mode 100644 index 00000000..c100b15c --- /dev/null +++ b/backend/tests/test_copilot_answer_ops.py @@ -0,0 +1,173 @@ +"""生成过程抽成可直接调用的 answer_ops(细案 01 做法 2):SSE 路由只包一层,同一个剧本下输出逐条不变。 + +评测和问数据页要走同一条生成路径:评测直接迭代 answer_ops 拿 model / look / check / final / reply / error 这些操作, +问数据页照旧收 SSE。这里守: + +- 几种结局(建图交付、直接回答、只问不改、自查改不好、模型出错、没给操作、输出被截断)的 SSE 输出,和抽出来之前 + 逐条一样(剧本见 SCENARIOS,抽出来之前录下的输出在 fixtures/copilot_answer_ops_baseline.json),末尾只多一条 usage, + final 只多一个 next(问数据一轮的决定,chat/turn.py); +- 直接迭代 answer_ops 和 SSE 收到的是同一串; +- 开流前的拒绝(模型没配好、限定的数据源都不在)照旧是 HTTP 错误,不进流。 + +剧本、需求都是虚构的。 +""" +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +from httpx import ASGITransport, AsyncClient +from langchain_core.messages import AIMessageChunk + +from app.main import app + +BASELINE = Path(__file__).parent / "fixtures" / "copilot_answer_ops_baseline.json" +#: 和时间有关、每次都不一样的字段:比较前去掉 +_TIMING = ("elapsed_ms", "ms", "duration_ms") + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +def _node(nid: str, ntype: str, config: dict) -> dict: + return {"op": "add_node", "node": {"id": nid, "type": ntype, "label": nid, "config": config}} + + +def _edge(a: str, b: str, **extra) -> dict: + return {"op": "add_edge", "edge": {"source": a, "target": b, **extra}} + + +def _loop_graph(condition: str) -> list[dict]: + return [ + {"op": "plan", "summary": "数一数"}, + _node("start", "input", {"fields": [{"name": "question"}]}), + _node("init", "transform", {"mode": "expression", "expression": "[1, 2]", "assign_to": "items"}), + _edge("start", "init"), + _node("lp", "loop", {"mode": "while", "condition": condition, "max_iterations": 2}), + _edge("init", "lp"), + _node("body", "transform", {"mode": "expression", "expression": "1"}), + _edge("lp", "body", sourceHandle="body"), + _edge("body", "lp"), + _node("out", "output", {"fields": [{"name": "结果", "value": "{{ vars.items }}"}]}), + _edge("lp", "out", sourceHandle="done"), + {"op": "done", "explanation": "数一数", "run": True}, + ] + + +def _fix(condition: str) -> list[dict]: + return [{"op": "plan", "summary": "改条件"}, + {"op": "update_node", "id": "lp", "config": {"mode": "while", "condition": condition, "max_iterations": 2}}, + {"op": "done", "explanation": "改好了"}] + + +def _chunks(ops: list) -> list[AIMessageChunk]: + return [op if isinstance(op, AIMessageChunk) else AIMessageChunk(content=json.dumps(op, ensure_ascii=False) + "\n") + for op in ops] + + +#: 剧本:(第一次调用吐的, 自查修正时吐的)。None 表示这一次调用直接抛错 +SCENARIOS: dict[str, tuple[list | None, list | None]] = { + "graph_ok": (_loop_graph("len(vars.items) > 5"), []), + "graph_fixed": (_loop_graph("{{ vars.items | length }} > 5"), _fix("len(vars.items) > 5")), + "check_failed": (_loop_graph("foo(vars.items)"), _fix("bar(vars.items)")), + "reply": ([{"op": "reply", "text": "上一轮已经查过了:一共 3 家门店。"}], []), + "ask_only": ([{"op": "plan", "summary": "先问清楚"}, + {"op": "ask", "question": "要看哪个月的?", "options": ["九月", "十月"]}], []), + "error": (None, []), + "no_ops": (["模型的随口注释,不是 JSON"], []), + "truncated": ([{"op": "plan", "summary": "想了很久"}, + AIMessageChunk(content="", response_metadata={"finish_reason": "length"})], []), +} + + +class _Scripted: + """照 SCENARIOS 吐:human 里有「上线前自查」时吐修正那一份。""" + + def __init__(self, first: list | None, fix: list | None) -> None: + self.first, self.fix = first, fix + + async def astream(self, messages): + human = next(text for role, text in messages if role == "human") + lines = self.fix if "上线前自查" in human else self.first + if lines is None: + raise RuntimeError("接入返回了 502") + for chunk in _chunks(lines): + yield chunk + + +def _use(monkeypatch, model) -> None: + import app.api.copilot as copilot + + async def _model(*_a, **_k): + return model, "mock-fast" + + monkeypatch.setattr(copilot, "get_chat_model", _model) + # 心跳和时间有关:拍子调大,剧本里不会有 + monkeypatch.setattr(copilot, "_HEARTBEAT_SECONDS", 3600.0) + + +def _clean(value): + if isinstance(value, dict): + return {k: _clean(v) for k, v in value.items() if k not in _TIMING} + if isinstance(value, list): + return [_clean(v) for v in value] + return value + + +async def _sse(client, body: dict) -> list[dict]: + out: list[dict] = [] + async with client.stream("POST", "/api/copilot/generate-stream", json=body) as r: + assert r.status_code == 200, await r.aread() + async for line in r.aiter_lines(): + if line.startswith("data: "): + out.append(json.loads(line[6:])) + return [_clean(op) for op in out if op.get("op") != "heartbeat"] + + +BODY = {"instruction": "数一数门店", "intent": "answer"} + + +#: final 带的 next(chat/turn.next_step):问数据页照它走 +NEXT = {"graph_ok": "run", "graph_fixed": "run", "check_failed": "fail", "ask_only": "ready"} + + +@pytest.mark.parametrize("name", list(SCENARIOS)) +async def test_the_sse_output_is_unchanged_but_for_a_trailing_usage(client, monkeypatch, name): + _use(monkeypatch, _Scripted(*SCENARIOS[name])) + got = await _sse(client, BODY) + expected = json.loads(BASELINE.read_text(encoding="utf-8"))[name] + assert got[-1]["op"] == "usage", got[-1] + # final 多了 next(问数据一轮的决定下沉到后端),别的一个字不变 + finals = [op for op in got if op["op"] == "final"] + assert [op.pop("next") for op in finals] == ([NEXT[name]] if name in NEXT else []) + assert got[:-1] == expected + + +@pytest.mark.parametrize("name", ["graph_ok", "reply", "error"]) +async def test_iterating_answer_ops_directly_gives_the_same_ops(client, monkeypatch, name): + from app.api.copilot import GenerateIn, answer_ops + + _use(monkeypatch, _Scripted(*SCENARIOS[name])) + sse = await _sse(client, BODY) + direct = [_clean(op) async for op in answer_ops(GenerateIn(**BODY)) if op.get("op") != "heartbeat"] + assert direct == sse + + +async def test_refusals_before_the_stream_stay_http_errors(client, monkeypatch): + import app.api.copilot as copilot + from app.providers.factory import ProviderNotConfigured + + async def _unconfigured(*_a, **_k): + raise ProviderNotConfigured("provider 'gw' 需要填 base_url") + + monkeypatch.setattr(copilot, "get_chat_model", _unconfigured) + r = await client.post("/api/copilot/generate-stream", json=BODY) + assert r.status_code == 400 and "模型接入「gw」" in r.json()["detail"] + + _use(monkeypatch, _Scripted(*SCENARIOS["reply"])) + r = await client.post("/api/copilot/generate-stream", json={**BODY, "datasource_ids": ["no-such-source"]}) + assert r.status_code == 400 and r.json()["code"] == "datasource_scope_empty" diff --git a/backend/tests/test_copilot_evidence_guide.py b/backend/tests/test_copilot_evidence_guide.py index 9473da07..8a2a184e 100644 --- a/backend/tests/test_copilot_evidence_guide.py +++ b/backend/tests/test_copilot_evidence_guide.py @@ -213,7 +213,8 @@ async def test_guessed_columns_are_sent_back_and_fixed_before_running(sales, mon async with client.stream("POST", "/api/copilot/generate-stream", json={"instruction": "上周销售额多少", "intent": "answer"}) as r: async for line in r.aiter_lines(): - if line.startswith("data: "): + # 末尾那条 usage 是这一轮助手的用量(answer_ops),不是生成的操作 + if line.startswith("data: ") and not line.startswith('data: {"op": "usage"'): events.append(json.loads(line[6:])) checks = [e for e in events if e["op"] == "check"] assert [c["status"] for c in checks] == ["repairing", "passed"], checks diff --git a/backend/tests/test_copilot_look.py b/backend/tests/test_copilot_look.py index 86ade6a8..fdc96299 100644 --- a/backend/tests/test_copilot_look.py +++ b/backend/tests/test_copilot_look.py @@ -99,7 +99,8 @@ async def _events(client, monkeypatch, model, body: dict, path: str = "/api/copi async with client.stream("POST", path, json=body) as r: assert r.status_code == 200, await r.aread() async for line in r.aiter_lines(): - if line.startswith("data: "): + # 末尾那条 usage 是这一轮助手的用量(answer_ops),不是生成的操作 + if line.startswith("data: ") and not line.startswith('data: {"op": "usage"'): out.append(json.loads(line[6:])) return out diff --git a/backend/tests/test_copilot_merge.py b/backend/tests/test_copilot_merge.py index 9f8c97da..e425652f 100644 --- a/backend/tests/test_copilot_merge.py +++ b/backend/tests/test_copilot_merge.py @@ -110,7 +110,8 @@ async def _stream(client, monkeypatch, model, body: dict) -> list[dict]: async with client.stream("POST", "/api/copilot/generate-stream", json=body) as r: assert r.status_code == 200, await r.aread() async for line in r.aiter_lines(): - if line.startswith("data: "): + # 末尾那条 usage 是这一轮助手的用量(answer_ops),不是生成的操作 + if line.startswith("data: ") and not line.startswith('data: {"op": "usage"'): events.append(json.loads(line[6:])) return events diff --git a/backend/tests/test_copilot_patch_layout.py b/backend/tests/test_copilot_patch_layout.py index 27525fbd..647d9e01 100644 --- a/backend/tests/test_copilot_patch_layout.py +++ b/backend/tests/test_copilot_patch_layout.py @@ -64,7 +64,8 @@ async def _events(client, monkeypatch, ops: list[dict], base: dict | None) -> li out: list[dict] = [] async with client.stream("POST", "/api/copilot/generate-stream", json=body) as r: async for line in r.aiter_lines(): - if line.startswith("data: "): + # 末尾那条 usage 是这一轮助手的用量(answer_ops),不是生成的操作 + if line.startswith("data: ") and not line.startswith('data: {"op": "usage"'): out.append(json.loads(line[6:])) return out diff --git a/backend/tests/test_copilot_prompt_evidence.py b/backend/tests/test_copilot_prompt_evidence.py index fe4250e1..ee6f9beb 100644 --- a/backend/tests/test_copilot_prompt_evidence.py +++ b/backend/tests/test_copilot_prompt_evidence.py @@ -59,7 +59,8 @@ async def generate(client, monkeypatch, model: Scripted) -> list[dict]: events: list[dict] = [] async with client.stream("POST", "/api/copilot/generate-stream", json={"instruction": "写一份订单周报"}) as r: async for line in r.aiter_lines(): - if line.startswith("data: "): + # 末尾那条 usage 是这一轮助手的用量(answer_ops),不是生成的操作 + if line.startswith("data: ") and not line.startswith('data: {"op": "usage"'): events.append(json.loads(line[6:])) return events diff --git a/backend/tests/test_copilot_scope.py b/backend/tests/test_copilot_scope.py index 0a6f9322..48d738a5 100644 --- a/backend/tests/test_copilot_scope.py +++ b/backend/tests/test_copilot_scope.py @@ -96,7 +96,8 @@ async def _stream(client, monkeypatch, model, body: dict) -> tuple[int, list[dic if r.status_code != 200: return r.status_code, [json.loads(await r.aread())] async for line in r.aiter_lines(): - if line.startswith("data: "): + # 末尾那条 usage 是这一轮助手的用量(answer_ops),不是生成的操作 + if line.startswith("data: ") and not line.startswith('data: {"op": "usage"'): events.append(json.loads(line[6:])) return 200, events diff --git a/backend/tests/test_copilot_selfcheck.py b/backend/tests/test_copilot_selfcheck.py index 6e56e193..add5b950 100644 --- a/backend/tests/test_copilot_selfcheck.py +++ b/backend/tests/test_copilot_selfcheck.py @@ -85,7 +85,8 @@ async def _generate(client, monkeypatch, model: _Scripted) -> list[dict]: async with client.stream("POST", "/api/copilot/generate-stream", json={"instruction": "数一数"}) as r: async for line in r.aiter_lines(): - if line.startswith("data: "): + # 末尾那条 usage 是这一轮助手的用量(answer_ops),不是生成的操作 + if line.startswith("data: ") and not line.startswith('data: {"op": "usage"'): events.append(json.loads(line[6:])) return events diff --git a/backend/tests/test_copilot_sqlcheck.py b/backend/tests/test_copilot_sqlcheck.py index 2263ccc3..2dd9f809 100644 --- a/backend/tests/test_copilot_sqlcheck.py +++ b/backend/tests/test_copilot_sqlcheck.py @@ -195,7 +195,8 @@ async def _stream(monkeypatch, model, source): json={"instruction": "上周订单金额多少", "intent": "answer", "datasource_ids": [source]}) as r: async for line in r.aiter_lines(): - if line.startswith("data: "): + # 末尾那条 usage 是这一轮助手的用量(answer_ops),不是生成的操作 + if line.startswith("data: ") and not line.startswith('data: {"op": "usage"'): events.append(json.loads(line[6:])) return events diff --git a/backend/tests/test_copilot_truncation.py b/backend/tests/test_copilot_truncation.py index c81a73e0..b3f0733c 100644 --- a/backend/tests/test_copilot_truncation.py +++ b/backend/tests/test_copilot_truncation.py @@ -87,7 +87,8 @@ async def _events(client, monkeypatch, model, base: dict | None = BASE) -> list[ async with client.stream("POST", "/api/copilot/generate-stream", json={"instruction": "并行算同比和环比", "base_graph": base}) as r: async for line in r.aiter_lines(): - if line.startswith("data: "): + # 末尾那条 usage 是这一轮助手的用量(answer_ops),不是生成的操作 + if line.startswith("data: ") and not line.startswith('data: {"op": "usage"'): out.append(json.loads(line[6:])) return out @@ -263,7 +264,8 @@ async def test_a_rejected_budget_falls_back_to_the_old_one(client, monkeypatch, async with client.stream("POST", "/api/copilot/generate-stream", json={"instruction": "加一支同比", "base_graph": BASE}) as r: async for line in r.aiter_lines(): - if line.startswith("data: "): + # 末尾那条 usage 是这一轮助手的用量(answer_ops),不是生成的操作 + if line.startswith("data: ") and not line.startswith('data: {"op": "usage"'): events.append(json.loads(line[6:])) assert 8192 in asked assert big.calls == 1 and small.calls == 1 @@ -284,7 +286,8 @@ async def test_other_rejections_are_not_retried(client, monkeypatch): async with client.stream("POST", "/api/copilot/generate-stream", json={"instruction": "改一下", "base_graph": BASE}) as r: async for line in r.aiter_lines(): - if line.startswith("data: "): + # 末尾那条 usage 是这一轮助手的用量(answer_ops),不是生成的操作 + if line.startswith("data: ") and not line.startswith('data: {"op": "usage"'): events.append(json.loads(line[6:])) assert small.calls == 0 assert events[-1]["op"] == "error" diff --git a/backend/tests/test_csv_grid.py b/backend/tests/test_csv_grid.py new file mode 100644 index 00000000..dca02df7 --- /dev/null +++ b/backend/tests/test_csv_grid.py @@ -0,0 +1,149 @@ +"""CSV 走配方(Excel 期 5 第 4 项):CSV 适配成网格,后面的执行器、核对、溯源全部复用。 + +格子坐标沿用「工作表!A1」(见 csv_grid 的模块说明):CSV 没有工作表,名字固定是「CSV」,每月的文件名变了也认得出。 +""" +from __future__ import annotations + +import pytest + +from app.data import csv_grid, recipe_engine, xlsx_cells +from app.data.recipe_types import Recipe +from app.data.xlsx_scan import SheetScan, UnsupportedTable, scan_from_json, scan_to_json + +CSV = ("订单号,下单日期,门店,金额,备注\n" + "A001,2026-08-01,一号店,120,\n" + "A002,2026-08-02,二号店,80.5,\"带,逗号\"\n" + "\n" + "A003,2026-08-03,一号店,1,234,多出的一格\n").encode("utf-8") + + +def test_text_is_sniffed_as_csv_and_binary_is_not(): + assert csv_grid.looks_like_csv(CSV) + assert csv_grid.looks_like_csv("a;b\n1;2\n".encode("gb18030")) + assert csv_grid.looks_like_csv(b"\xff\xfe" + "a\tb\n".encode("utf-16-le")) + assert not csv_grid.looks_like_csv(b"PK\x03\x04rest-of-zip") + assert not csv_grid.looks_like_csv(b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1" + b"\x00" * 100) # 旧版 .xls + assert not csv_grid.looks_like_csv(b"\x89PNG\r\n\x1a\n\x00\x00") + assert not csv_grid.looks_like_csv(b"") + + +def test_scan_gives_one_visible_sheet_with_bounds_and_row_counts(): + scan = csv_grid.scan(CSV) + assert scan.format == "csv" and len(scan.sheets) == 1 + sh = scan.sheets[0] + assert sh.name == csv_grid.CSV_SHEET and sh.state == "visible" and sh.path == csv_grid.CSV_PATH + "," + assert (sh.bounds.min_row, sh.bounds.min_col, sh.bounds.max_row, sh.bounds.max_col) == (1, 1, 5, 6) + # 空字段不算非空格;第 4 行是空行 + assert sh.row_runs == [(1, 1, 5), (2, 2, 4), (3, 3, 5), (5, 5, 6)] + assert sh.nonempty == 20 + again = scan_from_json(scan_to_json(scan)) + assert again.format == "csv" and again.sheets[0].path == csv_grid.CSV_PATH + "," + + +def test_cell_values_are_typed_only_when_unambiguous(): + """整数、小数写成数;千分位、前导零、正号、科学计数法、日期都留作文字,交给配方的规则(text_number、类型)判。""" + v = csv_grid.cell_value + assert v("120") == 120 and v(" 80.5 ") == 80.5 and v("-3") == -3 and v("0") == 0 + for text in ("1,234", "007", "+5", "1e5", "2026-08-01", "12%", "TRUE", "=SUM(A1:A2)", "1234567890123456"): + assert v(text) == text, text + assert v("") is None and v(" ") is None + + +def test_grid_and_rows_read_the_same_cells(): + scan = csv_grid.scan(CSV) + grid = xlsx_cells.read_grid(CSV, scan, csv_grid.CSV_SHEET) + assert grid.get(2, 4).value == 120 and grid.get(3, 4).value == 80.5 and grid.get(3, 5).value == "带,逗号" + assert grid.get(2, 5) is None and grid.get(1, 1).value == "订单号" + assert grid.get(5, 4).value == 1 and grid.get(5, 5).value == 234 # 没加引号的千分位被逗号拆开了:照实读 + rows = list(xlsx_cells.iter_rows(CSV, scan, csv_grid.CSV_SHEET, min_row=1, max_row=5, min_col=1, max_col=6)) + assert [r for r, _ in rows] == [1, 2, 3, 5] + assert sum(len(cells) for _, cells in rows) == len(grid.cells) == scan.sheets[0].nonempty + small = xlsx_cells.read_grid(CSV, scan, csv_grid.CSV_SHEET, max_rows=2) + assert small.truncated and set(r for r, _ in small.cells) == {1, 2} + + +def test_semicolon_and_tab_delimiters_and_gbk(): + scan = csv_grid.scan("名称;数量\n甲;3\n".encode("gb18030")) + grid = xlsx_cells.read_grid("名称;数量\n甲;3\n".encode("gb18030"), scan, csv_grid.CSV_SHEET) + assert grid.get(2, 1).value == "甲" and grid.get(2, 2).value == 3 + raw = "a\tb\n1\t2\n".encode("utf-8") + assert xlsx_cells.read_grid(raw, csv_grid.scan(raw), csv_grid.CSV_SHEET).get(2, 2).value == 2 + + +def test_scan_any_dispatches_and_keeps_the_xlsx_refusal(): + assert csv_grid.scan_any(CSV, "orders.csv").format == "csv" + assert csv_grid.scan_any(CSV, "ORDERS.CSV").format == "csv" + with pytest.raises(UnsupportedTable): + csv_grid.scan_any(b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1" + b"\x00" * 100, "old.xls") + + +def test_only_csv_and_tsv_names_are_read_as_csv(): + """复核反例:以前只看内容,不是 zip / OLE、前 64KB 没有 NUL 就当 CSV 读。系统导出的「HTML 表格冒充 .xls」、 + 误传的文本文件,都被拆成一张叫「CSV」的工作表去起草。现在和简单导入(tabular)一样按扩展名定:只有 .csv / .tsv + 按 CSV 读;别的照旧交给 xlsx_scan 拒收。""" + with pytest.raises(UnsupportedTable) as e: + csv_grid.scan_any(CSV, "orders.txt") + assert "不是有效的 .xlsx" in str(e.value) + html = "
订单号金额
".encode("utf-8") + for raw in (html, b"\xef\xbb\xbf \r\n" + html, b"\xff\xfe" + "
a
".encode("utf-16-le"), + b''): + with pytest.raises(UnsupportedTable) as e: + csv_grid.scan_any(raw, "report.xls") + assert "网页或 XML" in str(e.value) and "另存为 .xlsx 或 .csv" in str(e.value) + # 扩展名是 .csv、内容却不是文字(图片之类改了扩展名):按 CSV 读不了,照实说 + with pytest.raises(UnsupportedTable) as e: + csv_grid.scan_any(b"\x89PNG\r\n\x1a\n\x00\x00", "orders.csv") + assert "不是文字" in str(e.value) + + +def test_tsv_is_always_tab_separated_and_the_delimiter_is_kept_for_reading(): + """.tsv 一律按制表符分隔(同 tabular):每行逗号个数一样的 TSV,嗅探会挑逗号(csv.Sniffer 优先逗号)。 + 扫描时定下的分隔符记在工作表的 path 里,之后读格子(重放、试运行)用同一个,不再各嗅各的。""" + raw = "名称,规格\t金额\n甲,1kg\t10\n乙,2kg\t20\n".encode("utf-8") # 每行一个逗号、一个制表符 + scan = csv_grid.scan_any(raw, "goods.tsv") + assert scan.sheets[0].path == csv_grid.CSV_PATH + "\t" + grid = xlsx_cells.read_grid(raw, scan_from_json(scan_to_json(scan)), csv_grid.CSV_SHEET) + assert grid.get(2, 1).value == "甲,1kg" and grid.get(2, 2).value == 10 + # 这一版之前的暂存区:path 只有「csv:」,读的时候再嗅 + assert csv_grid.is_csv_sheet(SheetScan(name="CSV", state="visible", path=csv_grid.CSV_PATH)) + assert csv_grid.sheet_delimiter(SheetScan(name="CSV", state="visible", path=csv_grid.CSV_PATH)) is None + assert not csv_grid.is_csv_sheet(SheetScan(name="明细", state="visible", path="xl/worksheets/sheet1.xml")) + sniffed = csv_grid.scan_any(raw, "goods.csv") + assert sniffed.sheets[0].path == csv_grid.CSV_PATH + "," # 同样的内容叫 .csv:照嗅探的来(逗号) + grid = xlsx_cells.read_grid(raw, sniffed, csv_grid.CSV_SHEET) + assert grid.get(2, 1).value == "甲" and grid.get(2, 2).value == "1kg\t10" + + +def test_too_many_columns_is_refused(): + raw = (",".join(["x"] * (csv_grid.MAX_COLS + 1)) + "\n").encode() + with pytest.raises(UnsupportedTable) as e: + csv_grid.scan(raw) + assert "列" in str(e.value) + + +LIST_RECIPE = { + "recipe_format": "agentlab-recipe/2", + "sheets": [{"id": "s1", "match": {"name": "CSV"}, + "blocks": [{"id": "列表1", "layout": "list", "table": "订单", + "columns": [{"header": "订单号", "name": "订单号", "type": "TEXT"}, + {"header": "下单日期", "name": "下单日期", "type": "DATE"}, + {"header": "门店", "name": "门店", "type": "TEXT"}, + {"header": "金额", "name": "金额", "type": "REAL"}], + "rows": {"blank_rows": "stop"}}]}], + "tables": [{"name": "订单", "grain": ["订单号"]}], +} + + +def test_engine_runs_a_list_recipe_on_csv(tmp_path): + raw = "订单号,下单日期,门店,金额\nA001,2026-08-01,一号店,120\nA002,2026-08-02,二号店,80.5\n".encode() + scan = csv_grid.scan(raw) + db = tmp_path / "t.sqlite" + ex = recipe_engine.execute(Recipe.model_validate(LIST_RECIPE), raw, "orders_2026-08.csv", scan, str(db)) + assert ex.ok, [p.message for p in ex.problems] + assert ex.tables[0].rows == 2 and ex.tables[0].sheet == "CSV" + # 溯源照常:订单号这一列从 CSV!A2 起往下 2 格 + run = ex.lineage["订单"]["订单号"][0] + assert run[1:] == ["CSV", "A2", 2, "down"] + import sqlite3 + rows = sqlite3.connect(db).execute('SELECT "订单号", "下单日期", "金额" FROM "订单" ORDER BY rowid').fetchall() + assert rows == [("A001", "2026-08-01", 120.0), ("A002", "2026-08-02", 80.5)] diff --git a/backend/tests/test_directory_candidates.py b/backend/tests/test_directory_candidates.py new file mode 100644 index 00000000..c8fe3a36 --- /dev/null +++ b/backend/tests/test_directory_candidates.py @@ -0,0 +1,266 @@ +"""使用者选人:按用途给的候选池 + 开登录时服务端核对(docs/design/next-round/05-已知遗留.md (b))。 + +以前使用者在入口表单、待办表单的选人控件里没有候选(名录只给搭建者),服务端也不核对:填错的名字会把待办派给一个不存在 +的人,没人能处理,也不报错。现在: +- GET /api/directory/candidates 按用途给候选池,只有用户名和显示名,停用的不列;form 用途缺省是能用这个应用的人(使用 + 及以上的授权展开组成员,加上管理员和建应用的人),字段可以用 x-principal 声明别的池;应用授权给了所有人时不带前缀不列; + 调用的人不能用这个应用时 403;review 用途是启用的搭建者和管理员; +- 开了登录时,入口表单和待办表单的选人控件按同一个池核对,不在池里 422(errors 的 code=principal);关登录照旧手填; +- /api/directory 仍然只给搭建者。 + +名字都是虚构的。 +""" +from __future__ import annotations + +import asyncio +import uuid + +import pytest +from sqlalchemy import delete, select + +from app.apps.forms import validate_form_schema +from app.core.artifact_store import graph_hash +from app.db.base import SessionLocal, utcnow +from app.db.models import App, AppGrant, Group, GroupMember, Run, Task, User, Workflow, WorkflowVersion +from app.engine.runner import run_manager +from tests.fakes import auth_env + + +@pytest.fixture(autouse=True) +async def engine_up(): + await run_manager.setup() + yield + await run_manager.shutdown() + + +def node(nid, ntype, label=None, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": label or nid, "config": config}} + + +def chain(*nodes): + return {"nodes": list(nodes), "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]} + + +PICK = {"type": "string", "title": "审批人", "x-widget": "principal"} + +GRAPH = chain( + node("start", "input", "开始", fields=[{"name": "approver"}, {"name": "reviewer"}]), + node("review", "human", "复核", mode="input", title="请指定复核人", + form={"type": "object", "properties": {"reviewer": {**PICK, "title": "复核人"}}}), + node("out", "output", "成果", fields=[{"name": "r", "value": "{{ input.approver }}"}])) + + +async def _workflow() -> tuple[str, str]: + async with SessionLocal() as session: + wf = Workflow(name=f"选人-{uuid.uuid4().hex[:6]}", graph=GRAPH, status="published", published_version=1, + version=1) + session.add(wf) + await session.flush() + session.add(WorkflowVersion(workflow_id=wf.id, version=1, graph=GRAPH, graph_hash=graph_hash(GRAPH), + level="published", published_at=utcnow())) + await session.commit() + return wf.id, graph_hash(GRAPH) + + +def _spec(wf: str, hash_: str, approver: dict | None = None) -> dict: + return { + "format": "agentlab.app/1", "name": "选人", + "pins": [{"alias": "main", "workflow_id": wf, "version": 1, "graph_hash": hash_, "use": "entry"}], + "entries": [{"id": "apply", "label": "发起", "pin": "main", "form": {"$ref": "#/forms/apply"}, + "map": {"approver": "form.approver"}}], + "forms": {"apply": {"type": "object", "properties": {"approver": approver or PICK}}}, + "pages": [{"id": "home", "title": "首页", "body": [ + {"id": "new", "type": "entry_button", "props": {"entry": "apply"}}]}], + } + + +async def _new_app(root, spec: dict) -> str: + """root 建一个应用并发布,授权使用给 uma、停用的 dan 和组「财务」。""" + res = await root.post("/api/apps", json={"name": f"选人-{uuid.uuid4().hex[:4]}", "spec": spec}) + assert res.status_code == 201, res.text + app_id = res.json()["id"] + published = (await root.post(f"/api/apps/{app_id}/publish", json={})).json() + assert published["ok"], published + async with SessionLocal() as session: + session.add_all([AppGrant(app_id=app_id, principal_kind="user", principal="uma", role="use"), + AppGrant(app_id=app_id, principal_kind="user", principal="dan", role="use"), + AppGrant(app_id=app_id, principal_kind="group", principal="财务", role="use")]) + await session.commit() + return app_id + + +@pytest.fixture +async def world(): + """开账号密码登录:管理员 root(建应用)、搭建者 bob、使用者 uma(授权使用)、ulf(没授权)、停用的 dan(授权使用); + 组「财务」(fay、fin),整组授权使用。""" + await auth_env.reset() + root = await auth_env.turn_on_local("root") + try: + await auth_env.make_user("bob", "builder") + await auth_env.make_user("uma", "user") + await auth_env.make_user("ulf", "user") + await auth_env.make_user("dan", "user", disabled=True) + fay = await auth_env.make_user("fay", "user") + fin = await auth_env.make_user("fin", "user") + async with SessionLocal() as session: + for username, shown in (("uma", "乌玛"), ("fay", "费伊"), ("bob", "鲍勃")): + user = (await session.execute(select(User).where(User.username == username))).scalar_one() + user.display_name = shown + group = Group(name="财务", label="财务组", source="local") + session.add(group) + await session.flush() + session.add_all([GroupMember(group_id=group.id, user_id=fay.id), GroupMember(group_id=group.id, user_id=fin.id)]) + await session.commit() + wf, hash_ = await _workflow() + app_id = await _new_app(root, _spec(wf, hash_)) + yield {"root": root, "app_id": app_id, "wf": wf, "hash": hash_} + finally: + await root.aclose() + async with SessionLocal() as session: + for model in (AppGrant, App): + await session.execute(delete(model)) + await session.commit() + await auth_env.reset() + + +def _params(app_id: str, **more) -> dict: + return {"purpose": "form", "app_id": app_id, "entry": "apply", "field": "approver", **more} + + +async def test_users_get_the_people_who_can_use_the_app_and_only_their_names(world): + uma = await auth_env.login("uma") + r = await uma.get("/api/directory/candidates", params=_params(world["app_id"])) + assert r.status_code == 200, r.text + body = r.json() + assert [u["username"] for u in body["users"]] == ["fay", "fin", "root", "uma"] + assert all(set(u) == {"username", "display_name"} for u in body["users"]), "不给停用、角色、邮箱" + assert {u["username"]: u["display_name"] for u in body["users"]}["uma"] == "乌玛" + assert body["groups"] == [{"name": "财务", "label": "财务组"}] and body["truncated"] is False + # 前缀按用户名或显示名筛 + assert [u["username"] for u in (await uma.get("/api/directory/candidates", + params=_params(world["app_id"], q="费"))).json()["users"]] == ["fay"] + # 名录整份仍然只给搭建者 + assert (await uma.get("/api/directory")).status_code == 403 + # 不能用这个应用的人拿不到它的候选 + ulf = await auth_env.login("ulf") + assert (await ulf.get("/api/directory/candidates", params=_params(world["app_id"]))).status_code == 403 + for c in (uma, ulf): + await c.aclose() + + +async def test_an_app_granted_to_everyone_needs_a_prefix(world): + async with SessionLocal() as session: + session.add(AppGrant(app_id=world["app_id"], principal_kind="everyone", principal="", role="use")) + await session.commit() + ulf = await auth_env.login("ulf") + body = (await ulf.get("/api/directory/candidates", params=_params(world["app_id"]))).json() + assert body["users"] == [] and body["needs_query"] is True + found = (await ulf.get("/api/directory/candidates", params=_params(world["app_id"], q="u"))).json() + assert [u["username"] for u in found["users"]] == ["ulf", "uma"], "停用的 dan 不列" + capped = (await ulf.get("/api/directory/candidates", params=_params(world["app_id"], q="f", limit=1))).json() + assert len(capped["users"]) == 1 and capped["truncated"] is True + await ulf.aclose() + + +async def test_review_candidates_are_builders_and_admins(world): + uma = await auth_env.login("uma") + body = (await uma.get("/api/directory/candidates", params={"purpose": "review"})).json() + assert [u["username"] for u in body["users"]] == ["bob", "root"] and body["groups"] == [] + assert (await uma.get("/api/directory/candidates", params={"purpose": "nope"})).status_code == 422 + await uma.aclose() + + +async def test_starting_with_a_name_outside_the_pool_is_refused(world): + uma = await auth_env.login("uma") + url = f"/api/apps/{world['app_id']}/entries/apply" + bad = await uma.post(url, json={"values": {"approver": "wangwu"}}) + assert bad.status_code == 422, bad.text + [error] = bad.json()["errors"] + assert (error["path"], error["code"]) == ("approver", "principal") + assert error["message"].startswith("「wangwu」不在可选的人里") + disabled = await uma.post(url, json={"values": {"approver": "dan"}}) + assert disabled.status_code == 422 and disabled.json()["errors"][0]["code"] == "principal" + for ok in ("fay", "财务"): + good = await uma.post(url, json={"values": {"approver": ok}}) + assert good.status_code == 201, good.text + await uma.aclose() + + +async def test_a_task_form_is_checked_against_the_same_pool(world): + uma = await auth_env.login("uma") + started = await uma.post(f"/api/apps/{world['app_id']}/entries/apply", json={"values": {"approver": "fay"}}) + assert started.status_code == 201, started.text + run_id = started.json()["run_id"] + task = None + for _ in range(200): + async with SessionLocal() as session: + task = (await session.execute(select(Task).where(Task.run_id == run_id, Task.status == "pending")))\ + .scalars().first() + if task is not None: + break + await asyncio.sleep(0.05) + assert task is not None + cands = await uma.get("/api/directory/candidates", params={"purpose": "form", "task_id": task.id, + "field": "reviewer"}) + assert cands.status_code == 200 and "fin" in [u["username"] for u in cands.json()["users"]] + root = world["root"] + bad = await root.post(f"/api/tasks/{task.id}/respond", json={"decision": "submit", "values": {"reviewer": "wangwu"}}) + assert bad.status_code == 422, bad.text + assert bad.json()["errors"][0]["code"] == "principal" + good = await root.post(f"/api/tasks/{task.id}/respond", json={"decision": "submit", "values": {"reviewer": "fin"}}) + assert good.status_code == 200, good.text + await uma.aclose() + + +async def test_a_declared_pool_narrows_the_candidates(world): + """x-principal 声明了 group:财务:候选只有组里的人,提交组外的人也拒。""" + app_id = await _new_app(world["root"], _spec(world["wf"], world["hash"], + {**PICK, "x-principal": {"pool": "group:财务"}})) + uma = await auth_env.login("uma") + body = (await uma.get("/api/directory/candidates", params=_params(app_id))).json() + assert [u["username"] for u in body["users"]] == ["fay", "fin"] and body["groups"] == [] + refused = await uma.post(f"/api/apps/{app_id}/entries/apply", json={"values": {"approver": "uma"}}) + assert refused.status_code == 422 and refused.json()["errors"][0]["code"] == "principal" + await uma.aclose() + + +async def test_login_off_takes_any_name(): + await auth_env.reset() + wf, hash_ = await _workflow() + from httpx import ASGITransport, AsyncClient + + from app.main import app as asgi + + async with AsyncClient(transport=ASGITransport(app=asgi), base_url="http://test") as c: + res = await c.post("/api/apps", json={"name": f"关登录-{uuid.uuid4().hex[:4]}", "spec": _spec(wf, hash_)}) + app_id = res.json()["id"] + assert (await c.post(f"/api/apps/{app_id}/publish", json={})).json()["ok"] + started = await c.post(f"/api/apps/{app_id}/entries/apply", json={"values": {"approver": "随手填的名字"}}) + assert started.status_code == 201, started.text + async with SessionLocal() as session: + await session.execute(delete(Run).where(Run.id == started.json()["run_id"])) + await session.execute(delete(App).where(App.id == app_id)) + await session.commit() + + +def test_x_principal_is_checked_in_the_form_schema(): + ok = {"type": "object", "properties": {"a": {**PICK, "x-principal": {"pool": "slot:finance"}}}} + assert validate_form_schema(ok) == [] + bad = {"type": "object", "properties": { + "a": {**PICK, "x-principal": {"pool": "everyone"}}, + "b": {"type": "string", "x-principal": {"pool": "app"}}, + "c": {**PICK, "x-principal": "app"}}} + problems = validate_form_schema(bad) + assert len(problems) == 3, problems + assert any("「b」不是选人控件" in p for p in problems) + + +async def test_users_table_is_untouched_by_candidates(world): + """候选接口只读:不建用户、不改名录。""" + async with SessionLocal() as session: + before = len((await session.execute(select(User.id))).all()) + uma = await auth_env.login("uma") + await uma.get("/api/directory/candidates", params=_params(world["app_id"], q="zz")) + await uma.aclose() + async with SessionLocal() as session: + assert len((await session.execute(select(User.id))).all()) == before diff --git a/backend/tests/test_dual_check_node.py b/backend/tests/test_dual_check_node.py index 436cd9bc..3ca9af15 100644 --- a/backend/tests/test_dual_check_node.py +++ b/backend/tests/test_dual_check_node.py @@ -519,15 +519,19 @@ async def test_a_missing_choice_is_recorded_as_no_choice(monkeypatch, shop, clie assert not logs(evs, "dual_check_catalog") and await notes_of(source_id) is None -async def test_rejecting_the_approval_fails_the_node(monkeypatch, shop, client): +async def test_rejecting_the_approval_ends_the_run_as_rejected(monkeypatch, shop, client): + """明确驳回裁决:人的决定,不是故障(05-已知遗留 (a))。运行收成 cancelled + end_reason=rejected,原话照旧。""" name, _ = shop Blind(monkeypatch, name, {"销售额": (RIGHT, "gmv")}) run = await start(graph(name, BY_ORDER_DATE)) approval = await pending(run.id) res = await client.post(f"/api/approvals/{approval.id}/decide", json={"approved": False, "note": "口径要重新对"}) assert res.status_code == 200, res.text - done = await wait(run.id, ("succeeded", "failed")) - assert done.status == "failed" and "人工驳回" in (done.error or "") and "口径要重新对" in done.error + done = await wait(run.id, ("succeeded", "failed", "cancelled")) + assert (done.status, done.end_reason) == ("cancelled", "rejected"), done.error + assert "人工驳回" in (done.error or "") and "口径要重新对" in done.error + [ended] = [e for e in await events(run.id) if e.type == "run.cancelled"] + assert (ended.data["kind"], ended.data["note"]) == ("dual_check", "口径要重新对") async def test_on_reject_fail_stops_the_run(monkeypatch, shop, client): diff --git a/backend/tests/test_eval_origin.py b/backend/tests/test_eval_origin.py new file mode 100644 index 00000000..951b12be --- /dev/null +++ b/backend/tests/test_eval_origin.py @@ -0,0 +1,232 @@ +"""系统派生的运行不进使用统计(docs/design/next-round/00-总方案.md X1,细案 01 做法 4)。 + +评测、口径对账、口径召回(engine/origin.SYSTEM_ORIGINS)是服务端替人跑的完整运行,证据点得开,但不是有人在用。这里守: + +- 客户端声明不了这几种来源;ledger_issue(替用户正式出具)照常计入; +- counted_runs() 只去掉这三种,来源为空的老运行照常计入(防 NULL); +- 探索问题聚类只取问题原文(不含 history),问数据一轮一条(自动重来不多算),评测运行不进; +- 工具白名单衰减、目录的使用次数和「被哪些运行用过」、审阅队列的使用权重都不计评测运行; +- 记录页默认不列这三种,按来源点名就列;GET /api/runs/unlisted 照实说没列出几次。 + +表名、问题、工作流名都是虚构的。 +""" +from __future__ import annotations + +import uuid +from datetime import datetime, timedelta, timezone + +import pytest +from httpx import ASGITransport, AsyncClient +from sqlalchemy import delete, select + +from app.data import catalog, catalog_review +from app.db.base import SessionLocal +from app.db.models import Artifact, Conversation, ConversationTurn, DataSource, Run, RunEvent, Workflow +from app.engine.origin import CLIENT_ORIGINS, ORIGIN_KINDS, SYSTEM_ORIGINS, counted_runs +from app.main import app +from tests.fixtures.sources import drop_source + +BASE = datetime(2026, 9, 1, tzinfo=timezone.utc) +MARK = f"计入统计-{uuid.uuid4().hex[:6]}" + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +@pytest.fixture(autouse=True) +async def _clean(): + yield + async with SessionLocal() as session: + await session.execute(delete(Run).where(Run.workflow_name.like(f"{MARK}%"))) + await session.execute(delete(Conversation).where(Conversation.title.like(f"{MARK}%"))) + await session.commit() + + +async def _run(origin: str | None, *, minutes: int = 0, **extra) -> str: + rid = uuid.uuid4().hex + async with SessionLocal() as session: + session.add(Run(id=rid, graph={}, workflow_name=f"{MARK}-{origin}", run_class="exploratory", status="succeeded", + origin_kind=origin, created_at=BASE + timedelta(minutes=minutes), **extra)) + await session.commit() + return rid + + +def test_system_origins_are_server_only_and_ledger_issue_counts(): + assert set(SYSTEM_ORIGINS) == {"eval", "reconcile", "recall"} + assert set(SYSTEM_ORIGINS) <= set(ORIGIN_KINDS) + assert not set(SYSTEM_ORIGINS) & set(CLIENT_ORIGINS) + assert "ledger_issue" in ORIGIN_KINDS and "ledger_issue" not in SYSTEM_ORIGINS, "替用户正式出具照常计入" + + +async def test_a_client_cannot_claim_a_system_origin(client): + for origin in (*SYSTEM_ORIGINS, "ledger_issue"): + r = await client.post("/api/runs", json={"graph": {"nodes": [], "edges": []}, "origin": origin}) + assert r.status_code == 422 and r.json()["code"] == "origin_not_allowed", (origin, r.text) + + +async def test_counted_runs_keeps_legacy_null_origins(): + ids = {origin: await _run(origin) for origin in (None, "chat", "ledger_issue", *SYSTEM_ORIGINS)} + async with SessionLocal() as session: + counted = set((await session.execute( + select(Run.id).where(Run.id.in_(list(ids.values())), counted_runs()))).scalars()) + assert counted == {ids[None], ids["chat"], ids["ledger_issue"]} + + +# -------------------------------------------------------------------------- +# 探索问题聚类 +# -------------------------------------------------------------------------- + + +async def _turn(conversation_id: str, question: str, run_id: str | None, seq: int, minutes: int) -> str: + async with SessionLocal() as session: + turn = ConversationTurn(conversation_id=conversation_id, seq=seq, question=question, run_id=run_id, + status="done", created_at=BASE + timedelta(minutes=minutes)) + session.add(turn) + await session.commit() + return turn.id + + +async def _conversation() -> str: + async with SessionLocal() as session: + row = Conversation(title=f"{MARK}-对话", kind="chat") + session.add(row) + await session.commit() + return row.id + + +async def test_clusters_take_only_the_question_once_per_turn_and_skip_eval(client, monkeypatch): + from app.memory import embeddings + + question = f"{MARK} 上周各门店的销售额" + + async def by_text(texts): + # 同一句话同一个向量,别的话各不相同:聚类只看取了什么文字 + return [[1.0, 0.0] if t == question else [0.0, float(i + 1)] for i, t in enumerate(texts)] + + monkeypatch.setattr(embeddings, "embed_texts", by_text) + conv = await _conversation() + # 第一轮:答坏了自动重来一次——两次运行,轮次只挂着后一次;输入里的 history 各不相同也不影响 + first = await _run("chat", minutes=1, input={"question": question, "history": "问:别的\n答:别的"}) + retried = await _run("chat", minutes=2, input={"question": question, "history": "问:别的\n答:别的"}) + turn_a = await _turn(conv, question, retried, 1, 2) + # 第二轮:同一个问题又问了一次,这次只回了一句、没有运行 + turn_b = await _turn(conv, question, None, 2, 3) + # 评测作答的运行:同一个问题,不进聚类 + graded = await _run("eval", minutes=4, input={"question": question}) + # 画布试跑:输入里有 question 的照样收 + canvas = await _run("canvas", minutes=5, input={"question": question}) + + body = (await client.get("/api/governance/exploratory-clusters", params={"limit": 500})).json() + mine = [c for c in body["clusters"] if c["representative"] == question] + assert len(mine) == 1, body + cluster = mine[0] + assert cluster["count"] == 3, "一轮一条(自动重来不多算)+ 画布那一次;评测不算" + assert set(cluster["turn_ids"]) == {turn_a, turn_b} + assert set(cluster["run_ids"]) == {retried, canvas} + assert first not in cluster["run_ids"] and graded not in cluster["run_ids"] + assert cluster["promote_candidate"] is True + assert body["embedder"] + + +async def test_the_history_never_reaches_the_clusterer(monkeypatch): + from app.api.deps import current_principal + from app.chat.questions import recent_questions + + conv = await _conversation() + rid = await _run("chat", minutes=10, input={"question": f"{MARK} 本月退款", "history": "问:上周\n答:100"}) + await _turn(conv, f"{MARK} 本月退款", rid, 1, 10) + async with SessionLocal() as session: + got = [q for q in await recent_questions(session, current_principal(), limit=500) if MARK in q.text] + assert [q.text for q in got] == [f"{MARK} 本月退款"] + + +# -------------------------------------------------------------------------- +# 工具白名单衰减 +# -------------------------------------------------------------------------- + + +async def test_tool_usage_ignores_eval_runs(client): + graph = {"nodes": [{"id": "a", "type": "agent", "position": {"x": 0, "y": 0}, + "data": {"label": "a", "config": {"tools": ["calc", "kb_search"]}}}], "edges": []} + async with SessionLocal() as session: + wf = Workflow(name=f"{MARK}-白名单", graph=graph) + session.add(wf) + await session.commit() + wf_id = wf.id + try: + used = await _run("canvas", workflow_id=wf_id) + graded = await _run("eval", workflow_id=wf_id) + async with SessionLocal() as session: + session.add(RunEvent(run_id=used, seq=1, type="tool.start", node_id="a", data={"tool": "calc"}, ts=0.0)) + session.add(RunEvent(run_id=graded, seq=1, type="tool.start", node_id="a", data={"tool": "kb_search"}, ts=0.0)) + await session.commit() + body = (await client.get("/api/governance/tool-usage", params={"workflow_id": wf_id})).json() + assert body["runs_considered"] == 1 + node = body["nodes"][0] + assert node["used"] == {"calc": 1} and node["unused"] == ["kb_search"], "评测用过的工具不能让白名单显得都在用" + finally: + async with SessionLocal() as session: + await session.execute(delete(Workflow).where(Workflow.id == wf_id)) + await session.commit() + + +# -------------------------------------------------------------------------- +# 目录的使用次数、被哪些运行用过、审阅队列的权重 +# -------------------------------------------------------------------------- + + +async def test_catalog_usage_ignores_eval_runs(): + name = f"eo_{uuid.uuid4().hex[:8]}" + tables = {"orders": {"qualified": "orders", "primary_key": ["id"], "comment": None, "is_view": False, + "foreign_keys": [], "unique": [], + "columns": [{"name": "id", "type": "INTEGER", "nullable": True, "comment": None}]}} + async with SessionLocal() as session: + row = DataSource(name=name, kind="sqlite", database=":memory:", schema_cache={"tables": tables}) + session.add(row) + await session.commit() + sid = row.id + try: + used = await _run("chat") + graded = await _run("eval") + legacy = await _run(None) + async with SessionLocal() as session: + for rid in (used, graded, legacy): + session.add(Artifact(id=uuid.uuid4().hex, run_id=rid, kind=catalog.QUERY_SNAPSHOT_KIND, + meta={"source_id": sid, "source": name, "tables": ["orders"]})) + await session.commit() + source = await session.get(DataSource, sid) + assert await catalog.table_usage(session, source) == {"orders": 2} + counts = await catalog_review.table_usage_counts(session, source) + assert counts.queries == {"orders": 2} and counts.runs == {"orders": 2} + listed = await catalog_review.table_runs(session, source, "orders") + assert listed["total"] == 2 and {r["run_id"] for r in listed["runs"]} == {used, legacy} + finally: + await drop_source(sid) + + +# -------------------------------------------------------------------------- +# 记录页 +# -------------------------------------------------------------------------- + + +async def test_the_run_list_hides_system_runs_by_default_and_says_how_many(client): + chat = await _run("chat") + graded = [await _run("eval", minutes=i) for i in range(2)] + recon = await _run("reconcile") + legacy = await _run(None) + q = {"q": MARK, "limit": 200} + + listed = {r["id"] for r in (await client.get("/api/runs", params=q)).json()} + assert {chat, legacy} <= listed and not (set(graded) | {recon}) & listed + unlisted = (await client.get("/api/runs/unlisted", params={"q": MARK})).json() + assert unlisted == {"total": 3, "by_origin": {"eval": 2, "reconcile": 1}} + + # 按来源点名就列,此时没有藏起来的 + only = {r["id"] for r in (await client.get("/api/runs", params={**q, "origin_kind": "eval"})).json()} + assert only == set(graded) + assert (await client.get("/api/runs/unlisted", params={"q": MARK, "origin_kind": "eval"})).json()["total"] == 0 + # 同样的筛选一起作用:按状态筛时只数这个状态的 + assert (await client.get("/api/runs/unlisted", params={"q": MARK, "status": "failed"})).json()["total"] == 0 diff --git a/backend/tests/test_integration_errors.py b/backend/tests/test_integration_errors.py new file mode 100644 index 00000000..f3e39b6f --- /dev/null +++ b/backend/tests/test_integration_errors.py @@ -0,0 +1,169 @@ +"""集成接口的错误码与 /docs(04 细案 A7)。 + +外部系统按 code 认是哪种错,不按中文句子:应用(api/apps.py)、待办(api/tasks.py)、工件(api/artifacts.py)的错误, +和角色、运行读权限的拒绝(api/deps.py、auth.require)全部带 code;detail 还是原来那一句,老前端照常显示。 + +开了登录,/docs、/redoc、/openapi.json 和 /api 一样要登录、要搭建者:以前中间件只拦 /api/ 开头的路径,接口全貌对 +匿名的人敞开。关登录照旧(tests/test_provenance_contract.py 的断言不受影响)。 +""" +from __future__ import annotations + +import ast +import uuid +from pathlib import Path + +import pytest +from httpx import ASGITransport, AsyncClient + +from app.api import coded +from app.db.base import SessionLocal +from app.db.models_apps import App +from app.main import app as main_app +from tests.fakes import auth_env + +APP_DIR = Path(__file__).resolve().parents[1] / "app" +DOCS = ("/docs", "/redoc", "/openapi.json") + + +@pytest.fixture(autouse=True) +async def clean(): + await auth_env.reset() + try: + yield + finally: + await auth_env.reset() + + +def client() -> AsyncClient: + return AsyncClient(transport=ASGITransport(app=main_app), base_url="http://test") + + +async def new_app() -> str: + async with SessionLocal() as session: + row = App(name=f"报销-{uuid.uuid4().hex[:6]}") + session.add(row) + await session.commit() + return row.id + + +def _called(node: ast.AST, name: str) -> bool: + return isinstance(node, ast.Call) and (getattr(node.func, "id", None) or getattr(node.func, "attr", None)) == name + + +def test_no_error_without_a_code_left_in_the_integration_modules(): + """这几个模块里不许再直接抛 HTTPException(没有 code),一律 CodedHTTPException;直接回 JSONResponse 的错误(表单 + 没通过校验那几处)内容里也要有 code。""" + for rel in ("api/apps.py", "api/tasks.py", "api/artifacts.py", "api/deps.py", "auth/__init__.py"): + tree = ast.parse((APP_DIR / rel).read_text()) + bare = [node.lineno for node in ast.walk(tree) if _called(node, "HTTPException")] + assert not bare, f"{rel} 第 {bare} 行抛的 HTTPException 没有 code" + for node in ast.walk(tree): + if not _called(node, "JSONResponse"): + continue + content = next((k.value for k in node.keywords if k.arg == "content"), node.args[0] if node.args else None) + keys = {k.value for k in getattr(content, "keys", []) if isinstance(k, ast.Constant)} + assert "code" in keys, f"{rel} 第 {node.lineno} 行回的 JSONResponse 没有 code" + + +def test_codes_agree_with_the_ones_already_in_use(): + from app.apps import entries + + assert coded.APP_NOT_FOUND == entries.APP_GONE, "入口发起时的「应用不存在」和这里同一个码" + + +async def test_app_errors_carry_codes_and_keep_their_detail(): + app_id = await new_app() + async with client() as c: + r = await c.get("/api/apps/nope-nope") + assert (r.status_code, r.json()) == (404, {"detail": "应用不存在,可能已被删除", "code": "app_not_found"}) + r = await c.get(f"/api/apps/{app_id}/versions/9") + assert r.status_code == 404 and r.json()["code"] == "app_version_not_found" and "没有 v9" in r.json()["detail"] + r = await c.get(f"/api/apps/{app_id}/pages/home", params={"params": "[1, 2]", "draft": 1}) + assert (r.status_code, r.json()) == (422, {"detail": "页面参数需要是 JSON 对象", "code": "app_params_invalid"}) + r = await c.patch(f"/api/apps/{app_id}", json={"draft_bindings": {"slots": {"财务": "不是列表"}}}) + assert r.status_code == 422 and r.json()["code"] == "app_bindings_invalid", r.text + r = await c.post(f"/api/apps/{app_id}/hooks/nope/rotate") + assert r.status_code == 404 and r.json()["code"] == "app_hook_not_declared" + r = await c.patch(f"/api/apps/{app_id}/hooks/nope", json={"enabled": False}) + assert r.status_code == 404 and r.json()["code"] == "app_hook_not_found", r.text + r = await c.post("/api/apps/derive", json={"workflow_id": "nope-nope", "version": 1}) + assert (r.status_code, r.json()["code"]) == (404, "workflow_not_found") + + +async def test_task_errors_carry_codes(): + async with client() as c: + r = await c.get("/api/tasks/nope-nope") + assert (r.status_code, r.json()) == (404, {"detail": "待办不存在,所属的运行可能已被删除", "code": "task_not_found"}) + # 服务层没带码的拒绝(这里是待办不存在)按状态补上同一类的码 + r = await c.post("/api/tasks/nope-nope/respond", json={"decision": "approve"}) + assert (r.status_code, r.json()["code"]) == (404, "task_not_found") + r = await c.post("/api/tasks/nope-nope/remind") + assert (r.status_code, r.json()["code"]) == (404, "task_not_found") + + +async def test_artifact_errors_carry_codes(): + async with client() as c: + r = await c.get(f"/api/artifacts/{'0' * 64}") + assert (r.status_code, r.json()) == (404, {"detail": "工件不存在,可能已被删除", "code": "artifact_not_found"}) + + +async def test_role_and_run_refusals_carry_codes(): + await auth_env.turn_on_local() + await auth_env.make_user("uma", "user") + uma = await auth_env.login("uma") + r = await uma.get("/api/workflows") + assert r.status_code == 403 and r.json() == {"detail": "没有权限:这项操作需要搭建者及以上的角色", + "code": "role_required"} + r = await uma.get("/api/tasks", params={"assigned": "all"}) + assert r.status_code == 403 and r.json()["code"] == "role_required" + r = await uma.get("/api/runs/nope-nope/evidence") + assert r.status_code == 403 and r.json() == {"detail": "没有权限查看这次运行", "code": "run_forbidden"} + r = await uma.get(f"/api/artifacts/{'0' * 64}") + assert r.status_code == 403 and r.json() == {"detail": "没有权限查看这件工件", "code": "artifact_forbidden"} + r = await uma.get("/api/runs") + assert r.status_code == 403 and r.json()["code"] == "role_required" + + +async def test_ensure_role_inside_an_endpoint_carries_codes(): + from app.api.deps import Principal, ensure_role, reset_principal, set_principal + + for principal, status, code in ((Principal(username=None, role="", mode="local"), 401, "auth_required"), + (Principal(username="bob", role="builder", verified=True, mode="local"), 403, + "role_required")): + token = set_principal(principal) + try: + with pytest.raises(coded.CodedHTTPException) as e: + ensure_role("admin") + finally: + reset_principal(token) + assert (e.value.status_code, e.value.code) == (status, code) + assert "管理员" in e.value.detail + + +# ---------------------------------------------------------------- /docs + + +async def test_docs_stay_open_when_login_is_off(): + async with client() as c: + for path in DOCS: + assert (await c.get(path)).status_code == 200, path + + +async def test_docs_need_a_builder_once_login_is_on(): + await auth_env.turn_on_local() + await auth_env.make_user("uma", "user") + await auth_env.make_user("bob", "builder") + async with client() as anonymous: + for path in DOCS: + r = await anonymous.get(path) + assert r.status_code == 401 and r.json()["code"] == "auth_required", path + # 子路径(Swagger 的 OAuth 回跳页)一样 + assert (await anonymous.get("/docs/oauth2-redirect")).status_code == 401 + uma = await auth_env.login("uma") + for path in DOCS: + r = await uma.get(path) + assert r.status_code == 403 and r.json()["code"] == "role_required", path + bob = await auth_env.login("bob") + for path in DOCS: + assert (await bob.get(path)).status_code == 200, path + assert "/api/apps/{app_id}" in (await bob.get("/openapi.json")).json()["paths"] diff --git a/backend/tests/test_job_runtime.py b/backend/tests/test_job_runtime.py new file mode 100644 index 00000000..ffda8c90 --- /dev/null +++ b/backend/tests/test_job_runtime.py @@ -0,0 +1,79 @@ +"""后台任务运行时上移到 core/job_runtime.py(docs/design/next-round/00-总方案.md X5)。 + +守三件事:老路径 data/catalog_job_runtime 拿到的是同一个模块(同一条队、同一批名字,起草、剖析的 import 不用改); +同源队里的事是开放的——新的一种登记一行就和起草、剖析排进同一条队、同源串行;没登记的 kind 当场报错,不会悄悄排成 +一种谁都不认识的事。 +""" +from __future__ import annotations + +import asyncio + +import pytest + +from app.core import job_runtime + + +def test_the_old_path_is_the_same_module(): + from app.data import catalog_job_runtime as old + from app.data.catalog_job_runtime import lanes + + assert old is job_runtime + assert lanes is job_runtime.lanes + # 起草、剖析照旧从老路径导入,登记的几种事都在 + import app.data.catalog_jobs # noqa: F401 + import app.data.catalog_profile_jobs # noqa: F401 + + assert {"draft", "profile", "sync_draft", "import"} <= set(job_runtime.lane_kinds()) + assert job_runtime.lane_label("profile") == "剖析" + + +def test_an_unregistered_kind_is_refused(): + lanes = job_runtime.SourceLanes() + + async def go() -> None: + with pytest.raises(ValueError, match="没有登记"): + lanes.enter("src", "nonsense-kind") + + asyncio.run(go()) + + +def test_a_registered_label_cannot_change(): + job_runtime.register_lane_kind("test-open-kind", "测试用") + job_runtime.register_lane_kind("test-open-kind", "测试用") # 同一个叫法再登记无害 + with pytest.raises(ValueError, match="已经登记"): + job_runtime.register_lane_kind("test-open-kind", "别的叫法") + + +def test_a_new_kind_queues_behind_the_existing_ones_on_the_same_source(): + """新登记的一种(第 2 波的后台探查就是这样加进来的)和起草排进同一条队:同源串行,不同源并行。""" + import app.data.catalog_jobs # noqa: F401 - 登记 draft + + job_runtime.register_lane_kind("test-introspect", "测试探查") + lanes = job_runtime.SourceLanes() + + async def go() -> None: + first = lanes.enter("src-a", "draft", "job-1") + second = lanes.enter("src-a", "test-introspect", "job-2") + other = lanes.enter("src-b", "test-introspect", "job-3") + assert first.granted.is_set() and not second.granted.is_set() + assert other.granted.is_set(), "不同源并行" + assert lanes.ahead("src-a", "job-2") == ["draft"] + first.release() + await asyncio.wait_for(second.wait(), timeout=1) + second.release() + other.release() + + asyncio.run(go()) + + +def test_a_job_without_a_lane_skips_the_queue(): + """lane_of 返回 None 的任务(以后不占源的评测)不进队、直接执行。""" + + class _NoLane(job_runtime.JobRun): + kind = "draft" + + def lane_of(self, job): # noqa: ANN001 + return None + + run = _NoLane("whatever") + assert run.lane_of(object()) is None diff --git a/backend/tests/test_notify_events.py b/backend/tests/test_notify_events.py new file mode 100644 index 00000000..211d4dc1 --- /dev/null +++ b/backend/tests/test_notify_events.py @@ -0,0 +1,164 @@ +"""通知事件的登记表(app/notify/service.EVENTS;总方案 X6)。 + +本轮新增的六个事件(数据源变化、口径召回、评测退步 / 失败、待办汇总、密钥到期)在第 1 波一次登记好:名字、界面上的叫法、 +推给哪几条路都在 EVENTS 一处,前端的叫法在 lib/terms.ts 的 NOTIFY_TEXT 一处。第 2、3 波实现推送的那一块只改自己那一行 +的范围,不再去改同一个元组、同一张叫法表(几块并行改同一处必然冲突)。这里守: + +- 登记表:六个事件都在,叫法不空;推给哪几条路是空的就是「登记了、还没有推送」——接口不收这种订阅、GET /api/notify/events + 照实给空的 scopes,设置页和应用的通知里都不列(勾了收不到就是「看着生效、其实没生效」,个人渠道「实例完成」的老问题); +- 前后端对照:terms.ts 的 eventLabel 和 EVENTS 一一对应、逐字一致;投递记录才有的事件(改派、升级、催办、测试消息)另有一张 + 叫法表,和服务端 _AS 对上(写法同 test_chat_turn_policy.py,两边一改就对不上); +- 推给哪几条路是真的:_channels 只走事件登记了的路;没登记的事件推不出去,直接报错; +- 订阅的范围不对(全局渠道勾只推个人的事件)拒收,但只核对这次新勾的:以前存下的照旧,开关、改名不会因此 422。 + +渠道名、地址都是虚构的。 +""" +from __future__ import annotations + +import re +from pathlib import Path + +import pytest +from httpx import ASGITransport, AsyncClient +from sqlalchemy import delete + +from app.core.crypto import encrypt +from app.db.base import SessionLocal +from app.db.models import NotifyChannel +from app.notify import service + +TERMS_TS = Path(__file__).resolve().parents[2] / "frontend" / "src" / "lib" / "terms.ts" +URL = "https://hooks.example.com/agentlab/notify-events-5d1c" + +#: 本轮登记的六个(总方案 X6):都还没有推送 +THIS_ROUND = ("source.changed", "ledger.recall", "eval.regressed", "eval.failed", "task.digest", "key.expiring") + + +@pytest.fixture +async def client(): + from app.main import app + + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + async with SessionLocal() as session: + await session.execute(delete(NotifyChannel).where(NotifyChannel.name.like("事件登记-%"))) + await session.commit() + + +# -------------------------------------------------------------------------- +# 登记表 +# -------------------------------------------------------------------------- + + +def test_this_rounds_events_are_registered_once_and_not_pushed_yet(): + for name in THIS_ROUND: + assert name in service.EVENTS, f"{name} 没有登记" + assert service.EVENTS[name].label, name + assert service.EVENTS[name].scopes == (), f"{name} 还没有推送:不该有人订阅得到" + assert len({e.label for e in service.EVENTS.values()}) == len(service.EVENTS), "两个事件叫成了同一句" + assert service.EVENT_LABEL == {k: v.label for k, v in service.EVENTS.items()} + + +def test_events_that_are_pushed_today_keep_their_audience(): + """现有五个事件推给哪几条路,照 _channels 和各个 notify_* 现在的做法登记(行为不变)。""" + every = {"global", "user", "app"} + assert {k: set(v.scopes) for k, v in service.EVENTS.items() if v.scopes} == { + "task.opened": every, "run.finished": every, "run.failed": every, "action.unknown": every, + # 发给提交人自己(和订阅了它的全局渠道),不归哪个应用 + "review.done": {"global", "user"}, + } + for event in service.EVENTS.values(): + assert set(event.scopes) <= every + + +def _object_block(source: str, name: str) -> str: + """terms.ts 里 NOTIFY_TEXT 下面名叫 name 的那个对象字面量(到配对的右花括号为止)。""" + head = source.index("export const NOTIFY_TEXT = {") + start = source.index(f"{name}: {{", head) + len(name) + 2 + depth, i = 1, start + 1 + while depth: + depth += {"{": 1, "}": -1}.get(source[i], 0) + i += 1 + return source[start:i] + + +def _entries(block: str) -> dict[str, str]: + return dict(re.findall(r"'?([a-z]+(?:\.[a-z_]+)?)'?:\s*'([^']*)'", block)) + + +def test_frontend_labels_match_the_registry_word_for_word(): + source = TERMS_TS.read_text(encoding="utf-8") + labels = _entries(_object_block(source, "eventLabel")) + assert labels == service.EVENT_LABEL, "设置页的叫法和服务端登记的对不上(一种说法)" + delivery = _entries(_object_block(source, "deliveryEventLabel")) + assert set(delivery) == set(service._AS) | {"test"}, "投递记录里会出现的事件都要有叫法" + assert not set(delivery) & set(labels) + + +# -------------------------------------------------------------------------- +# 接口 +# -------------------------------------------------------------------------- + + +async def test_events_endpoint_says_who_receives_each_event(client): + rows = (await client.get("/api/notify/events")).json() + assert [r["event"] for r in rows] == list(service.EVENTS) + by = {r["event"]: r for r in rows} + assert by["eval.regressed"] == {"event": "eval.regressed", "label": "评测有退步", "scopes": []} + assert by["review.done"]["scopes"] == ["global", "user"] + + +async def test_events_not_pushed_yet_cannot_be_subscribed(client): + resp = await client.post("/api/notify/channels", json={ + "name": "事件登记-评测", "url": URL, "events": ["task.opened", "eval.regressed"]}) + assert resp.status_code == 422, resp.text + detail = resp.json()["detail"] + assert "评测有退步" in detail and "还没有推送" in detail + unknown = await client.post("/api/notify/channels", json={"name": "事件登记-未知", "url": URL, "events": ["nope"]}) + assert unknown.status_code == 422 + assert "评测有退步" not in unknown.json()["detail"], "可选的里面不列还没有推送的" + assert "待办打开" in unknown.json()["detail"] + + +async def test_scope_mismatch_is_refused_only_for_newly_ticked_events(client): + refused = await client.post("/api/notify/channels", json={ + "name": "事件登记-应用", "scope": "app", "url": URL, "events": ["review.done"]}) + assert refused.status_code == 422, refused.text + assert "我提交审核的应用有结果" in refused.json()["detail"] and "应用" in refused.json()["detail"] + # 以前存下的(当时界面对哪种渠道都列这一项):开关、改名照旧,不因此 422 + async with SessionLocal() as session: + row = NotifyChannel(name="事件登记-旧", scope="app", url=encrypt(URL), events=["task.opened", "review.done"]) + session.add(row) + await session.commit() + channel_id = row.id + toggled = await client.patch(f"/api/notify/channels/{channel_id}", json={"enabled": False}) + assert toggled.status_code == 200, toggled.text + assert toggled.json()["events"] == ["task.opened", "review.done"] + again = await client.patch(f"/api/notify/channels/{channel_id}", json={"events": ["review.done", "run.failed"]}) + assert again.status_code == 200, again.text + added = await client.patch(f"/api/notify/channels/{channel_id}", json={"events": ["review.done", "eval.failed"]}) + assert added.status_code == 422 and "评测失败" in added.json()["detail"] + + +# -------------------------------------------------------------------------- +# 推送只走登记了的路 +# -------------------------------------------------------------------------- + + +async def test_channels_only_walk_the_paths_the_event_is_registered_for(monkeypatch): + monkeypatch.setitem(service.EVENTS, "demo.personal", service.Event("只推个人的演示事件", ("user",))) + async with SessionLocal() as session: + row = NotifyChannel(name="事件登记-全局", scope="global", url=encrypt(URL), + events=["demo.personal", "task.opened", "eval.failed"]) + session.add(row) + await session.commit() + try: + ids = lambda found: [c.id for c in found] # noqa: E731 + assert row.id not in ids(await service._channels(session, "demo.personal", None, [])), "全局渠道这条路没登记" + assert row.id in ids(await service._channels(session, "task.opened", None, [])) + assert await service._channels(session, "eval.failed", None, []) == [], "还没有推送的:谁都不推" + with pytest.raises(ValueError, match="没有登记"): + await service._channels(session, "demo.unregistered", None, []) + finally: + await session.delete(row) + await session.commit() diff --git a/backend/tests/test_notify_run.py b/backend/tests/test_notify_run.py new file mode 100644 index 00000000..a57e32ff --- /dev/null +++ b/backend/tests/test_notify_run.py @@ -0,0 +1,199 @@ +"""应用实例结束的通知(notify/service.notify_run):说法取自实例页,个人渠道按「我发起的实例」推。 + +- 被驳回的实例(cancelled + end_reason=rejected)走「实例结束」,状态写实例页那一句(「「review」被驳回」),不推 + 「实例失败」(05-已知遗留 (a)、总方案 X6); +- 走到「已退回」那种结局的运行,实例页写成果节点的标题、记成人的决定,推送以前却写「已完成」:现在两边一致; +- 个人渠道勾了「实例完成 / 失败」以前永远收不到(_channels 拿到的 users 是空的):发起人是认证过的身份时,他自己的 + 个人渠道收得到;自报的署名不算(04-对外集成与触达 B1)。 +""" +from __future__ import annotations + +import asyncio +import json +import uuid + +import httpx +import pytest +from sqlalchemy import select + +from app.core.crypto import encrypt +from app.db.base import SessionLocal +from app.db.models import App, AppVersion, NotifyChannel, NotifyDelivery, Run, User +from app.engine.runner import run_manager +from app.notify import core, service + +URL = "https://hooks.example.com/agentlab/run-3b8e1f" + + +@pytest.fixture(autouse=True) +async def engine_up(): + await run_manager.setup() + yield + await service.drain() + await run_manager.shutdown() + + +@pytest.fixture +def received(monkeypatch): + got: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + got.append(request) + return httpx.Response(200, json={"ok": True}) + + monkeypatch.setattr(core, "_transport", httpx.MockTransport(handler)) + monkeypatch.setattr(core, "RETRY_DELAYS", (0, 0, 0)) + # 个人渠道发送前要解析域名、核对不是内网:测试里不连外网 + async def public(url: str) -> None: + return None + + monkeypatch.setattr(service, "personal_url_problem", public) + return got + + +def node(nid, ntype, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}} + + +def flow(**human): + nodes = [node("start", "input", fields=[{"name": "q"}]), + node("review", "human", mode="approve", title="审一下", **human), + node("ok", "output", fields=[{"name": "结论", "value": "通过"}]), + node("no", "output", fields=[{"name": "结论", "value": "退回"}])] + edges = [{"source": "start", "target": "review"}, + {"source": "review", "target": "ok", "sourceHandle": "approved"}, + {"source": "review", "target": "no", "sourceHandle": "rejected"}] + return {"nodes": nodes, "edges": edges} + + +async def _app(events: list[str]) -> tuple[str, str]: + async with SessionLocal() as session: + channel = NotifyChannel(name=f"群-{uuid.uuid4().hex[:6]}", format="generic", url=encrypt(URL), events=[]) + session.add(channel) + app = App(name=f"报销-{uuid.uuid4().hex[:6]}", published_version=1) + session.add(app) + await session.flush() + session.add(AppVersion(app_id=app.id, version=1, spec_hash="h" * 64, bindings={}, + spec={"notify": [{"channel": channel.id, "events": events}]})) + await session.commit() + return app.id, channel.id + + +async def _wait(run_id: str, statuses: tuple[str, ...]) -> Run: + for _ in range(400): + async with SessionLocal() as session: + row = await session.get(Run, run_id) + if row is not None and row.status in statuses: + return row + await asyncio.sleep(0.05) + raise AssertionError(f"运行 {run_id} 没有进入 {statuses}") + + +async def _decide(run_id: str, approved: bool) -> Run: + """经待办服务回复(同界面上点同意 / 驳回):实例页认结局靠待办的结论。""" + from app.api.deps import Principal + from app.db.models import Task + from app.process import tasks + + await _wait(run_id, ("interrupted",)) + await run_manager.wait_idle(run_id) + for _ in range(100): + async with SessionLocal() as session: + task = (await session.execute(select(Task).where(Task.run_id == run_id, Task.status == "pending")))\ + .scalars().first() + if task is not None: + break + await asyncio.sleep(0.05) + await tasks.respond(task.id, Principal(username=None, role="admin", mode="off"), + decision="approve" if approved else "reject", note="金额 45678 不对") + final = await _wait(run_id, ("succeeded", "failed", "cancelled")) + await run_manager.wait_idle(run_id) + await service.drain() + return final + + +async def _deliveries(channel_id: str) -> list[tuple[str, str]]: + async with SessionLocal() as session: + rows = (await session.execute(select(NotifyDelivery).where(NotifyDelivery.channel_id == channel_id))).scalars() + return [(r.event, r.status) for r in rows] + + +async def test_a_rejected_instance_is_pushed_as_finished_in_the_instance_page_words(received): + app_id, channel_id = await _app(["run.finished", "run.failed"]) + run = await run_manager.start(graph=flow(stop_on_reject=True), input_payload={"q": "x"}, app_id=app_id, + app_version=1) + final = await _decide(run.id, approved=False) + assert (final.status, final.end_reason) == ("cancelled", "rejected") + assert await _deliveries(channel_id) == [("run.finished", "ok")], "驳回不进「实例失败」" + [request] = received + body = json.loads(request.content) + assert body["event"] == "run.finished" and body["title"].endswith("的实例已结束") + assert "状态:「review」被驳回" in body["text"] and "失败" not in body["text"] + assert "45678" not in request.content.decode(), "驳回的备注不进消息(可能带着数)" + + +async def test_a_manual_stop_is_not_pushed(received): + app_id, channel_id = await _app(["run.finished", "run.failed"]) + run = await run_manager.start(graph=flow(), input_payload={"q": "x"}, app_id=app_id, app_version=1) + await _wait(run.id, ("interrupted",)) + await run_manager.wait_idle(run.id) + assert await run_manager.cancel(run.id) == "cancelled" + await service.drain() + assert await _deliveries(channel_id) == [] and received == [] + + +async def test_reaching_the_returned_outcome_no_longer_says_completed(received): + app_id, channel_id = await _app(["run.finished"]) + run = await run_manager.start(graph=flow(), input_payload={"q": "x"}, app_id=app_id, app_version=1) + final = await _decide(run.id, approved=False) + assert final.status == "succeeded" and final.output.get("结论") == "退回" + [request] = received + body = json.loads(request.content) + # 实例页写走到的成果节点(no),记成人的决定:推送也这么写,不再说「已完成」 + assert "状态:no" in body["text"] and "已完成" not in body["title"] and body["title"].endswith("的实例已结束") + + +async def _personal(username: str, events: list[str]) -> str: + async with SessionLocal() as session: + user = (await session.execute(select(User).where(User.username == username))).scalar_one_or_none() + if user is None: + user = User(username=username, role="user") + session.add(user) + await session.flush() + channel = NotifyChannel(name="我的", scope="user", owner_id=user.id, format="generic", url=encrypt(URL), + events=events) + session.add(channel) + await session.commit() + return channel.id + + +async def _finished_instance(app_id: str, *, started_by: str, verified: bool) -> str: + async with SessionLocal() as session: + run = Run(workflow_name="报销", status="succeeded", graph=flow(), input={}, app_id=app_id, app_version=1, + started_by=started_by, origin_kind="app_entry", + origin={"kind": "app_entry", "actor": started_by, "actor_verified": verified}) + session.add(run) + await session.commit() + return run.id + + +async def test_the_initiator_s_personal_channel_receives_my_instance_finished(received): + """B1:个人渠道订阅了 run.finished,发起人自己发起的实例结束了收得到;应用自己没订阅也一样。""" + app_id, _ = await _app([]) + name = f"uma{uuid.uuid4().hex[:6]}" + mine = await _personal(name, ["run.finished"]) + run_id = await _finished_instance(app_id, started_by=name, verified=True) + assert await service.notify_run(run_id, "run.finished") == 1 + assert await _deliveries(mine) == [("run.finished", "ok")] + body = json.loads(received[0].content) + assert body["title"].endswith("的实例已完成") and "状态:已完成" in body["text"] + + +async def test_a_self_declared_initiator_does_not_reach_a_personal_channel(received): + """关登录时署名是自己填的,谁都能填别人的名字:不按它推给那个人的个人渠道。""" + app_id, _ = await _app([]) + name = f"vic{uuid.uuid4().hex[:6]}" + mine = await _personal(name, ["run.finished"]) + run_id = await _finished_instance(app_id, started_by=name, verified=False) + assert await service.notify_run(run_id, "run.finished") == 0 + assert await _deliveries(mine) == [] and received == [] diff --git a/backend/tests/test_personal_apps.py b/backend/tests/test_personal_apps.py index 8439a8d7..ff25c447 100644 --- a/backend/tests/test_personal_apps.py +++ b/backend/tests/test_personal_apps.py @@ -964,9 +964,10 @@ async def test_the_other_doors_to_a_private_run_are_shut_too(monkeypatch): bob_trial = await wait(r.json()["run_id"]) question = {"question": "上周各门店的销售额"} async with SessionLocal() as session: - # 聚类只列两条以上的簇:别的探索运行放两条 + # 聚类只列两条以上的簇:别的探索运行放两条。用画布试跑:问数据的问题按轮次取(app/chat/questions.py), + # 没挂在任何一轮上的问数据运行不单独算 shared, also = (Run(workflow_name="共享的运行", graph=KPI, status="succeeded", run_class="exploratory", - origin_kind="chat", started_by="bob", input=question) for _ in range(2)) + origin_kind="canvas", started_by="bob", input=question) for _ in range(2)) session.add_all([shared, also]) await session.flush() for run_id in (trial.id, refresh.id, bob_trial.id): diff --git a/backend/tests/test_process_actions.py b/backend/tests/test_process_actions.py index f11ae634..b0d0e992 100644 --- a/backend/tests/test_process_actions.py +++ b/backend/tests/test_process_actions.py @@ -173,8 +173,8 @@ async def test_unknown_result_with_lookup_is_reconciled_automatically_and_sent_o await run_manager.continue_failed(run_id) assert refused.value.code == "action_unresolved" - assert await actions.tick(utcnow()) == 0 # 还没到第一次查询的时候 - assert await actions.tick(utcnow() + timedelta(minutes=2)) == 1 + assert rec.id not in await actions.tick(utcnow()) # 还没到第一次查询的时候 + assert rec.id in await actions.tick(utcnow() + timedelta(minutes=2)) final = await _wait(run_id, ("succeeded", "failed")) assert final.status == "succeeded", final.error assert len(erp.writes()) == 1 and len(erp.lookups) == 1 @@ -410,7 +410,7 @@ async def test_a_changed_params_schema_does_not_fail_a_write_that_already_went_o with pytest.raises(actions.ActionPending): await actions.prepare(run_id=run_id, node_id="pay", exec_no=1, capability=caps[PAY], args=args) - assert await actions.tick(utcnow() + timedelta(minutes=2)) == 1 + assert rec.id in await actions.tick(utcnow() + timedelta(minutes=2)) final = await _wait(run_id, ("succeeded", "failed")) assert final.status == "succeeded", final.error assert len(erp.writes()) == 1 @@ -457,9 +457,15 @@ async def test_a_rejected_confirmation_reads_as_a_decision_not_a_fault(erp, capa run_id = await _formal(pay_graph()) task = await _task(run_id, "action_confirm") await tasks.respond(task.id, ANN, decision="reject", note="金额不对") - final = await _wait(run_id, ("succeeded", "failed")) - # 引擎里驳回就是节点失败、运行结束(没有可走的出口);说法写成未确认(被驳回),实例页按人的决定说 - assert final.status == "failed" and final.error.startswith("「pay」未确认(被驳回),没有发送:金额不对") + final = await _wait(run_id, ("succeeded", "failed", "cancelled")) + # 驳回是人的决定,不是故障(05-已知遗留 (a)):运行收成 cancelled + end_reason=rejected,原话照旧写未确认(被驳回), + # run.cancelled 带上在哪一步、谁、备注;实例页按人的决定说 + assert (final.status, final.end_reason) == ("cancelled", "rejected") + assert final.error.startswith("「pay」未确认(被驳回),没有发送:金额不对") + [ended] = await _events(run_id, "run.cancelled") + assert {k: ended.data[k] for k in ("reason", "node_id", "kind", "decision", "note")} == { + "reason": "rejected", "node_id": "pay", "kind": "action_confirm", "decision": "reject", "note": "金额不对"} + assert not await _events(run_id, "run.failed") view = await _instance_view(run_id) assert (view["hdr"]["status_code"], view["hdr"]["status_text"]) == ("cancelled", "「pay」未确认(被驳回)") step = next(s for s in view["tl"]["steps"] if s["node"] == "pay") @@ -481,7 +487,7 @@ async def test_unknown_then_reconciled_reads_as_waiting_then_a_footnote(erp, cap [row] = view["act"]["records"] assert row["state"] == "unknown" and "error" not in row and "ReadTimeout" in row["note"] - assert await actions.tick(utcnow() + timedelta(minutes=2)) == 1 + assert row["id"] in await actions.tick(utcnow() + timedelta(minutes=2)) final = await _wait(run_id, ("succeeded", "failed")) assert final.status == "succeeded", final.error view = await _instance_view(run_id) diff --git a/backend/tests/test_process_tasks.py b/backend/tests/test_process_tasks.py index 3bb85db9..b1f875fc 100644 --- a/backend/tests/test_process_tasks.py +++ b/backend/tests/test_process_tasks.py @@ -271,7 +271,24 @@ async def test_login_on_only_assignees_can_respond(): # -------------------------------------------------------------------------- -async def test_escalation_adds_the_director_whose_answer_decides(): +@pytest.fixture +async def stray_due_task(): + """别的测试留下的、早就到期、可升级的待办(-n 8 下同一个库里常有)。tick 扫全库,按条数断言的测试会被它带偏: + 这里先放一条,测试只断言自己造的那条在不在 tick 交回的 id 里(05-已知遗留 (f))。""" + async with SessionLocal() as session: + run = Run(workflow_name="别的测试留下的", status="interrupted", graph={}, input={}) + session.add(run) + await session.flush() + task = Task(run_id=run.id, node_id="x", kind="approve", title="残留", due_at=utcnow() - timedelta(days=1), + on_timeout={"action": "escalate", "to_slot": "nobody"}) + session.add(task) + await session.commit() + yield task.id + await session.execute(delete(Run).where(Run.id == run.id)) + await session.commit() + + +async def test_escalation_adds_the_director_whose_answer_decides(stray_due_task): a, b, g, boss = f"a{_u()}", f"b{_u()}", f"finance{_u()}", f"boss{_u()}" await _people(a, b, group=g) run_id = await _app_run( @@ -280,9 +297,9 @@ async def test_escalation_adds_the_director_whose_answer_decides(): {"finance": [{"kind": "group", "name": g}], "director": [{"kind": "user", "name": boss}]}) task = await _task(run_id) assert task.due_at is not None - assert await tasks.tick(utcnow()) == 0 # 还没到时限 - assert await tasks.tick(utcnow() + timedelta(hours=2)) == 1 - assert await tasks.tick(utcnow() + timedelta(hours=3)) == 0 # 只升一次 + assert task.id not in await tasks.tick(utcnow()) # 还没到时限 + assert task.id in await tasks.tick(utcnow() + timedelta(hours=2)) + assert task.id not in await tasks.tick(utcnow() + timedelta(hours=3)) # 只升一次 detail = await tasks.detail(task.id, off(None)) assert detail["escalation_level"] == 1 and detail["status"] == "pending" assert ("user", boss, "escalation") in [(x["principal_kind"], x["principal"], x["source"]) @@ -293,13 +310,13 @@ async def test_escalation_adds_the_director_whose_answer_decides(): assert (await _wait(run_id, ("succeeded", "failed"))).output.get("结论") == "通过" -async def test_timeout_reject_never_approves(): +async def test_timeout_reject_never_approves(stray_due_task): run = await run_manager.start( graph=approval_flow(due="PT30M", on_timeout={"action": "reject"}), input_payload={"q": "x"}) await _wait(run.id, ("interrupted",)) await run_manager.wait_idle(run.id) task = await _task(run.id) - assert await tasks.tick(utcnow() + timedelta(hours=1)) == 1 + assert task.id in await tasks.tick(utcnow() + timedelta(hours=1)) final = await _wait(run.id, ("succeeded", "failed")) assert final.output.get("结论") == "退回" assert len(await _events(run.id, "task.expired")) == 1 @@ -308,16 +325,16 @@ async def test_timeout_reject_never_approves(): assert row.status == "resolved" and row.outcome["decision"] == "expired_reject" -async def test_remind_repeats_every_interval(): +async def test_remind_repeats_every_interval(stray_due_task): run = await run_manager.start( graph=approval_flow(due="PT1H", on_timeout={"action": "remind", "every": "PT1H"}), input_payload={"q": "x"}) await _wait(run.id, ("interrupted",)) await run_manager.wait_idle(run.id) task = await _task(run.id) later = utcnow() + timedelta(hours=1, minutes=1) - assert await tasks.tick(later) == 1 - assert await tasks.tick(later) == 0 - assert await tasks.tick(later + timedelta(hours=1)) == 1 + assert task.id in await tasks.tick(later) + assert task.id not in await tasks.tick(later) + assert task.id in await tasks.tick(later + timedelta(hours=1)) assert len(await _events(run.id, "task.reminded")) == 2 async with SessionLocal() as session: assert (await session.get(Task, task.id)).status == "pending" diff --git a/backend/tests/test_profile_dialect_golden.py b/backend/tests/test_profile_dialect_golden.py new file mode 100644 index 00000000..7e7c6321 --- /dev/null +++ b/backend/tests/test_profile_dialect_golden.py @@ -0,0 +1,178 @@ +"""剖析 SQL 的方言金样(下一轮 03 细案 C2):没有真库时挡住方言写法的回退。 + +三层,各挡各的: + +1. **金样**:每种方言(mysql、mariadb、postgresql、oracle、sqlite)× 每类查询(统计信息、有上限的计数、取不同键值、 + 对键值、唯一性、取值分布、日期最小最大、时间窗口里的取值分布、按日行数、块采样),完整 SQL 钉在 + tests/fixtures/dialect/<方言>.sql,人审过一次;以后拼法有任何改动都会出 diff。要更新金样: + ``UPDATE_DIALECT_GOLDEN=1 pytest tests/test_profile_dialect_golden.py``,再逐行看 git diff。 +2. **sqlglot 逐条解析**:按方言读一遍(mariadb 按 mysql 读),挡住语法写坏了(少括号、关键字拼错)。 +3. **IN 两边类型族一致**:用 sqlglot 的 qualify + annotate_types,配上表结构里的类型,核对 IN 列表的字面量和被指向 + 列是同一类型族(PostgreSQL 上 varchar 父列配整数子列时,字面量必须带引号)。 + +挡不住的照实写:sqlglot 不知道真库的统计信息是不是过期、零日期驱动怎么返回、Oracle 小写对象是否存在、排序规则 +怎么比——这些要靠 scripts/dialect-it.sh 在本机容器里跑真库(不进 CI)。 +""" +from __future__ import annotations + +import os +from pathlib import Path + +import pytest +import sqlglot +from sqlglot import exp +from sqlglot.optimizer.annotate_types import annotate_types +from sqlglot.optimizer.qualify import qualify + +from app.data import catalog, guard +from app.data.catalog_profile import IN_CHUNK, SqlDialect, key_literals + +GOLDEN = Path(__file__).parent / "fixtures" / "dialect" +DIALECTS = ("mysql", "mariadb", "postgresql", "oracle", "sqlite") +#: sqlglot 的方言名 +READ = {"mysql": "mysql", "mariadb": "mysql", "postgresql": "postgres", "oracle": "oracle", "sqlite": "sqlite"} + +#: 夹具表结构:一张带 schema 的订单表、一张门店表(编码是文字),Oracle 的一张小写带引号的表和一个小写带引号的列 +META = { + "orders": {"schema": "sales", "columns": [ + {"name": "id", "type": "INTEGER"}, {"name": "store_code", "type": "INTEGER"}, + {"name": "status", "type": "CHAR(2)"}, {"name": "paid_at", "type": "TIMESTAMP"}, + {"name": "biz_day", "type": "VARCHAR(10)"}, {"name": "level", "type": "INTEGER"}], + "primary_key": ["id"]}, + "stores": {"schema": None, "columns": [{"name": "code", "type": "VARCHAR(10)"}], "primary_key": ["code"]}, + "visits": {"schema": None, "quote_name": True, "columns": [ + {"name": "id", "type": "INTEGER"}, {"name": "level", "type": "INTEGER", "quote_name": True}], + "primary_key": ["id"]}, +} + + +def _queries(kind: str) -> list[tuple[str, str | None]]: + """一种方言的全部剖析、观察查询:(名字, SQL)。没有这种查询(SQLite 没有统计信息、MySQL 没有块采样)的 SQL 为 None。""" + d = SqlDialect(kind) + orders, stores, visits = d.table(META["orders"], "orders"), d.table(META["stores"], "stores"), \ + d.table(META["visits"], "visits") + status, code = d.column(META["orders"], "status"), d.column(META["stores"], "code") + level = d.column(META["visits"], "level") + keys = key_literals([12, 7], "number", parent="text", child_type="INTEGER").literals + out: list[tuple[str, str | None]] = [ + ("stats", d.stats_sql(META["orders"], "orders")), + ("stats_quoted", d.stats_sql(META["visits"], "visits")), + ("bounded", d.bounded_count_sql(orders, 100_001)), + ("distinct", d.distinct_sample_sql(orders, d.column(META["orders"], "store_code"), 2000, scan_cap=None)), + ("distinct_capped", d.distinct_sample_sql(orders, d.column(META["orders"], "store_code"), 2000, + scan_cap=100_000)), + ("match", d.match_count_sql(stores, code, keys)), + ("unique", d.unique_check_sql(visits, level)), + ("value_counts", d.value_counts_sql(orders, status, 21, scan_cap=None)), + ("value_counts_capped", d.value_counts_sql(orders, status, 21, scan_cap=100_000)), + ("min_max", d.min_max_sql(orders, [(d.column(META["orders"], "paid_at"), False), + (d.column(META["orders"], "biz_day"), True)])), + ("date_window", d.window_value_counts_sql(orders, status, d.column(META["orders"], "paid_at"), + "2026-08-27", 21)), + ("day_volume", d.day_volume_sql(orders, d.column(META["orders"], "paid_at"), "2026-08-27")), + ("sample", d.sample_value_counts_sql(orders, status, 1, 21)), + ] + return out + + +def _render(kind: str) -> str: + lines = [f"-- 剖析与取值观察的 SQL 金样:{kind}(tests/test_profile_dialect_golden.py 生成,改动要人审)"] + for name, sql in _queries(kind): + lines.append(f"-- {name}") + lines.append(f"{sql};" if sql is not None else "-- (这种方言没有这类查询)") + return "\n".join(lines) + "\n" + + +@pytest.mark.parametrize("kind", DIALECTS) +def test_profile_sql_matches_the_golden_file(kind): + path = GOLDEN / f"{kind}.sql" + text = _render(kind) + if os.environ.get("UPDATE_DIALECT_GOLDEN") == "1": + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + assert path.read_text(encoding="utf-8") == text, f"{kind} 的剖析 SQL 和金样不一致:看 diff,确认无误后重新生成金样" + + +@pytest.mark.parametrize("kind", DIALECTS) +def test_every_profile_query_parses_and_passes_the_guard(kind): + for name, sql in _queries(kind): + if sql is None: + continue + parsed = sqlglot.parse(sql, read=READ[kind], error_level=sqlglot.ErrorLevel.RAISE) + assert len(parsed) == 1 and parsed[0] is not None, (kind, name) + assert guard.check(sql, readonly=True, dialect=kind) == sql, (kind, name) + + +@pytest.mark.parametrize("kind", ["postgresql", "oracle", "mysql"]) +def test_observation_queries_use_the_right_day_and_sample_functions(kind): + """按天截断和块采样各家写法不同,解析成语法树再看用的是哪个函数,不只比字符串。""" + sql = dict(_queries(kind)) + day = sqlglot.parse_one(sql["day_volume"], read=READ[kind]) + if kind == "oracle": + assert any(isinstance(n, (exp.DateTrunc, exp.TimestampTrunc, exp.Anonymous)) or n.key == "trunc" + for n in day.walk()), sql["day_volume"] + assert "TRUNC(paid_at)" in sql["day_volume"] + elif kind == "postgresql": + assert any(isinstance(n, exp.Cast) for n in day.walk()) + else: + assert "DATE(paid_at)" in sql["day_volume"] + if kind == "mysql": + assert sql["sample"] is None + else: + sampled = sqlglot.parse_one(sql["sample"], read=READ[kind]) + assert any(isinstance(n, exp.TableSample) for n in sampled.walk()), sql["sample"] + + +# ---------------------------------------------------------------- IN 两边的类型族 + + +_FAMILY_OF = [(exp.DataType.TEXT_TYPES, "text"), (exp.DataType.NUMERIC_TYPES, "number"), + (exp.DataType.TEMPORAL_TYPES, "time")] + + +def _sg_family(t: exp.DataType | None) -> str | None: + if t is None: + return None + for group, name in _FAMILY_OF: + if t.this in group: + return name + return None + + +def _in_families(sql: str, kind: str, schema: dict[str, dict[str, str]]) -> list[tuple[str | None, set[str | None]]]: + """sql 里每个 IN:(被比较的列的类型族, 字面量的类型族集合),按 annotate_types 配上表结构的类型推出来。""" + tree = qualify(sqlglot.parse_one(sql, read=READ[kind]), schema=schema, dialect=READ[kind], + validate_qualify_columns=False) + tree = annotate_types(tree, schema=schema, dialect=READ[kind]) + return [(_sg_family(node.this.type), {_sg_family(x.type) for x in node.expressions}) + for node in tree.find_all(exp.In)] + + +@pytest.mark.parametrize("kind", DIALECTS) +@pytest.mark.parametrize("parent_type, child_values, child_type", [ + ("VARCHAR(10)", [12, 7, 30], "INTEGER"), # 父列文字、子列数:字面量要带引号 + ("INTEGER", ["12", "0007", "A1"], "VARCHAR(10)"), # 父列数、子列文字:只收整数写法,写成数字 + ("VARCHAR(10)", ["A ", "BC"], "CHAR(2)"), # 定长子列:去掉补齐的空格 + ("INTEGER", [1, 2], "INTEGER"), +]) +def test_in_list_literals_have_the_parent_columns_family(kind, parent_type, child_values, child_type): + d = SqlDialect(kind) + parent = {"name": "code", "type": parent_type} + literals = key_literals(child_values, None, parent=catalog.column_family(parent), child_type=child_type).literals + assert literals + sql = d.match_count_sql("stores", d.quote("code"), literals[:IN_CHUNK]) + found = _in_families(sql, kind, {"stores": {"code": parent_type}}) + assert found, sql + for column_family, literal_families in found: + assert literal_families == {column_family}, (kind, sql, column_family, literal_families) + + +def test_the_old_way_would_mismatch_on_postgres(): + """对照:以前按子表取回的值写字面量,varchar 父列配整数子列在 sqlglot 推出的类型上就两边不同——PostgreSQL 真库上 + 这是「operator does not exist: character varying = integer」。""" + from app.data.catalog_profile import sql_literal + + old = [lit for v in (12, 7) if (lit := sql_literal(v, "number"))] + sql = SqlDialect("postgresql").match_count_sql("stores", "code", old) + [(column_family, literal_families)] = _in_families(sql, "postgresql", {"stores": {"code": "VARCHAR(10)"}}) + assert column_family == "text" and literal_families == {"number"} diff --git a/backend/tests/test_profile_dialect_units.py b/backend/tests/test_profile_dialect_units.py new file mode 100644 index 00000000..d2bc226d --- /dev/null +++ b/backend/tests/test_profile_dialect_units.py @@ -0,0 +1,428 @@ +"""剖析与探查的方言修正(下一轮 03 细案 C1):不连库的部分。 + +- 列类型按方言编译的完整写法:类型族、日期列、码值候选都认得 ENUM / SET、VARCHAR2(20 CHAR)、INTERVAL 这些写法; +- IN 列表的字面量按被指向列的类型族生成(PG 上文字列配整数子列会直接报错,Oracle 会把父列转成数报 ORA-01722); +- Oracle 小写带引号的对象(quote_name)强制加引号、不转大写; +- 统计信息估出的行数不超过整表统计上限时再数一次确认; +- 日期列的最小值、最大值排除零日期和哨兵日期,照实写排除了几个; +- 取值观察要用的三个方言方法(date_literal / day_trunc / sample_from); +- 说明文字不再说「抽样」:大表看的是按存储顺序的前 N 行。 +""" +from __future__ import annotations + +import asyncio +import datetime as _dt +from typing import Any + +import pytest +from sqlalchemy import VARCHAR, Date, Numeric +from sqlalchemy.dialects import mysql, oracle, postgresql, sqlite +from sqlalchemy.sql.elements import quoted_name + +from app.data import catalog, catalog_profile, introspect +from app.data.catalog_profile import ProfileSettings, SqlDialect, TableSize +from app.data.engine import QueryResult + +DIALECTS = ("mysql", "mariadb", "postgresql", "oracle", "sqlite") + + +# ---------------------------------------------------------------- 类型的写法 + + +@pytest.mark.parametrize("type_name, family", [ + ("ENUM('paid','refund')", "text"), + ("ENUM('int','date')", "text"), # 取值里的词不能让它变成数字或时间 + ("SET('a','b')", "text"), + ("VARCHAR2(20 CHAR)", "text"), + ("NVARCHAR2(10)", "text"), + ("VARCHAR(20) COLLATE utf8mb4_bin", "text"), + ("NUMBER(10, 0)", "number"), + ("INTEGER(11)", "number"), + ("BIGINT UNSIGNED", "number"), + ("DOUBLE PRECISION", "number"), + ("TIMESTAMP WITH TIME ZONE", "time"), + ("TIMESTAMP(3)", "time"), + ("INTERVAL DAY(2) TO SECOND(6)", None), # 时长不是数也不是时刻;以前按「INT」认成了数字 + ("order_status", None), + ("", None), +]) +def test_type_family_reads_dialect_full_types(type_name, family): + assert catalog._type_family(type_name) == family + + +def test_type_family_of_a_column_uses_its_enum_values(): + """PostgreSQL 的枚举类型编译出来是类型名(order_status),只有取值列表能说明它是文字。""" + assert catalog.column_family({"name": "status", "type": "order_status", "enum_values": ["a", "b"]}) == "text" + assert catalog.column_family({"name": "n", "type": "INTEGER"}) == "number" + assert catalog.column_family({"name": "x"}) is None + + +def test_enum_values_never_make_a_date_column(): + assert not catalog.is_date_column("status", "ENUM('date','time')") + assert catalog.is_date_column("paid_at", "TIMESTAMP WITH TIME ZONE") + assert catalog.is_date_column("biz_dt", "VARCHAR2(10 CHAR)") # 文字存日期、名字像日期 + assert not catalog.is_date_column("gap", "INTERVAL DAY(2) TO SECOND(6)") + + +# PostgreSQL 的枚举、域编译出来是类型名本身:名字以 set、enum 开头不能被当成 MySQL 的 SET / ENUM,PG 的枚举一律靠 +# 探查记下的 enum_values 认 +@pytest.mark.parametrize("type_name, family", [ + ("setup_status", None), + ("settlement_amount", None), + ("enum_kind", None), + ("SET ('a','b')", "text"), + ("enum('paid')", "text"), +]) +def test_only_mysql_enum_and_set_spellings_are_enum(type_name, family): + assert catalog._type_family(type_name) == family + assert catalog_profile._is_enum(type_name) is (family == "text") + + +def test_postgres_type_names_starting_with_set_or_enum(): + # 带取值列表的 PG 枚举:是码值候选、是文字,但不是 MySQL 的 SET + assert catalog_profile._is_enum("setup_status", ["draft", "done"]) + assert catalog.column_family({"name": "stage", "type": "setup_status", "enum_values": ["draft"]}) == "text" + # 名字以 set 开头的数值域:不是文字、不是码值候选 + assert not catalog_profile._code_type_ok("settlement_amount") + # 名字以 set 开头的日期域:照样是日期列 + assert catalog.is_date_column("settled_on", "settlement_date") + # 枚举的类型名里碰巧有 DATE(UPDATE 里的):有取值列表就不是日期列 + assert not catalog.is_date_column("reason", "update_reason", enum_values=["manual", "sync"]) + assert not catalog_profile._is_date_column({"name": "reason", "type": "update_reason", + "enum_values": ["manual", "sync"]}) + + +def test_postgres_enum_named_like_set_keeps_its_unseen_values(): + """PG 枚举 setup_status 不是 MySQL 的 SET:列定义里这次没出现的取值也写进码值(含义待填)。""" + settings = ProfileSettings(enabled=True) + source = type("S", (), {"id": "s1", "name": "shop", "kind": "postgresql", "options": {}})() + tables = {"jobs": {"columns": [{"name": "stage", "type": "setup_status", + "enum_values": ["draft", "running", "done"]}]}} + job = catalog_profile.ProfileContext(source=source, settings=settings, dialect=SqlDialect("postgresql"), + runner=None, tables=tables, masked=set()) + job.sizes["jobs"] = TableSize(10, "count") + result = catalog_profile.TableResult(table="jobs") + cx = catalog_profile._TableCx(job, "jobs", result, catalog_profile.TableHooks()) + + async def run(sql: str, **_kw: Any) -> QueryResult: + return QueryResult(columns=["v", "n"], rows=[["draft", 6], ["done", 4]], row_count=2, truncated=False, + elapsed_ms=1, sql=sql) + cx.run = run # type: ignore[method-assign] + out = catalog_profile._TableOutcome() + asyncio.run(catalog_profile._check_codes(cx, "jobs", "stage", out, result)) + [codes] = out.codes + assert codes.declared == ["running"] and "列定义里另有 1 个取值" in codes.note + + +@pytest.mark.parametrize("col, ok", [ + ({"type": "VARCHAR2(20 CHAR)"}, True), + ({"type": "VARCHAR2(200 CHAR)"}, False), + ({"type": "CHAR(2)"}, True), + ({"type": "ENUM('paid','refund')"}, True), + ({"type": "SET('a','b')"}, True), + ({"type": "order_status", "enum_values": ["a", "b"]}, True), + ({"type": "NUMBER(10, 0)"}, True), + ({"type": "NUMBER(10, 2)"}, False), + ({"type": "TIMESTAMP(3)"}, False), + ({"type": "INTERVAL DAY(2) TO SECOND(6)"}, False), +]) +def test_code_type_accepts_enum_and_char_semantics(col, ok): + assert catalog_profile._code_type_ok(col.get("type"), col.get("enum_values")) is ok + + +def test_enum_columns_are_code_candidates_whatever_their_name(): + """ENUM 的取值列表有限,直接算码值候选:pay_way 这种名字不像码值的列以前被漏掉。""" + meta = {"columns": [{"name": "id", "type": "INTEGER"}, {"name": "pay_way", "type": "ENUM('cash','card')", + "enum_values": ["cash", "card"]}, + {"name": "pay_note", "type": "VARCHAR(20)"}], + "primary_key": ["id"]} + assert catalog_profile._code_columns(meta, {}) == ["pay_way"] + + +# ---------------------------------------------------------------- 探查:类型写法、取值列表、引号、索引首列 + + +@pytest.mark.parametrize("dialect, col_type, text", [ + (mysql.dialect(), mysql.ENUM("paid", "refund"), "ENUM('paid','refund')"), + (mysql.dialect(), mysql.TINYINT(display_width=1), "TINYINT(1)"), + (oracle.dialect(), oracle.VARCHAR2(20), "VARCHAR2(20 CHAR)"), + # Oracle 的 VARCHAR2 反射回来是通用的 VARCHAR(容器集成测试实测):照数据字典写成 VARCHAR2 + (oracle.dialect(), VARCHAR(10), "VARCHAR2(10 CHAR)"), + (oracle.dialect(), oracle.NUMBER(10, 0), "NUMBER(10, 0)"), + (postgresql.dialect(), postgresql.ENUM("a", "b", name="order_status"), "order_status"), + # PG 的域编译出来只是域名,名字里的词(set、UPDATE 里的 DATE)会让它被认错:记它底下的类型 + (postgresql.dialect(), postgresql.DOMAIN("settlement_amount", Numeric(12, 2)), "NUMERIC(12, 2)"), + (postgresql.dialect(), postgresql.DOMAIN("update_day", postgresql.DOMAIN("biz_day", Date())), "DATE"), + (sqlite.dialect(), sqlite.DATETIME(), "DATETIME"), +]) +def test_reflected_type_is_compiled_for_the_dialect(dialect, col_type, text): + assert introspect.type_text(col_type, dialect) == text + + +def test_type_that_cannot_compile_falls_back_to_str(): + from sqlalchemy.types import NullType + + assert introspect.type_text(NullType(), sqlite.dialect()) == "NULL" + assert introspect.type_text(None, sqlite.dialect()) == "" + + +def test_column_entry_records_enum_values_and_oracle_quoted_names(): + ora = oracle.dialect() + entry = introspect.column_entry({"name": quoted_name("level", quote=True), "type": oracle.VARCHAR2(10), + "nullable": True, "comment": None}, ora) + assert entry == {"name": "level", "type": "VARCHAR2(10 CHAR)", "nullable": True, "comment": None, + "quote_name": True} + plain = introspect.column_entry({"name": "status", "type": mysql.ENUM("a", "b"), "nullable": False}, mysql.dialect()) + assert plain["enum_values"] == ["a", "b"] and "quote_name" not in plain + members = introspect.column_entry({"name": "tags", "type": mysql.SET("x", "y")}, mysql.dialect()) + assert members["enum_values"] == ["x", "y"] and members["type"] == "SET('x','y')" + + +def test_domain_over_an_enum_keeps_the_enum_values(): + status = postgresql.ENUM("draft", "done", name="setup_status") + entry = introspect.column_entry({"name": "stage", "type": postgresql.DOMAIN("stage_kind", status)}, + postgresql.dialect()) + assert entry["type"] == "setup_status" and entry["enum_values"] == ["draft", "done"] + + +def test_index_first_columns_are_recorded(): + indexes = [{"column_names": ["paid_at", "id"]}, {"column_names": [None]}, {"column_names": ["store_id"]}, + {"column_names": ["paid_at"]}] + assert introspect.index_first_columns(indexes, ["id"]) == ["id", "paid_at", "store_id"] + + +# ---------------------------------------------------------------- IN 列表的字面量按被指向列的类型族 + + +def test_key_literals_follow_the_parent_family(): + assert catalog_profile.key_literals([12, 7], "number", parent="text", child_type="INTEGER") == ( + ["'12'", "'7'"], 0, 0) + # 父列是数、子列是文字:只拼整数写法;'A1' 按类型就对不上整数父键,记进 unmatchable(调用方算作没对上) + assert catalog_profile.key_literals(["12", "A1", "-3", "A1 "], "text", parent="number", + child_type="VARCHAR(10)") == (["12", "-3"], 1, 0) + # 父列是时间的关系整条不核对(_check_relation 先跳过),这里不收 + with pytest.raises(ValueError): + catalog_profile.key_literals([1, 2], "number", parent="time", child_type="INTEGER") + # 父列类型不知道:照子表取回的值的类型写(以前的做法) + assert catalog_profile.key_literals([3, "x"], None, parent=None, child_type=None) == (["3", "'x'"], 0, 0) + + +def test_key_literals_count_unsafe_values_but_not_duplicate_spellings(): + """不能安全拼进 SQL 的值记进 unsafe(备注写「类型不支持核对,未计入」);'0012' 和 '12' 写成同一个字面量,只算一个, + 不能被当成「不支持核对」。""" + found = catalog_profile.key_literals(["0012", "12", "a\\b", True], "text", parent="text", child_type="TEXT") + assert found == (["'0012'", "'12'"], 0, 2) + found = catalog_profile.key_literals(["0012", "12"], "text", parent="number", child_type="TEXT") + assert found == (["12"], 0, 0) + + +_REL = {"id": "r1", "columns": ["customer_ref"], "to_table": "customers", "to_columns": ["id"], "source": "llm", + "status": "proposed"} + + +def _relation_cx(values: list[Any], parent_type: str, matched: int): + """orders.customer_ref(文字)→ customers.id(parent_type)的关系核对用的执行状态:取值查询回 values,父表的核对 + 查询回 matched,唯一性计数回「有重复」。seen 记下发出的每条 SQL。""" + settings = ProfileSettings(enabled=True) + source = type("S", (), {"id": "s1", "name": "shop", "kind": "postgresql", "options": {}})() + tables = {"orders": {"columns": [{"name": "id", "type": "INTEGER"}, {"name": "customer_ref", "type": "TEXT"}], + "primary_key": ["id"]}, + "customers": {"columns": [{"name": "id", "type": parent_type}], "primary_key": ["id"]}} + job = catalog_profile.ProfileContext(source=source, settings=settings, dialect=SqlDialect("postgresql"), + runner=None, tables=tables, masked=set()) + job.sizes.update({"orders": TableSize(130, "count"), "customers": TableSize(3, "count")}) + cx = catalog_profile._TableCx(job, "orders", catalog_profile.TableResult(table="orders"), + catalog_profile.TableHooks()) + seen: list[str] = [] + + async def run(sql: str, **_kw: Any) -> QueryResult: + seen.append(sql) + if "SELECT DISTINCT" in sql: + rows = [[v] for v in values] + return QueryResult(columns=["k"], rows=rows, row_count=len(rows), truncated=False, elapsed_ms=1, sql=sql, + column_types={"k": "text"}) + if " IN (" in sql: + return _result(matched) + return QueryResult(columns=["a", "b"], rows=[[130, len(values)]], row_count=1, truncated=False, elapsed_ms=1, + sql=sql) + cx.run = run # type: ignore[method-assign] + return cx, seen + + +@pytest.mark.parametrize("values, parent_type, coverage, sample, matched", [ + # 九成是散客代号、只有 1 个整数写法对得上:以前分母里拿掉了对不上类型的,测成 100% + (["WALKIN", "1"] + [f"GUEST{i}" for i in range(30)], "INTEGER", round(1 / 32, 4), 32, 1), + # 全都不是整数写法:一条 IN 都不用发,覆盖率 0%,照样是「可能不是这条关系」,不是「无法核对」 + (["WALKIN", "GUEST1"], "INTEGER", 0.0, 2, 0), + (["1", "2"], "INTEGER", 1.0, 2, 2), +]) +def test_relation_coverage_counts_type_mismatches_as_unmatched(values, parent_type, coverage, sample, matched): + cx, seen = _relation_cx(values, parent_type, matched) + out, result = catalog_profile._TableOutcome(), catalog_profile.TableResult(table="orders") + asyncio.run(catalog_profile._check_relation(cx, "orders", _REL, out, result)) + [rel] = out.relations + assert (rel.coverage, rel.sample, rel.matched, rel.holds) == (coverage, sample, matched, coverage == 1.0) + if coverage < 1: + assert "可能不是这条关系" in rel.note and "未计入" not in rel.note + if sample > matched and matched == 0: + assert not any("IN (" in sql for sql in seen), "没有整数写法的键值:不发父表的核对查询" + + +def test_time_keys_are_not_checked_and_send_no_query(): + cx, seen = _relation_cx(["2026-09-01"], "DATE", 0) + out, result = catalog_profile._TableOutcome(), catalog_profile.TableResult(table="orders") + with pytest.raises(catalog_profile._Skip) as skipped: + asyncio.run(catalog_profile._check_relation(cx, "orders", _REL, out, result)) + assert skipped.value.reason == "unsupported" and "日期时间" in str(skipped.value) + assert "两端类型不同" not in str(skipped.value) + assert seen == [], "时间键不核对,一条查询都不发" + + +def test_char_keys_lose_their_padding(): + """PG、Oracle 的 CHAR(n) 取回来带尾随空格:拼成 'A ' 去和 VARCHAR 父列比就对不上。""" + lits = catalog_profile.key_literals(["A ", "BC"], "text", parent="text", child_type="CHAR(2)").literals + assert lits == ["'A'", "'BC'"] + lits = catalog_profile.key_literals(["A "], "text", parent="text", child_type="VARCHAR(2)").literals + assert lits == ["'A '"] + + +# ---------------------------------------------------------------- Oracle 小写带引号的对象 + + +def test_oracle_quote_name_forces_quotes_and_keeps_case(): + ora = SqlDialect("oracle") + meta = {"schema": "sales", "quote_name": True, + "columns": [{"name": "level", "type": "VARCHAR2(10 CHAR)", "quote_name": True}, + {"name": "amount", "type": "NUMBER(10, 2)"}]} + assert ora.table(meta, "visits") == 'sales."visits"' + assert ora.column(meta, "level") == '"level"' and ora.column(meta, "amount") == "amount" + stats = ora.stats_sql(meta, "visits") + assert "TABLE_NAME = 'visits'" in stats and "OWNER = 'SALES'" in stats + # 没有 quote_name 的照旧:不加引号,Oracle 自己转大写 + assert ora.table({"schema": None}, "visits") == "visits" + assert "TABLE_NAME = 'VISITS'" in ora.stats_sql({}, "visits") + + +# ---------------------------------------------------------------- 统计信息过期:再数一次确认 + + +def _result(value: Any) -> QueryResult: + return QueryResult(columns=["n"], rows=[[value]], row_count=1, truncated=False, elapsed_ms=1, sql="") + + +def _runner(answers: list[Any], seen: list[str]): + async def run(sql: str, **_kw: Any) -> QueryResult: + seen.append(sql) + return _result(answers.pop(0)) + return run + + +@pytest.mark.parametrize("stats, counted, expected", [ + (500, 500, TableSize(500, "count")), # 统计信息说小、数过也小:准数 + (500, 100_001, TableSize(None, "count", at_least=100_001)), # 统计信息过期:其实是大表 + (5_000_000, None, TableSize(5_000_000, "stats")), # 统计信息说大:不必再数 +]) +def test_small_stats_are_confirmed_with_a_bounded_count(stats, counted, expected): + seen: list[str] = [] + answers = [stats] + ([counted] if counted is not None else []) + size, skips = asyncio.run(catalog_profile.estimate_size(_runner(answers, seen), SqlDialect("postgresql"), + {"schema": None}, "visits", max_scan_rows=100_000)) + assert size == expected and skips == [] + assert len(seen) == (2 if counted is not None else 1) + if counted is not None: + assert "LIMIT 100001" in seen[1] + + +# ---------------------------------------------------------------- 日期列:排除零日期和哨兵日期 + + +@pytest.mark.parametrize("kind", DIALECTS) +def test_min_max_excludes_invalid_dates_and_counts_them(kind): + d = SqlDialect(kind) + sql = d.min_max_sql("t", [("paid_at", False), ("biz_day", True)]) + # 日期类型的列按日期字面量比,文字存的日期按字符串比(PG 上 varchar 和 date 比较直接报错) + lo, hi = d.date_literal("0001-01-02"), d.date_literal("9999-12-31") + assert f"paid_at >= {lo}" in sql and f"paid_at < {hi}" in sql + assert "biz_day >= '0001-01-02'" in sql and "biz_day < '9999-12-31'" in sql + assert sql.count("SUM(CASE WHEN") == 2 + + +def _dates_cx(row: list[Any], kind: str = "mysql"): + settings = ProfileSettings(enabled=True) + source = type("S", (), {"id": "s1", "name": "shop", "kind": kind, "options": {}})() + job = catalog_profile.ProfileContext(source=source, settings=settings, dialect=SqlDialect(kind), + runner=None, tables={"orders": {"columns": [ + {"name": "paid_at", "type": "DATETIME"}]}}, masked=set()) + job.sizes["orders"] = TableSize(10, "count") + result = catalog_profile.TableResult(table="orders") + cx = catalog_profile._TableCx(job, "orders", result, catalog_profile.TableHooks()) + + async def run(sql: str, **_kw: Any) -> QueryResult: + return QueryResult(columns=["a", "b", "c"], rows=[row], row_count=1, truncated=False, elapsed_ms=1, sql=sql) + cx.run = run # type: ignore[method-assign] + return cx, result + + +def test_dates_note_says_how_many_invalid_values_were_left_out(): + cx, result = _dates_cx(["2026-01-03 08:00:00", "2026-09-30 21:00:00", 4]) + out = catalog_profile._TableOutcome() + asyncio.run(catalog_profile._check_dates(cx, "orders", ["paid_at"], 1, out, result)) + assert result.date_ranges == [{"column": "paid_at", "min": "2026-01-03 08:00:00", "max": "2026-09-30 21:00:00", + "invalid": 4}] + assert "另有 4 个无效日期" in out.business_date.note and "0000-00-00" in out.business_date.note + + +def test_only_invalid_dates_is_no_data(): + cx, result = _dates_cx([None, None, 9]) + with pytest.raises(catalog_profile._Skip) as e: + asyncio.run(catalog_profile._check_dates(cx, "orders", ["paid_at"], 1, catalog_profile._TableOutcome(), + result)) + assert e.value.reason == "no_data" and "9 个无效日期" in e.value.detail + + +# ---------------------------------------------------------------- 取值观察要用的方言方法 + + +@pytest.mark.parametrize("kind, literal, day, sample", [ + ("mysql", "DATE '2026-09-01'", "DATE(paid_at)", None), + ("mariadb", "DATE '2026-09-01'", "DATE(paid_at)", None), + ("postgresql", "DATE '2026-09-01'", "CAST(paid_at AS DATE)", "orders TABLESAMPLE SYSTEM (1.5)"), + ("oracle", "DATE '2026-09-01'", "TRUNC(paid_at)", "orders SAMPLE BLOCK (1.5)"), + ("sqlite", "'2026-09-01'", "substr(paid_at, 1, 10)", None), +]) +def test_observation_dialect_methods(kind, literal, day, sample): + d = SqlDialect(kind) + assert d.date_literal(_dt.date(2026, 9, 1)) == literal == d.date_literal("2026-09-01") + assert d.date_literal("2026-09-01", text=True) == "'2026-09-01'" + assert d.day_trunc("paid_at") == day + assert d.sample_from("orders", 1.5) == sample + + +@pytest.mark.parametrize("bad", ["2026-9-1", "2026-09-01'; --", "0000-00-00", 20260901, None]) +def test_date_literal_refuses_anything_but_a_real_date(bad): + with pytest.raises(ValueError): + SqlDialect("postgresql").date_literal(bad) + + +@pytest.mark.parametrize("bad", [0, -1, 100, 101, float("nan")]) +def test_sample_percent_must_be_between_0_and_100(bad): + with pytest.raises(ValueError): + SqlDialect("postgresql").sample_from("orders", bad) + + +# ---------------------------------------------------------------- 说明文字 + + +def test_notes_say_storage_order_not_sampling(): + note = catalog_profile._relation_note(sample=2000, matched=2000, coverage=1.0, scan_cap=100_000, skipped_values=0, + unmatchable=0, parent=(True, "主键"), child=(False, "count"), holds=True, + collation=None) + assert "按存储顺序只看前 100000 行" in note and "抽样" not in note + assert catalog_profile.FILTER_SAMPLED.format(rows="100", share="1%").startswith("剖析核对:按存储顺序取前 100 行统计") + + +def test_mysql_notes_mention_the_collation(): + note = catalog_profile._relation_note(sample=10, matched=10, coverage=1.0, scan_cap=None, skipped_values=0, + unmatchable=0, parent=(True, "主键"), child=(False, "count"), holds=True, + collation=catalog_profile.MYSQL_COLLATION_NOTE) + assert "排序规则" in note and "不分大小写" in note diff --git a/backend/tests/test_public_url.py b/backend/tests/test_public_url.py new file mode 100644 index 00000000..c13667dc --- /dev/null +++ b/backend/tests/test_public_url.py @@ -0,0 +1,169 @@ +"""对外访问地址只有一个说法(04 细案 B4,core/public_url.py)。 + +以前有两个:环境变量 AGENTLAB_PUBLIC_URL 只管单点登录的回调和来源核对,设置里 notify 一组的 public_base_url 只管通知里的 +链接——只配了环境变量的部署,通知里不带链接。现在全站经 public_base_url() 取: + +- 环境变量优先,其次设置,都没有是 None; +- 设置的值缓存在进程里(中间件按请求核对来源,不能每次查库),改设置的接口写完就刷新,不用重启; +- 环境变量设了:设置页显示「由环境变量指定」(GET /api/settings 的 public_url 一组),改成别的值回 409、带 code; +- 通知、定时任务 Webhook 的链接,单点登录的回调地址,非 GET 请求的 Origin 核对都用它。 +""" +from __future__ import annotations + +import pytest +from httpx import ASGITransport, AsyncClient +from sqlalchemy import delete + +from app.api.settings import NOTIFY_GROUP +from app.core import public_url +from app.core.config import Settings, settings +from app.db.base import SessionLocal +from app.db.models import Setting +from app.main import app as main_app +from tests.fakes import auth_env + +ENV_URL = "https://agentlab.example.com" +SET_URL = "https://intra.example.net/agentlab" + + +@pytest.fixture(autouse=True) +async def clean(monkeypatch): + monkeypatch.setattr(settings, "public_url", "") + async with SessionLocal() as session: + await session.execute(delete(Setting).where(Setting.key == NOTIFY_GROUP)) + await session.commit() + public_url.remember(None) + yield + async with SessionLocal() as session: + await session.execute(delete(Setting).where(Setting.key == NOTIFY_GROUP)) + await session.commit() + public_url.remember(None) + + +def client() -> AsyncClient: + return AsyncClient(transport=ASGITransport(app=main_app), base_url="http://test") + + +async def put_address(c: AsyncClient, value: str | None): + return await c.put("/api/settings", json={"values": {NOTIFY_GROUP: {"public_base_url": value}}}) + + +# ---------------------------------------------------------------- 取值的先后 + + +def test_env_wins_over_the_setting(monkeypatch): + public_url.remember(SET_URL) + assert public_url.public_base_url() == SET_URL and not public_url.env_locked() + monkeypatch.setattr(settings, "public_url", ENV_URL) + assert public_url.public_base_url() == ENV_URL and public_url.env_locked() + public_url.remember(None) + assert public_url.public_base_url() == ENV_URL, "设置清空了也还是环境变量的" + monkeypatch.setattr(settings, "public_url", "") + assert public_url.public_base_url() is None, "两处都没有:None(通知里不附链接,回调按请求的 Host 拼)" + + +def test_remember_normalizes_and_drops_broken_values(): + public_url.remember(" https://agentlab.example.com/ ") + assert public_url.public_base_url() == "https://agentlab.example.com" + for broken in ("agentlab.example.com", "ftp://files.example.com", "https://a.example.com/?x=1", 8080, ""): + public_url.remember(broken) + assert public_url.public_base_url() is None, broken + + +def test_env_value_is_normalized_and_checked_at_startup(monkeypatch): + monkeypatch.setenv("AGENTLAB_PUBLIC_URL", " https://agentlab.example.com/ ") + assert Settings().public_url == "https://agentlab.example.com" + monkeypatch.setenv("AGENTLAB_PUBLIC_URL", "") + assert Settings().public_url == "" + # 写错了启动时就报出来:以前拼出 agentlab.example.com/api/auth/oidc/callback 这样的回调,到登录时才发现 + for broken in ("agentlab.example.com", "https://agentlab.example.com/?from=env"): + monkeypatch.setenv("AGENTLAB_PUBLIC_URL", broken) + with pytest.raises(ValueError, match="AGENTLAB_PUBLIC_URL"): + Settings() + + +async def test_refresh_reads_the_stored_setting(): + async with SessionLocal() as session: + session.add(Setting(key=NOTIFY_GROUP, value={"public_base_url": SET_URL + "/"})) + await session.commit() + assert public_url.public_base_url() is None, "缓存还没对齐" + assert await public_url.refresh() == SET_URL + assert public_url.public_base_url() == SET_URL + + +# ---------------------------------------------------------------- 设置接口 + + +async def test_saving_the_setting_refreshes_the_cache_without_restart(): + async with client() as c: + assert (await put_address(c, SET_URL + "/")).status_code == 200 + assert public_url.public_base_url() == SET_URL + body = (await c.get("/api/settings")).json() + assert body["public_url"] == {"value": SET_URL, "env_locked": False} + assert (await put_address(c, None)).status_code == 200 + assert public_url.public_base_url() is None + + +async def test_env_locks_the_setting(monkeypatch): + monkeypatch.setattr(settings, "public_url", ENV_URL) + async with client() as c: + body = (await c.get("/api/settings")).json() + assert body["public_url"] == {"value": ENV_URL, "env_locked": True} + assert body[NOTIFY_GROUP] == {"public_base_url": None}, "notify 一组照旧是库里存的那一份" + r = await put_address(c, SET_URL) + assert r.status_code == 409, r.text + assert r.json()["code"] == "public_url_env_locked" and r.json()["field"] == "notify.public_base_url" + assert "AGENTLAB_PUBLIC_URL" in r.json()["detail"] + # 没改这一项(整组原样写回)不算改:不挡同一请求里别的设置 + assert (await put_address(c, None)).status_code == 200 + # 只读的 public_url 一组写不进库 + r = await c.put("/api/settings", json={"values": {"public_url": {"value": SET_URL, "env_locked": False}}}) + assert r.status_code == 200 + assert (await c.get("/api/settings")).json()["public_url"] == {"value": ENV_URL, "env_locked": True} + async with SessionLocal() as session: + assert await session.get(Setting, "public_url") is None + + +# ---------------------------------------------------------------- 用到它的地方 + + +async def test_notifications_link_with_the_env_address_alone(monkeypatch): + """只配了环境变量的部署,通知里也带链接(以前只认设置里的那一项)。""" + from app.notify import service + + monkeypatch.setattr(settings, "public_url", ENV_URL) + async with SessionLocal() as session: + assert await service._base_url(session) == ENV_URL + monkeypatch.setattr(settings, "public_url", "") + async with SessionLocal() as session: + session.add(Setting(key=NOTIFY_GROUP, value={"public_base_url": SET_URL})) + await session.commit() + async with SessionLocal() as session: + assert await service._base_url(session) == SET_URL + + +async def test_oidc_callback_uses_the_one_address(monkeypatch): + from starlette.requests import Request + + from app.api.auth import _redirect_uri + + request = Request({"type": "http", "method": "GET", "path": "/api/auth/oidc/start", "headers": [], + "scheme": "http", "server": ("10.0.0.5", 8000), "query_string": b""}) + assert _redirect_uri(request) == "http://10.0.0.5:8000/api/auth/oidc/callback", "都没有:按请求的地址拼" + public_url.remember(SET_URL) + assert _redirect_uri(request) == f"{SET_URL}/api/auth/oidc/callback", "只在设置里填了的也按它拼" + monkeypatch.setattr(settings, "public_url", ENV_URL) + assert _redirect_uri(request) == f"{ENV_URL}/api/auth/oidc/callback" + + +async def test_origin_check_accepts_the_one_address(): + await auth_env.reset() + try: + c = await auth_env.turn_on_local() + r = await c.post("/api/groups", json={"name": "ops"}, headers={"Origin": "https://intra.example.net"}) + assert r.status_code == 403 and r.json()["code"] == "csrf_origin" + public_url.remember(SET_URL) + r = await c.post("/api/groups", json={"name": "ops"}, headers={"Origin": "https://intra.example.net"}) + assert r.status_code == 201, r.text + finally: + await auth_env.reset() diff --git a/backend/tests/test_publish_plan.py b/backend/tests/test_publish_plan.py index e38b8d97..4fcdf785 100644 --- a/backend/tests/test_publish_plan.py +++ b/backend/tests/test_publish_plan.py @@ -186,6 +186,30 @@ async def test_a_failed_trial_blocks_the_plan(client, tmp_path): assert "plan.untrialed" not in codes +async def test_a_rejected_trial_is_a_warning_not_a_failure(client, tmp_path): + """最近一次试运行在某一步被驳回(cancelled + end_reason=rejected):那是人的决定,不说成「试运行失败了」、不挡发布, + 照实提醒没走完(05-已知遗留 (a))。""" + from sqlalchemy import update + + bp = await _board_from_run(client, tmp_path) + async with SessionLocal() as session: + trial = await session.get(Run, bp["last_trial_run_id"]) + node_id = trial.graph["nodes"][-1]["id"] + label = trial.graph["nodes"][-1]["data"]["label"] + await session.execute(update(Run).where(Run.id == trial.id).values( + status="cancelled", end_reason="rejected", error_node_id=node_id, error="人工驳回:不对")) + await session.commit() + plan = (await client.post(f"/api/blueprints/{bp['id']}/publish-plan", json={"level": "published"})).json() + codes = {p["code"]: p for p in plan["problems"]} + assert "plan.trial_failed" not in codes and "plan.untrialed" not in codes + rejected = codes["plan.trial_rejected"] + assert rejected["level"] == "warning" and rejected["message"] == f"最近一次试运行在「{label}」被驳回、没有走完" + assert rejected["plain"]["title"] == "最近一次试运行被驳回、没有走完" + assert plan["ok"] is True, [p["code"] for p in plan["problems"] if p["level"] == "error"] + summary = {x["step"]: x["text"] for x in plan["summary"]} + assert "被驳回、没有走完" in summary["publish_workflow"] + + async def test_a_process_with_an_unbound_step_stops_at_preflight(client): wf_id = await s.published_workflow(s.expense_graph(), f"报销-{uuid.uuid4().hex[:4]}", description="报销审批") bp = (await client.post("/api/blueprints", json={"source": {"kind": "workflow_version", "id": wf_id}, @@ -194,8 +218,12 @@ async def test_a_process_with_an_unbound_step_stops_at_preflight(client): assert (await client.post(f"/api/blueprints/{bp['id']}/instantiate")).status_code == 200 plan = (await client.post(f"/api/blueprints/{bp['id']}/publish-plan", json={"level": "published"})).json() assert plan["ok"] is False - unbound = [p for p in plan["problems"] if p["code"] == "app.slot_unbound"] - assert unbound and unbound[0]["plain"]["who"] == "you" and "谁来做" in unbound[0]["plain"]["action"] + # 自由形按导入的图列出「谁来做」(05-已知遗留 (d)):会签的 finance、超时升级的 director 各一行,没绑人报在那一行上; + # 应用检查说的是同一件事(app.slot_unbound),不再重复列——它指的「谁来做」以前在自由形蓝图上不存在 + unbound = {p["path"]: p for p in plan["problems"] if p["code"] == "bp.people_unbound"} + assert set(unbound) == {"/people/0/who", "/people/1/who"} + assert unbound["/people/0/who"]["plain"]["who"] == "you" and "谁来做" in unbound["/people/0/who"]["plain"]["action"] + assert "app.slot_unbound" not in {p["code"] for p in plan["problems"]} # 流程类不跑第一次;已发布的那一版按哈希复用,不另存 assert plan["options"]["first_run"] is False summary = {x["step"]: x["text"] for x in plan["summary"]} @@ -231,6 +259,59 @@ async def erp_write(): await registry.refresh_cache(session) +async def test_a_freeform_blueprint_binds_who_does_each_step_of_the_imported_graph(client, erp_write): + """从已发布的「财务审批(岗位 finance)+ 写回(没登记查询)」工作流建自由形蓝图(05-已知遗留 (d)):卡片上按图列出两行 + ——finance 和写入结果不明时的核实人;不选人时发布单在这两行上报 bp.people_unbound,不再有指不到卡片的 app.slot_unbound / + app.reconciler_unbound;选好人之后绑到应用的岗位,发布单不再挡。卡片上只定由谁做,改会签被拒(在画布上改)。""" + from app.blueprints.publish_plan import bindings_of + + graph = {"nodes": [ + s.node("start", "input", "发起", fields=[{"name": "amount", "type": "number"}]), + s.node("review", "human", "财务审批", mode="approve", title="审一下", assignee={"kind": "slot", "slot": "finance"}), + s.node("pay", "tool", "写回报销单", tool=erp_write, args={}), + s.node("done", "output", "完成", fields=[{"name": "结论", "value": "已写回"}])], + "edges": [{"source": "start", "target": "review"}, + {"source": "review", "target": "pay", "sourceHandle": "approved"}, + {"source": "pay", "target": "done"}]} + wf_id = await s.published_workflow(graph, f"写回-{uuid.uuid4().hex[:4]}", description="报销写回") + bp = (await client.post("/api/blueprints", json={"source": {"kind": "workflow_version", "id": wf_id}, + "scope": "shared"})).json() + await s.edit_spec(bp["id"], purpose=s.item("报销审批后写回")) + assert (await client.post(f"/api/blueprints/{bp['id']}/instantiate")).status_code == 200 + out = (await client.get(f"/api/blueprints/{bp['id']}")).json() + rows = out["derived_people"] + assert [(r["kind"], r["slot"], r["path"], r["who"]) for r in rows] == [ + ("slot", "finance", "/people/0/who", None), ("reconciler", "reconciler", "/writeback/reconciler", None)] + assert rows[0]["label"] == "「财务审批」" and rows[0]["detail"] == "「财务审批」处理" + assert rows[1]["detail"] == "「写回报销单」写入结果不明时去核实" + + plan = (await client.post(f"/api/blueprints/{bp['id']}/publish-plan", json={"level": "published"})).json() + assert plan["ok"] is False + codes = [p["code"] for p in plan["problems"]] + assert sorted(p["path"] for p in plan["problems"] if p["code"] == "bp.people_unbound") == [ + "/people/0/who", "/writeback/reconciler"] + assert "app.slot_unbound" not in codes and "app.reconciler_unbound" not in codes + summary_you = [p for p in plan["problems"] if p["code"] == "bp.people_unbound"] + assert all(p["who"] == "you" for p in summary_you) + + refused = await client.patch(f"/api/blueprints/{bp['id']}", json={"ops": [ + {"op": "bp_set", "path": "/people/0/policy", "value": {"kind": "all"}}]}) + assert refused.status_code == 422 and refused.json()["code"] == "bp.template_mismatch" + assert "在画布上改" in refused.json()["detail"] + r = await client.patch(f"/api/blueprints/{bp['id']}", json={"ops": [ + {"op": "bp_set", "path": "/people/0/who", "value": [{"kind": "user", "name": "zhang"}]}, + {"op": "bp_set", "path": "/writeback/reconciler", "value": [{"kind": "group", "name": "财务组"}]}]}) + assert r.status_code == 200, r.text + rows = r.json()["derived_people"] + assert rows[0]["who"] == [{"kind": "user", "name": "zhang"}] and rows[1]["who"] == [{"kind": "group", "name": "财务组"}] + assert not [i for i in r.json()["issues"] if i["code"] == "bp.people_unbound"] + assert bindings_of(await s.blueprint(bp["id"]))["slots"] == { + "finance": [{"kind": "user", "name": "zhang"}], "reconciler": [{"kind": "group", "name": "财务组"}]} + plan = (await client.post(f"/api/blueprints/{bp['id']}/publish-plan", json={"level": "published"})).json() + codes = [p["code"] for p in plan["problems"]] + assert "bp.people_unbound" not in codes and "app.slot_unbound" not in codes and "app.reconciler_unbound" not in codes + + async def test_the_reconciler_comes_from_the_writeback_card(client, erp_write): """审批通过后写回一项没登记查询的能力:结果不明时要人核实。发布单只列一条、指到「写回哪里」卡片上那一项(应用检查说的 核实岗位没绑、写入没人核实是同一件事,不再重复列);在卡片上选了人,绑到应用的核实岗位,这几条都不再有。""" @@ -500,8 +581,8 @@ async def test_problems_are_split_into_three_kinds(client): plan = (await client.post(f"/api/blueprints/{bp['id']}/publish-plan", json={"level": "published"})).json() assert all(p["who"] in ("you", "assistant", "admin") for p in plan["problems"]) assert {p["source"] for p in plan["problems"]} <= {"blueprint", "plan", "workflow", "schedule", "app", "grants"} - unbound = next(p for p in plan["problems"] if p["code"] == "app.slot_unbound") - assert unbound["who"] == "you" and unbound["source"] == "app" + unbound = next(p for p in plan["problems"] if p["code"] == "bp.people_unbound") + assert unbound["who"] == "you" and unbound["source"] == "blueprint" # -------------------------------------------------------------------------- diff --git a/backend/tests/test_recipe_contract.py b/backend/tests/test_recipe_contract.py index 34800e71..03827e7e 100644 --- a/backend/tests/test_recipe_contract.py +++ b/backend/tests/test_recipe_contract.py @@ -370,10 +370,12 @@ def test_accumulate_blockers_reference_recipe_is_eligible(): def test_accumulate_blockers_list_shape_is_unkeyed(): data = json.loads(json.dumps(C01_LIKE)) - data["sheets"][0]["context"] = [{}] # 即使配了统计期,列表写入的表也不能累积 + data["sheets"][0]["context"] = [{}] # 配了统计期,列表却没说每一行属于哪一期:不能累积 got = T.accumulate_blockers(T.Recipe.model_validate(data)) assert [(p.code, p.path) for p in got] == [("accumulate_unkeyed", "/tables/0"), ("accumulate_unkeyed", "/tables/1")] - assert all("列表形态的表目前只支持每期替换" in p.message for p in got) + assert "没有指定每一行属于哪一期" in got[0].message + # 合计行另存的表:各期的合计项相同,主键分不开各期(期 5 列表可以累积以后仍然不行) + assert "合计行、小计行另存的表不能按期累积" in got[1].message assert "「月报」" in got[0].message and "「月报_表内合计」" in got[1].message # c01 的草稿本来就没有统计期:两条都报,统计期那条在前(起草卡片取第一条的人话) got = T.accumulate_blockers(T.Recipe.model_validate(C01_LIKE)) @@ -479,13 +481,15 @@ def test_closed_value_sets(): assert {"union_rows", "union_pk", "union_period"} <= set(_literal(T.CheckResult, "kind")) assert "ignored" in typing.get_args(T.Role) and "ignored_column" in typing.get_args(T.Role) assert typing.get_args(T.ExcludedReason) == ( - "hidden_excluded", "blank_skipped", "ignored_rows", "ignored_outside", "after_stop", "total_not_kept") + "hidden_excluded", "blank_skipped", "ignored_rows", "ignored_outside", "after_stop", "total_not_kept", + "subtotal") assert _literal(T.FixAnchor, "kind") == ("problem", "recipe_problem", "sheet_renamed") assert _literal(T.Anchor, "kind") == ( "header", "row_label", "section_title", "total_word", "axis", "after_title", "outside_text") assert _literal(T.EditResult, "kind") == ("fix", "selection") assert _literal(T.ChangeClass, "change") == ("same", "compatible", "retire", "semantic") - assert T.FIX_KINDS == ("remove_label", "add_label", "edit_members", "rename_title", "declare_total", + assert T.FIX_KINDS[-1] == "declare_subtotals" # 期 5 加在末尾 + assert T.FIX_KINDS[:-1] == ("remove_label", "add_label", "edit_members", "rename_title", "declare_total", "ignore_cells", "declare_placeholder", "declare_hidden", "rename_sheet") assert T.SELECTION_AS == ("list", "crosstab", "segment", "derived", "section_title", "ignore_rows", "ignore_columns", "ignore_outside") diff --git a/backend/tests/test_recipe_engine_messages.py b/backend/tests/test_recipe_engine_messages.py index 26dd3d84..c2013ab1 100644 --- a/backend/tests/test_recipe_engine_messages.py +++ b/backend/tests/test_recipe_engine_messages.py @@ -318,6 +318,30 @@ def c_total_row_blank(mp): return _run(sales_book(edit=edit), LIST), [N] +def c_subtotal_blank(mp): + """期 5:穿插的小计行数字格全空:没有一列能核对,结构类。""" + data = list_recipe() + data["sheets"][0]["blocks"][0]["rows"]["subtotals"] = {"label_column": "地区", "pick": "小计"} + + def edit(ws): + ws["D4"] = N + ws.insert_rows(6) + ws["A6"] = "小计" # 第 6 行是小计:标签在,数字格全空 + return _run(sales_book(edit=edit), data), [N] + + +def c_subtotal_suspect(mp): + """期 5:没配小计时,首列以合计字样开头、其余文字列空着的行像夹在中间的小计:照常导入,要人确认。""" + data = list_recipe() + + def edit(ws): + ws["D4"] = N + ws.insert_rows(6) + ws["A6"] = "小计" + ws["D6"] = N + return _run(sales_book(edit=edit), data), [N] + + def c_value_not_integer(mp): return _run(_flow(_set(C11=9876.54)), FLOW), [9876.54] @@ -423,6 +447,7 @@ NEXT_STEPS = { "hidden_cols": "在配方面板的「隐藏列」中选择「照常导入」", "total_row_missing": "在配方面板的「合计行」中取消「有合计行」", "total_row_blank": "在配方面板的「合计行」中取消「有合计行」", + "subtotal_blank": "在配方面板的「小计行」中取消「有小计行」", } diff --git a/backend/tests/test_recipe_engine_p3.py b/backend/tests/test_recipe_engine_p3.py index a4c6ad63..f827a6bf 100644 --- a/backend/tests/test_recipe_engine_p3.py +++ b/backend/tests/test_recipe_engine_p3.py @@ -119,7 +119,9 @@ def test_problem_fix_table_follows_spec_3_1(): "label_unparsed": "declare_total", "value_not_number": "declare_placeholder", "axis_extra_cells": "ignore_cells", "column_extra": "ignore_cells", "row_unclaimed": "ignore_cells", "outside_number": "ignore_cells", "hidden_rows": "declare_hidden", "hidden_cols": "declare_hidden", - "sheet_missing": "rename_sheet"} + "sheet_missing": "rename_sheet", + # 期 5:没配小计的列表里像小计的行 + "subtotal_suspect": "declare_subtotals"} assert set(recipe_engine.PROBLEM_FIX.values()) <= set(FIX_KINDS) # edit_members 只来自静态校验;cell_unclaimed、rows_after_stop 没有修复按钮 assert "edit_members" not in recipe_engine.PROBLEM_FIX.values() @@ -367,9 +369,17 @@ def _column_extra(): return sales_book(edit=edit), LIST +def _subtotal_suspect(): + def edit(ws): + ws.insert_rows(6) + ws["A6"], ws["D6"], ws["E6"] = "小计", N, N + 1 + return sales_book(edit=edit), LIST + + @pytest.mark.parametrize("code", sorted(recipe_engine.PROBLEM_FIX)) def test_fix_args_never_carry_cell_values(code): - book, recipe = _column_extra() if code == "column_extra" else _crosstab_cases()[code] + book, recipe = (_column_extra() if code == "column_extra" else _subtotal_suspect() if code == "subtotal_suspect" + else _crosstab_cases()[code]) ex = run(book, recipe) hits = [p for p in ex.problems if p.code == code and p.fix_args is not None] assert hits, f"{code}:{[(p.code, p.fix, p.message) for p in ex.problems]}" diff --git a/backend/tests/test_recipe_imports.py b/backend/tests/test_recipe_imports.py index cdfed879..74b54535 100644 --- a/backend/tests/test_recipe_imports.py +++ b/backend/tests/test_recipe_imports.py @@ -260,7 +260,7 @@ def fake_pipeline(state: FakeState) -> Pipeline: return Pipeline( validate=validate, recipe_sha256=recipe.recipe_sha256, canonical=recipe.canonical_recipe, - apply_patch=recipe.apply_patch, scan=lambda raw: fake_scan(raw, state), + apply_patch=recipe.apply_patch, scan=lambda raw, _filename: fake_scan(raw, state), scan_to_json=xlsx_scan.scan_to_json, scan_from_json=xlsx_scan.scan_from_json, read_grid=fake_grid, preview=xlsx_cells.preview, execute=execute, run_checks=run_checks, column_null_counts=null_counts, build_notes=build_notes, apply_notes=recipe_notes.apply_notes, detect_facts=detect_facts, draft=draft, diff --git a/backend/tests/test_recipe_notes_p3.py b/backend/tests/test_recipe_notes_p3.py index ba454c55..cfc9a6e8 100644 --- a/backend/tests/test_recipe_notes_p3.py +++ b/backend/tests/test_recipe_notes_p3.py @@ -532,7 +532,8 @@ def test_problems_for_inconsistent_inputs(): def test_list_tables_cannot_be_merged_and_are_problems(): - """列表不能按期累积(P3-SPEC 2.2):真把两期列表并起来,列表合计没有并集说法,记 problems,绝不沿用单期说法。""" + """列表合计行另存的表不能按期累积(accumulate_blockers):真把两期这样的列表并起来,列表合计没有并集说法,记 + problems,绝不沿用单期说法。""" data = deepcopy(LIST_RECIPE) def lpart(span): @@ -544,7 +545,7 @@ def test_list_tables_cannot_be_merged_and_are_problems(): tgt = Recipe.model_validate(data) notes = build_union_notes(tgt, [lpart(AUG), lpart(SEP)], union_tables=derive_tables(tgt)[0], null_counts=None, part_null_counts=None, added=[], retired=[], label_sets=[], gaps=False) - assert any("列表合计行的核对结论不能按期合并" in p for p in notes.problems) + assert any("列表合计行、小计行的核对结论不能按期合并" in p for p in notes.problems) assert not any("list_total" in p for p in notes.problems) # 不露键名 assert "一致" not in notes.tables["销售_表内合计"].comment diff --git a/backend/tests/test_recipe_p5.py b/backend/tests/test_recipe_p5.py new file mode 100644 index 00000000..163a7bf7 --- /dev/null +++ b/backend/tests/test_recipe_p5.py @@ -0,0 +1,537 @@ +"""Excel 期 5(下一轮 03 细案 E):列表按期累积、穿插的小计、表格对象起草、CSV 走配方。 + +不连接口的部分:执行器、核对、静态校验、累积资格、并集物化、起草。夹具全是合成的通用名字(门店一、分区甲、 +订单 A001……),数字是手写的小数,手算得出。接口的端到端在 test_recipe_p5_e2e.py。 +""" +from __future__ import annotations + +import copy +import io +import os +import sqlite3 +import tempfile +from pathlib import Path +from typing import Any + +import pytest +from openpyxl import Workbook +from openpyxl.worksheet.table import Table + +from app.data import csv_grid, raw_store, recipe_checks, recipe_engine, recipe_suggest, table_versions, xlsx_cells +from app.data import xlsx_scan +from app.data.recipe import validate_recipe +from app.data.recipe_accumulate import materialize_union +from app.data.recipe_types import ( + LIST_PERIOD_END, LIST_PERIOD_START, SUBTOTAL_KEY, Recipe, UnionPart, accumulate_blockers, derive_tables, +) + + +def _book(rows: list[tuple[Any, ...]], *, sheet: str = "明细", top: list[tuple[str, Any]] | None = None, + start: int = 1) -> bytes: + wb = Workbook() + ws = wb.active + ws.title = sheet + for ref, value in top or []: + ws[ref] = value + for r, row in enumerate(rows, start=start): + for c, v in enumerate(row, start=1): + if v is not None: + ws.cell(r, c, v) + buf = io.BytesIO() + wb.save(buf) + return buf.getvalue() + + +def _run(raw: bytes, recipe: dict[str, Any], db: str | None = None, filename: str = "明细.xlsx", **kw: Any): + scan = csv_grid.scan_any(raw, filename) + return recipe_engine.execute(Recipe.model_validate(recipe), raw, filename, scan, db, **kw) + + +def _rows(db: str, sql: str) -> list[tuple[Any, ...]]: + conn = sqlite3.connect(db) + try: + return conn.execute(sql).fetchall() + finally: + conn.close() + + +def _checks(db: str, ex: Any, recipe: dict[str, Any]) -> dict[str, Any]: + return {c.id: c for c in recipe_checks.run_checks(db, ex, Recipe.model_validate(recipe))} + + +def _list(columns: list[tuple[str, str]], *, table: str = "明细", sheet: str = "明细", context: bool = False, + **block: Any) -> dict[str, Any]: + out = { + "recipe_format": "agentlab-recipe/2", + "sheets": [{"id": "s1", "match": {"name": sheet}, + "context": [{"id": "统计期"}] if context else [], + "blocks": [{"id": "列表1", "layout": "list", "table": table, + "columns": [{"header": h, "name": h, "type": t} for h, t in columns], **block}]}], + "tables": [{"name": table, "grain": []}], + } + return out + + +# ========================================================================== +# 穿插的小计 +# ========================================================================== + +STORES = [("地区", "门店", "金额", "数量"), + ("分区甲", "门店一", 100, 1), ("分区甲", "门店二", 200, 2), ("小计", None, 300, 3), + ("分区乙", "门店三", 150, 4), ("分区乙", "门店四", 150, 5), ("小计", None, 300, 9), + ("合计", None, 600, 12)] +STORE_COLS = [("地区", "TEXT"), ("门店", "TEXT"), ("金额", "INTEGER"), ("数量", "INTEGER")] + + +def _stores_recipe(**sub: Any) -> dict[str, Any]: + return _list(STORE_COLS, rows={"total_row": {"label_column": "地区", "pick": "合计"}, + "subtotals": {"label_column": "地区", "pick": "小计", **sub}}) + + +def test_subtotal_rows_are_checked_against_their_group_and_kept_out_of_the_data(tmp_path): + db = str(tmp_path / "s.db") + recipe = _stores_recipe() + ex = _run(_book(STORES), recipe, db) + assert ex.ok, [p.message for p in ex.problems] + # 只有 4 行明细进了数据;两行小计、一行合计都不在 + assert _rows(db, 'SELECT "门店", "金额" FROM "明细" ORDER BY rowid') == [ + ("门店一", 100), ("门店二", 200), ("门店三", 150), ("门店四", 150)] + subs = [d for d in ex.derived if d.segment == f"列表1{SUBTOTAL_KEY}"] + assert [(d.cell, d.rowid_first, d.rowid_last) for d in subs] == [ + ("明细!C4", 1, 2), ("明细!D4", 1, 2), ("明细!C7", 3, 4), ("明细!D7", 3, 4)] + [excluded] = [x for x in ex.rows_excluded if x.reason == "subtotal"] + assert excluded.rows == [[4, 4], [7, 7]] and excluded.anchor == "小计" and excluded.block == "列表1" + checks = _checks(db, ex, recipe) + assert checks["T1"].title == "合计行按列求和(块「列表1」)" and checks["T1"].status == "passed" + assert checks["T2"].title == "小计行按各组明细求和(块「列表1」)" and checks["T2"].status == "passed" + assert checks["T2"].checked == 4 + + +def test_a_wrong_subtotal_is_a_mismatch(tmp_path): + rows = list(STORES) + rows[6] = ("小计", None, 300, 10) # 4 + 5 ≠ 10 + db = str(tmp_path / "s.db") + recipe = _stores_recipe() + ex = _run(_book(rows), recipe, db) + checks = _checks(db, ex, recipe) + t2 = checks["T2"] + assert t2.status == "mismatch" and t2.failed == 1 and t2.cells == ["明细!D7"] + assert "按明细求和为 9" in t2.details[0] + + +def test_group_column_scope_only_covers_the_rows_of_the_same_group(tmp_path): + """按组小计:紧挨在小计上面、分组列取值相同的那几行。小计行的标签写在「门店」列,分组按「地区」。 + 同一份数据按「上一行小计之后的全部明细」核对就对不上(350 ≠ 50)。""" + rows = [("地区", "门店", "金额"), ("分区甲", "门店一", 100), ("分区甲", "门店二", 200), + ("分区乙", "门店三", 50), ("分区乙", "小计", 50)] + cols = [("地区", "TEXT"), ("门店", "TEXT"), ("金额", "INTEGER")] + recipe = _list(cols, rows={"subtotals": {"label_column": "门店", "pick": "小计", "scope": "group_column", + "group_column": "地区", "keep_as": "明细_小计"}}) + recipe["tables"].append({"name": "明细_小计", "kind": "reported_total"}) + db = str(tmp_path / "g.db") + ex = _run(_book(rows), recipe, db) + assert ex.ok, [p.message for p in ex.problems] + assert _checks(db, ex, recipe)["T1"].status == "passed" + assert _rows(db, 'SELECT * FROM "明细_小计"') == [("小计", "分区乙", 50)] + assert [c.name for c in derive_tables(Recipe.model_validate(recipe))[0]["明细_小计"]] == ["小计项", "地区", "金额"] + since = copy.deepcopy(recipe) + since["sheets"][0]["blocks"][0]["rows"]["subtotals"].update(scope="since_previous", group_column=None) + since["tables"][1]["name"] = "明细_小计" + db2 = str(tmp_path / "g2.db") + ex2 = _run(_book(rows), since, db2) + assert _checks(db2, ex2, since)["T1"].status == "mismatch" + + +def test_subtotal_validation(): + def problems(**sub: Any) -> list[str]: + data = _stores_recipe(**sub) + _, probs = validate_recipe(data, facts=None, origin="rules") + return [p.code for p in probs] + + assert problems() == [] + assert "subtotal_group_missing" in problems(scope="group_column") + assert "column_unknown" in problems(scope="group_column", group_column="城市") + assert "subtotal_group_missing" in problems(group_column="地区") + assert "period_literal" in problems(pick="2026小计") + bad = _stores_recipe() + bad["sheets"][0]["blocks"][0]["rows"]["subtotals"]["pick"] = "合计" + assert "subtotal_pick_conflict" in [p.code for p in validate_recipe(bad, facts=None, origin="rules")[1]] + + +# ========================================================================== +# 列表按期累积:每一行属于哪一期 +# ========================================================================== + +PERIOD_CELL = ("A1", "统计时间:2026年8月1日至2026年8月31日") +ORDERS = [("订单号", "下单日期", "金额"), ("A001", "2026-08-01", 120), ("A002", "2026-08-15", 80), + ("A003", "2026-08-31", 40)] +ORDER_COLS = [("订单号", "TEXT"), ("下单日期", "DATE"), ("金额", "INTEGER")] + + +def _orders(period: dict[str, Any], *, context: bool = True, grain: list[str] | None = None) -> dict[str, Any]: + data = _list(ORDER_COLS, table="订单", context=context, period=period) + data["tables"][0]["grain"] = grain if grain is not None else ["订单号"] + return data + + +def test_synthesized_period_columns_carry_this_periods_bounds(tmp_path): + recipe = _orders({"synthesize": "统计期"}, grain=["订单号", LIST_PERIOD_START]) + cols = derive_tables(Recipe.model_validate(recipe))[0]["订单"] + assert [(c.name, c.role) for c in cols][-2:] == [(LIST_PERIOD_START, "period"), (LIST_PERIOD_END, "period")] + db = str(tmp_path / "p.db") + ex = _run(_book(ORDERS, top=[PERIOD_CELL], start=3), recipe, db) + assert ex.ok, [p.message for p in ex.problems] + assert _rows(db, f'SELECT "订单号", "{LIST_PERIOD_START}", "{LIST_PERIOD_END}" FROM "订单" ORDER BY rowid') == [ + ("A001", "2026-08-01", "2026-08-31"), ("A002", "2026-08-01", "2026-08-31"), + ("A003", "2026-08-01", "2026-08-31")] + assert ex.table_hashes["订单"] + # 统计期起止列没有格子来历:溯源只有表头推出来的三列 + assert set(ex.lineage["订单"]) == {"订单号", "下单日期", "金额"} + # 干跑(不建库)同样走得通 + assert _run(_book(ORDERS, top=[PERIOD_CELL], start=3), recipe).ok + + +def test_synthesized_period_with_a_human_period(tmp_path): + import datetime as dt + + from app.data.recipe_types import PeriodInput + + recipe = _orders({"synthesize": "统计期"}) + raw = _book(ORDERS) + assert {p.code for p in _run(raw, recipe).problems} == {"period_missing"} + db = str(tmp_path / "h.db") + ex = _run(raw, recipe, db, context_inputs={"统计期": PeriodInput(dt.date(2026, 8, 1), dt.date(2026, 8, 31))}) + assert ex.ok and _rows(db, f'SELECT DISTINCT "{LIST_PERIOD_START}" FROM "订单"') == [("2026-08-01",)] + + +def test_rows_must_fall_inside_the_period_when_bound_to_a_date_column(tmp_path): + recipe = _orders({"column": "下单日期"}) + db = str(tmp_path / "w.db") + ex = _run(_book(ORDERS, top=[PERIOD_CELL], start=3), recipe, db) + assert ex.ok + d1 = _checks(db, ex, recipe)["D1"] + assert d1.kind == "period_rows" and d1.status == "passed" and d1.checked == 3 + late = list(ORDERS) + [("A004", "2026-09-02", 10)] + db2 = str(tmp_path / "w2.db") + ex2 = _run(_book(late, top=[PERIOD_CELL], start=3), recipe, db2) + d1 = _checks(db2, ex2, recipe)["D1"] + assert d1.status == "mismatch" and d1.category == "structure" and d1.failed == 1 + assert d1.cells == ["明细!B7"] and "2026-08-01 至 2026-08-31" in d1.details[0] + + +def test_list_period_validation(): + def codes(data: dict[str, Any]) -> list[str]: + return [p.code for p in validate_recipe(data, facts=None, origin="rules")[1]] + + assert codes(_orders({"column": "下单日期"})) == [] + assert "list_period_invalid" in codes(_orders({"column": "金额"})) + assert "column_unknown" in codes(_orders({"column": "付款日期"})) + assert "list_period_invalid" in codes(_orders({"column": "下单日期", "synthesize": "统计期"})) + assert "list_period_invalid" in codes(_orders({})) + assert "covers_without_context" in codes(_orders({"synthesize": "统计期"}, context=False)) + clash = _list(ORDER_COLS + [(LIST_PERIOD_START, "TEXT")], table="订单", context=True, + period={"synthesize": "统计期"}) + assert "name_collision" in codes(clash) + + +def test_lists_become_eligible_for_accumulation_once_bound(): + unbound = Recipe.model_validate({**_list(ORDER_COLS, table="订单", context=True), "mode": "accumulate"}) + [p] = accumulate_blockers(unbound) + assert "没有指定每一行属于哪一期" in p.message + for period in ({"column": "下单日期"}, {"synthesize": "统计期"}): + data = {**_orders(period), "mode": "accumulate"} + assert accumulate_blockers(Recipe.model_validate(data)) == [] + assert validate_recipe(data, facts=None, origin="rules")[1] == [] + kept = {**_orders({"column": "下单日期"}), "mode": "accumulate"} + kept["sheets"][0]["blocks"][0]["rows"] = {"total_row": {"label_column": "订单号", "pick": "合计", + "keep_as": "订单_表内合计"}} + kept["tables"].append({"name": "订单_表内合计", "grain": ["合计项"], "kind": "reported_total"}) + blockers = accumulate_blockers(Recipe.model_validate(kept)) + assert [p.path for p in blockers] == ["/tables/1"] and "合计行、小计行另存的表" in blockers[0].message + + +def _built(raw: bytes, recipe: dict[str, Any], tag: str) -> tuple[str, Any]: + folder = Path(tempfile.mkdtemp(prefix="p5-built-")) + db = folder / f"{tag}.db" + ex = _run(raw, recipe, str(db)) + assert ex.ok, [p.message for p in ex.problems] + table_versions.settle_journal(db) + return str(db), ex + + +def _part(db: str, ex: Any, recipe: dict[str, Any], **kw: Any) -> UnionPart: + return UnionPart(import_id=None, start=kw.get("start", ex.period.start), end=kw.get("end", ex.period.end), + db_path=db, recipe=Recipe.model_validate(recipe), db_sha256=raw_store.sha256_file(db), + table_hashes=dict(ex.table_hashes)) + + +@pytest.mark.parametrize("period", [{"column": "下单日期"}, {"synthesize": "统计期"}]) +def test_two_months_of_a_list_materialize_into_one_union(tmp_path, period): + recipe = {**_orders(period), "mode": "accumulate"} + aug = _built(_book(ORDERS, top=[PERIOD_CELL], start=3), recipe, "aug") + sep_rows = [ORDERS[0], ("B001", "2026-09-03", 70), ("B002", "2026-09-30", 30)] + sep = _built(_book(sep_rows, top=[("A1", "统计时间:2026年9月1日至2026年9月30日")], start=3), recipe, "sep") + out = tmp_path / "u.db" + report = materialize_union(out, target=Recipe.model_validate(recipe), + parts=[_part(*aug, recipe), _part(*sep, recipe)]) + assert report.ok, [c.details for c in report.checks] + assert report.rows == {"订单": 5} + assert {c.id: c.status for c in report.checks} == {"U1": "passed", "U2": "passed", "U3": "passed"} + assert report.checks[2].checked == 2 + # 登记的统计期和库里的对不上(把 8 月的库说成 9 月):U3 拒收 + bad = materialize_union(tmp_path / "bad.db", target=Recipe.model_validate(recipe), + parts=[_part(*aug, recipe, start="2026-09-01", end="2026-09-30")]) + u3 = next(c for c in bad.checks if c.id == "U3") + assert not bad.ok and u3.status == "mismatch" and "第 1 期" in u3.details[0] + + +def test_the_same_order_in_two_periods_is_a_union_key_conflict(tmp_path): + """明细跨期重复(同一订单两期都出现)按日期列分期时主键冲突,U2 拒收(细案风险条:语义要在配方里显式声明)。""" + recipe = {**_orders({"synthesize": "统计期"}), "mode": "accumulate"} + aug = _built(_book(ORDERS, top=[PERIOD_CELL], start=3), recipe, "aug2") + sep = _built(_book(ORDERS, top=[("A1", "统计时间:2026年9月1日至2026年9月30日")], start=3), recipe, "sep2") + report = materialize_union(tmp_path / "u.db", target=Recipe.model_validate(recipe), + parts=[_part(*aug, recipe), _part(*sep, recipe)]) + u2 = next(c for c in report.checks if c.id == "U2") + assert not report.ok and u2.status == "mismatch" + # 主键带上统计期起就分得开:每月一份的快照式明细 + keyed = {**_orders({"synthesize": "统计期"}, grain=["订单号", LIST_PERIOD_START]), "mode": "accumulate"} + aug = _built(_book(ORDERS, top=[PERIOD_CELL], start=3), keyed, "aug3") + sep = _built(_book(ORDERS, top=[("A1", "统计时间:2026年9月1日至2026年9月30日")], start=3), keyed, "sep3") + assert materialize_union(tmp_path / "k.db", target=Recipe.model_validate(keyed), + parts=[_part(*aug, keyed), _part(*sep, keyed)]).rows == {"订单": 6} + + +# ========================================================================== +# 起草:表格对象、穿插的小计、每一行属于哪一期的问题 +# ========================================================================== + + +def _ok_validate(data, facts, origin): + from app.data.recipe import validate_recipe as v + + return v(data, facts=facts, origin=origin) + + +def _draft(raw: bytes, filename: str = "明细.xlsx"): + scan = csv_grid.scan_any(raw, filename) + grids = {s.name: xlsx_cells.read_grid(raw, scan, s.name, max_rows=recipe_suggest.DRAFT_MAX_ROWS) + for s in scan.sheets if s.state == "visible" and s.nonempty} + + def dry(recipe: Recipe): + return recipe_engine.execute(recipe, raw, filename, scan, None) + return recipe_suggest.draft(scan, grids, filename, validate=_ok_validate, dry_run=dry), scan, grids + + +def _table_book(*, totals: bool = True, name: str = "门店销售") -> bytes: + """一个表格对象:B3:D7(表头、3 行明细、汇总行),表格上面有一行说明、右边有一个空格隔开的备注列。""" + wb = Workbook() + ws = wb.active + ws.title = "销售" + ws["B1"] = "门店销售明细" + rows = [("门店", "日期", "金额"), ("门店一", "2026-08-01", 10), ("门店二", "2026-08-02", 20), + (None, None, None), ("门店三", "2026-08-03", 30)] + if totals: + rows.append(("汇总", None, 60)) + for r, row in enumerate(rows, start=3): + for c, v in enumerate(row, start=2): + if v is not None: + ws.cell(r, c, v) + t = Table(displayName=name, ref=f"B3:D{2 + len(rows)}") + if totals: + t.totalsRowCount = 1 + ws.add_table(t) + buf = io.BytesIO() + wb.save(buf) + return buf.getvalue() + + +def test_a_table_object_is_drafted_as_a_list_without_ai(): + raw = _table_book() + scan = xlsx_scan.scan(raw) + assert [(t.name, t.ref, t.totals_row_count) for t in scan.tables] == [("门店销售", "B3:D8", 1)] + draft, _, _ = _draft(raw) + assert draft.complete and draft.origin == "rules", draft.failures + [block] = draft.recipe["sheets"][0]["blocks"] + assert block["layout"] == "list" and block["table"] == "门店销售" + assert [c["header"] for c in block["columns"]] == ["门店", "日期", "金额"] + assert block["rows"]["total_row"] == {"label_column": "门店", "pick": "汇总", "keep_as": "门店销售_表内合计"} + assert block["rows"]["blank_rows"] == "skip" + # 锚点按表头文字,不记坐标 + assert "B3" not in str(block) + + +def test_a_table_object_with_digits_in_its_name_falls_back_to_the_title(): + draft, _, _ = _draft(_table_book(name="表1")) + assert draft.complete and draft.recipe["sheets"][0]["blocks"][0]["table"] == "门店销售明细" + + +def test_drafted_subtotals_run_and_verify(tmp_path): + raw = _book(STORES) + draft, scan, _ = _draft(raw) + assert draft.complete, draft.failures + block = draft.recipe["sheets"][0]["blocks"][0] + assert block["rows"]["subtotals"]["pick"] == "小计" and block["rows"]["total_row"]["pick"] == "合计" + db = str(tmp_path / "d.db") + recipe = Recipe.model_validate(draft.recipe) + ex = recipe_engine.execute(recipe, raw, "明细.xlsx", scan, db) + assert ex.ok and ex.tables[0].rows == 4 + assert all(c.status == "passed" for c in recipe_checks.run_checks(db, ex, recipe) if c.kind == "column_sum") + + +def test_period_question_binds_the_list_and_switches_to_accumulation(): + raw = _book(ORDERS, top=[PERIOD_CELL], start=3) + draft, _, grids = _draft(raw) + assert draft.complete, draft.failures + from app.data import recipe as R + + recipe = draft.recipe + cards, questions = recipe_suggest.questions_for(recipe, draft.facts, grids) + q = next(q for q in questions if q.id == "q_list_period:列表1") + assert [o["value"] for o in q.options] == ["none", "column:下单日期", "synthesize"] and q.default == "none" + assert not any(q.id == "q_mode" for q in questions) + bound = R.apply_patch(recipe, q.effects["column:下单日期"]) + assert bound["sheets"][0]["blocks"][0]["period"]["column"] == "下单日期" and bound["mode"] == "accumulate" + assert validate_recipe(bound, facts=None, origin="rules")[1] == [] + assert R.apply_patch(recipe, q.effects["none"])["mode"] == "replace" + # 工作表没配统计期的,选了以后一并加上 + plain = _book(ORDERS) + draft2, _, grids2 = _draft(plain) + assert draft2.recipe["sheets"][0]["context"] == [] + _, qs = recipe_suggest.questions_for(draft2.recipe, draft2.facts, grids2) + q2 = next(q for q in qs if q.id == "q_list_period:列表1") + bound2 = R.apply_patch(draft2.recipe, q2.effects["synthesize"]) + assert bound2["sheets"][0]["context"][0]["id"] == "统计期" + assert bound2["sheets"][0]["blocks"][0]["period"]["synthesize"] == "统计期" + assert accumulate_blockers(Recipe.model_validate(bound2)) == [] + + +# ========================================================================== +# CSV +# ========================================================================== + + +def test_csv_drafts_a_list_and_the_engine_runs_it(tmp_path): + raw = ("订单号,下单日期,门店,金额\n" + "A001,2026-08-01,一号店,120\n" + "A002,2026-08-02,二号店,80.5\n" + "小计,,,200.5\n" + "A003,2026-08-03,一号店,1\n" + "合计,,,201.5\n").encode("gb18030") + draft, scan, _ = _draft(raw, "orders_2026-08.csv") + assert draft.complete and draft.origin == "rules", draft.failures + sheet = draft.recipe["sheets"][0] + assert sheet["match"]["name"] == csv_grid.CSV_SHEET + block = sheet["blocks"][0] + assert [c["type"] for c in block["columns"]] == ["TEXT", "DATE", "TEXT", "REAL"] + assert block["rows"]["subtotals"]["pick"] == "小计" + db = str(tmp_path / "c.db") + recipe = Recipe.model_validate(draft.recipe) + ex = recipe_engine.execute(recipe, raw, "orders_2026-08.csv", scan, db) + assert ex.ok and ex.tables[0].rows == 3 + assert all(c.status == "passed" for c in recipe_checks.run_checks(db, ex, recipe) if c.kind == "column_sum") + # 回执的坐标照 Excel 打开这个 CSV 时的 A1 写 + [excluded] = [x for x in ex.rows_excluded if x.reason == "subtotal"] + assert excluded.sheet == "CSV" and excluded.rows == [[4, 4]] + + +# ========================================================================== +# 没配小计时,像小计的行:照常导入、要人确认,带「改作小计核对」的修复按钮 +# ========================================================================== + + +def test_a_subtotal_like_row_without_subtotals_is_flagged_and_fixable(tmp_path): + from app.data import recipe_confirm, recipe_fixes + from app.data.recipe import apply_patch + + plain = _list(STORE_COLS, rows={"total_row": {"label_column": "地区", "pick": "合计"}}) + ex = _run(_book(STORES), plain) + [p] = [p for p in ex.problems if p.code == "subtotal_suspect"] + assert p.category == "confirm" and p.fix == "declare_subtotals" + assert p.cells == ["明细!A4", "明细!A7"] and p.fix_args["pick"] == "小计" and p.fix_args["label_column"] == "地区" + assert ex.ok # 需确认类:照常导入,提交时勾确认项 + full = Recipe.model_validate(plain).model_dump(mode="json") + [prop] = recipe_fixes.propose_fixes(full, [recipe_fixes.dataclasses.asdict(p)], [], None) + assert prop.kind == "declare_subtotals" and [o.value for o in prop.options] == ["check_only", "keep"] + res = recipe_fixes.fix_ops(full, prop, "check_only", None) + assert res.ok, res.problems + fixed = apply_patch(full, res.ops) + assert fixed["sheets"][0]["blocks"][0]["rows"]["subtotals"]["pick"] == "小计" + db = str(tmp_path / "f.db") + ex2 = _run(_book(STORES), fixed, db) + assert ex2.ok and not [p for p in ex2.problems if p.code == "subtotal_suspect"] + assert _checks(db, ex2, fixed)["T2"].status == "passed" + keep = apply_patch(full, recipe_fixes.fix_ops(full, prop, "keep", None).ops) + assert keep["tables"][-1] == {"name": "明细_小计", "grain": [], "kind": "reported_total", "units": {}, "note": ""} + assert validate_recipe(keep, facts=None, origin="rules")[1] == [] + # 确认项:没改作小计核对就提交的,要勾「确实是数据」 + items = recipe_confirm.confirm_items(recipe_confirm.ConfirmContext(kind="first", recipe=plain, extraction=ex, + checks=[])) + [item] = [i for i in items if i.id == "subtotal_suspect:列表1"] + assert item.required and "改作小计核对" in item.label + + +def test_ordinary_rows_starting_with_a_total_word_are_not_flagged(): + """首列以「累计」开头、别的文字列有字的是数据(「累计积分卡」这种商品名),不当成像小计的行。""" + rows = [("商品", "门店", "金额"), ("累计积分卡", "门店一", 10), ("普通卡", "门店二", 20)] + ex = _run(_book(rows), _list([("商品", "TEXT"), ("门店", "TEXT"), ("金额", "INTEGER")])) + assert ex.ok and not [p for p in ex.problems if p.code == "subtotal_suspect"] + + +def _suspects(data: list[tuple[Any, ...]], columns: list[tuple[str, str]], **block: Any) -> list[str]: + ex = _run(_book(data), _list(columns, **block)) + assert ex.ok, [p.message for p in ex.problems] + return [c for p in ex.problems if p.code == "subtotal_suspect" for c in p.cells] + + +def test_a_two_column_key_value_list_is_never_flagged(): + """复核反例:只有「标签文字列 + 数字列」的键值清单,「其余文字列为空」自然成立,不能据此说它像小计—— + 「累计收入」「汇总调整」是普通的数据行。最后一行以合计字样开头也一样。""" + cols = [("项目", "TEXT"), ("金额", "INTEGER")] + assert _suspects([("项目", "金额"), ("门票收入", 100), ("累计收入", 300), ("汇总调整", 5), ("餐饮收入", 50)], + cols) == [] + assert _suspects([("项目", "金额"), ("门票收入", 100), ("餐饮收入", 50), ("合计调整项", 5)], cols) == [] + + +def test_only_rows_between_details_are_flagged(): + """像小计的行要夹在明细中间:上下相邻的明细在某个文字列上有值、这一行空着;后面还得有明细(或者是合计行)。 + 放在最后、后面没有明细的不报(那不是「夹在中间」);上下的明细那一列本来就空着的也不报。""" + cols = [("地区", "TEXT"), ("门店", "TEXT"), ("金额", "INTEGER")] + head = ("地区", "门店", "金额") + # 最后一行:后面没有明细,也没有合计行 + assert _suspects([head, ("分区甲", "门店一", 100), ("分区甲", "门店二", 200), ("小计", None, 300)], cols) == [] + # 上面的明细「门店」也空着:不是因为小计才空的 + assert _suspects([head, ("分区甲", None, 100), ("累计调整", None, 5), ("分区乙", "门店三", 50)], cols) == [] + # 下面的明细「门店」空着:同样不报 + assert _suspects([head, ("分区甲", "门店一", 100), ("累计调整", None, 5), ("分区乙", None, 50)], cols) == [] + # 夹在明细中间:报 + assert _suspects([head, ("分区甲", "门店一", 100), ("小计", None, 100), ("分区乙", "门店三", 50)], + cols) == ["明细!A3"] + # 最后一组的小计后面紧跟合计行:也在列表里面,报 + total = {"rows": {"total_row": {"label_column": "地区", "pick": "合计"}}} + assert _suspects([head, ("分区甲", "门店一", 100), ("小计", None, 100), ("合计", None, 100)], cols, + **total) == ["明细!A3"] + + +def test_declaring_no_subtotals_keeps_such_rows_as_data_without_asking(tmp_path): + """人确认过「这张列表没有小计行」(rows.no_subtotals):以合计字样开头的行照常是数据,不再每期报 subtotal_suspect、 + 不再每期要勾确认;这个声明本身是放宽,配上的那一次要确认一次。和 subtotals 只能二选一。""" + from app.data import recipe_confirm + + recipe = _list(STORE_COLS, rows={"total_row": {"label_column": "地区", "pick": "合计"}, "no_subtotals": True}) + assert validate_recipe(recipe, facts=None, origin="rules")[1] == [] + db = str(tmp_path / "n.db") + ex = _run(_book(STORES), recipe, db) + assert ex.ok and not [p for p in ex.problems if p.code == "subtotal_suspect"] + assert _rows(db, 'SELECT COUNT(*) FROM "明细"') == [(6,)] # 两行「小计」照常是数据 + items = recipe_confirm.confirm_items(recipe_confirm.ConfirmContext(kind="first", recipe=recipe, extraction=ex, + checks=[])) + ids = [i.id for i in items] + assert "no_subtotals:列表1" in ids and not [i for i in ids if i.startswith("subtotal_suspect:")] + sigs = recipe_confirm.recipe_item_signatures(Recipe.model_validate(recipe)) + assert sigs["no_subtotals:列表1"] != recipe_confirm.recipe_item_signatures( + Recipe.model_validate(_list(STORE_COLS)))["no_subtotals:列表1"] + both = _stores_recipe() + both["sheets"][0]["blocks"][0]["rows"]["no_subtotals"] = True + probs = validate_recipe(both, facts=None, origin="rules")[1] + assert [p.code for p in probs] == ["subtotals_conflict"] and probs[0].path.endswith("/rows/no_subtotals") diff --git a/backend/tests/test_recipe_p5_e2e.py b/backend/tests/test_recipe_p5_e2e.py new file mode 100644 index 00000000..d5538bee --- /dev/null +++ b/backend/tests/test_recipe_p5_e2e.py @@ -0,0 +1,191 @@ +"""Excel 期 5 的端到端(真管线,经接口):每月一份的 CSV 明细按期累积成一张表,重叠的那期被拒,同一期再传要确认 +替换;带穿插小计的 Excel 规则起草成功、小计核对通过、小计行不进数据。 + +夹具全部合成(订单号 A001…、门店一…),不调用任何模型(规则起草不完整时的 AI 可用性查询换成「不可用」的桩)。 +""" +from __future__ import annotations + +import io +import json +import uuid +from typing import Any + +import pytest +from httpx import ASGITransport, AsyncClient +from openpyxl import Workbook + +from app.core.config import settings +from app.main import app + +API = "/api/datasources" +CSV_MIME = "text/csv" + + +@pytest.fixture(autouse=True) +def store(monkeypatch, tmp_path): + data = tmp_path / "data" + (data / "uploads").mkdir(parents=True) + monkeypatch.setattr(settings, "data_dir", data) + yield data + + +@pytest.fixture(autouse=True) +def no_model(monkeypatch): + from app.data import recipe_ai + from app.data.recipe_types import AiAvailability + + async def unavailable(session): + return None, "", AiAvailability(False, reason="验收测试不接模型") + + monkeypatch.setattr(recipe_ai, "resolve_draft_model", unavailable) + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +def _name() -> str: + return f"p5_{uuid.uuid4().hex[:10]}" + + +def month_csv(month: int, orders: list[tuple[str, int, int]]) -> tuple[bytes, str]: + """一个月的订单导出:订单号、下单日期、门店、金额。文件名带统计期(只作旁证,CSV 里没有统计期格)。""" + lines = ["订单号,下单日期,门店,金额"] + for oid, day, amount in orders: + lines.append(f"{oid},2026-{month:02d}-{day:02d},门店{'一二'[day % 2]},{amount}") + last = 31 if month in (8, 10) else 30 + return ("\n".join(lines) + "\n").encode("utf-8"), f"orders_2026-{month:02d}-01_2026-{month:02d}-{last}.csv" + + +AUG = [("A001", 1, 120), ("A002", 15, 80), ("A003", 31, 40)] +SEP = [("B001", 3, 70), ("B002", 30, 30)] + + +async def stage(client, raw: bytes, filename: str, name: str) -> dict[str, Any]: + resp = await client.post(f"{API}/imports/stage", files={"file": (filename, raw, CSV_MIME)}, data={"name": name}) + assert resp.status_code == 201, resp.text + return resp.json() + + +async def answer(client, sid: str, answers: dict[str, Any]) -> dict[str, Any]: + resp = await client.post(f"{API}/imports/{sid}/answers", json={"answers": answers}) + assert resp.status_code == 200, resp.text + return resp.json() + + +async def trial(client, sid: str, **body: Any) -> dict[str, Any]: + resp = await client.post(f"{API}/imports/{sid}/trial", json=body) + assert resp.status_code == 200, resp.text + return resp.json() + + +def _period(month: int) -> dict[str, Any]: + last = 31 if month in (8, 10) else 30 + return {"统计期": {"start": f"2026-{month:02d}-01", "end": f"2026-{month:02d}-{last}"}} + + +async def commit(client, st: dict[str, Any], confirmations: list[str] | None = None): + ids = [i["id"] for i in st["trial"]["confirm_items"]] if confirmations is None else confirmations + return await client.post(f"{API}/imports/{st['id']}/commit", + json={"trial_id": st["trial"]["trial_id"], "confirmations": ids}) + + +async def rows(client, name: str, sql: str) -> list[list[Any]]: + resp = await client.post(f"/api/tools/db_query__{name}/run", json={"args": {"sql": sql}}) + assert resp.status_code == 200, resp.text + out = resp.json() + assert out["ok"], out + return json.loads(out["result"])["rows"] + + +async def first_csv(client, name: str) -> dict[str, Any]: + raw, fn = month_csv(8, AUG) + st = await stage(client, raw, fn, name) + assert st["recipe"]["sheets"][0]["match"]["name"] == "CSV" + assert not st["recipe_problems"], st["recipe_problems"] + q = next(q for q in st["questions"] if q["id"].startswith("q_list_period:")) + # 列表的草稿还不能按期累积,没有 q_mode;「每期怎么更新」这一道题同时定下导入模式和每一行属于哪一期 + assert not any(x["id"] == "q_mode" for x in st["questions"]) + st = await answer(client, st["id"], {q["id"]: {"value": "synthesize"}}) + assert st["recipe"]["mode"] == "accumulate" and not st["recipe_problems"], st["recipe_problems"] + st = await trial(client, st["id"]) + assert st["trial"]["status"] == "needs_input", st["trial"]["problems"] + st = await trial(client, st["id"], context_inputs=_period(8)) + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + ids = [i["id"] for i in st["trial"]["confirm_items"]] + assert "list_period:列表1" in ids + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + return {**resp.json(), "table": st["recipe"]["sheets"][0]["blocks"][0]["table"]} + + +async def reupload(client, source_id: str, month: int, orders: list[tuple[str, int, int]], + period: dict[str, Any] | None = None) -> dict[str, Any]: + raw, fn = month_csv(month, orders) + resp = await client.post(f"{API}/{source_id}/reupload", files={"file": (fn, raw, CSV_MIME)}) + assert resp.status_code == 201, resp.text + st = resp.json() + assert st["trial"]["status"] == "needs_input", st["trial"]["problems"] + return await trial(client, st["id"], context_inputs=period or _period(month)) + + +async def test_monthly_csv_exports_accumulate_into_one_table(client): + name = _name() + first = await first_csv(client, name) + sid, table = first["source"]["id"], first["table"] + assert await rows(client, name, f'SELECT "订单号", "统计期起", "统计期止" FROM "{table}" ORDER BY rowid') == [ + ["A001", "2026-08-01", "2026-08-31"], ["A002", "2026-08-01", "2026-08-31"], ["A003", "2026-08-01", "2026-08-31"]] + st = await reupload(client, sid, 9, SEP) + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + assert st["trial"]["accumulate"]["action"] == "append" + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + got = await rows(client, name, f'SELECT "统计期起", COUNT(*) FROM "{table}" GROUP BY 1 ORDER BY 1') + assert got == [["2026-08-01", 3], ["2026-09-01", 2]] + # 部分重叠的一期被拒收 + st = await reupload(client, sid, 9, SEP, period={"统计期": {"start": "2026-08-15", "end": "2026-09-14"}}) + assert st["trial"]["status"] == "rejected" and any(p["code"] == "period_overlap" for p in st["trial"]["problems"]) + # 同一期再传(数不同):要确认替换 + st = await reupload(client, sid, 9, [("B001", 3, 75)]) + assert st["trial"]["status"] == "passed" and st["trial"]["accumulate"]["action"] == "replace_period" + ids = [i["id"] for i in st["trial"]["confirm_items"]] + replace = "period_replace:2026-09-01~2026-09-30" + assert replace in ids + resp = await commit(client, st, [i for i in ids if i != replace]) + assert resp.status_code == 422 + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + assert await rows(client, name, f'SELECT COUNT(*), SUM("金额") FROM "{table}"') == [[4, 315]] + + +async def test_excel_with_subtotals_drafts_checks_and_keeps_them_out(client): + wb = Workbook() + ws = wb.active + ws.title = "门店" + for row in [("门店", "金额"), ("门店一", 100), ("门店二", 200), ("小计", 300), ("门店三", 150), ("门店四", 150), + ("小计", 300), ("合计", 600)]: + ws.append(row) + buf = io.BytesIO() + wb.save(buf) + name = _name() + resp = await client.post(f"{API}/imports/stage", files={"file": ("门店.xlsx", buf.getvalue(), "application/octet-stream")}, + data={"name": name}) + assert resp.status_code == 201, resp.text + st = resp.json() + assert not st["draft_problems"] and not st["recipe_problems"], (st["draft_problems"], st["recipe_problems"]) + block = st["recipe"]["sheets"][0]["blocks"][0] + assert block["rows"]["subtotals"]["pick"] == "小计" + st = await trial(client, st["id"]) + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + checks = {c["id"]: c for c in st["trial"]["checks"]} + assert checks["T2"]["status"] == "passed" and "小计" in checks["T2"]["title"] + excluded = st["trial"]["receipt"]["rows_excluded"] + assert any(x["reason"] == "subtotal" and x["rows"] == [[4, 4], [7, 7]] for x in excluded), excluded + ids = [i["id"] for i in st["trial"]["confirm_items"]] + assert "subtotals:列表1" in ids + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + table = st["recipe"]["sheets"][0]["blocks"][0]["table"] + assert await rows(client, name, f'SELECT COUNT(*), SUM("金额") FROM "{table}"') == [[4, 600]] diff --git a/backend/tests/test_recipe_select.py b/backend/tests/test_recipe_select.py index 6824450b..9e408f16 100644 --- a/backend/tests/test_recipe_select.py +++ b/backend/tests/test_recipe_select.py @@ -210,10 +210,8 @@ def _subtotals() -> Book: @pytest.mark.parametrize("ref,bottom,where", [ - ("A1:C7", "box", "第 4 行像合计行"), # 小计夹在框中间 ("A1:C4", "box", "第 4 行像合计行"), # 框到小计为止:它下面紧接着还有明细,是夹在中间的小计 ("A1:C3", "box", "第 4 行像合计行"), # 框到小计上一行:下一行是小计,同上 - ("A1:C3", "auto", "第 4 行像合计行"), # 按规则推断:起草器在这里记失败,框选同样不收 ]) def test_subtotal_in_the_middle_is_never_imported_as_data(ref, bottom, where): b = _subtotals() @@ -222,6 +220,22 @@ def test_subtotal_in_the_middle_is_never_imported_as_data(ref, bottom, where): assert where in res.problems[0].message and "明细!A4" in res.problems[0].cells +@pytest.mark.parametrize("ref,bottom", [ + ("A1:C7", "box"), # 小计夹在框中间、合计在框底 + ("A1:C3", "auto"), # 按规则推断:起草器把夹在中间的小计提议成小计(期 5),框选同样 +]) +def test_subtotal_in_the_middle_becomes_subtotals_not_data(ref, bottom): + """期 5:夹在中间、第一列以合计字样开头的行提议成小计(ListRows.subtotals):不进数据,按各组明细核对。""" + b = _subtotals() + res = selection_edit(None, b.grid, sel(ref, "list", "明细", bottom=bottom), extraction=None) + assert res.ok, res.problems + [op] = res.ops # 配方为空:一条根替换,新建只有这一个块的配方 + block = op["value"]["sheets"][0]["blocks"][0] + assert block["rows"]["subtotals"] == {"label_column": "地区", "pick": "小计", "scope": "since_previous", + "group_column": None, "keep_as": None} + assert block["rows"]["total_row"]["pick"] == "合计" + + def test_total_word_in_another_column_is_not_imported_as_data(): """合计字样写在第二列(第一列空着):起草器记失败,框选同样报错,不当明细导入。""" wb = Workbook() diff --git a/backend/tests/test_recipe_suggest.py b/backend/tests/test_recipe_suggest.py index 40c62af1..44bd2164 100644 --- a/backend/tests/test_recipe_suggest.py +++ b/backend/tests/test_recipe_suggest.py @@ -951,7 +951,7 @@ def test_list_subtotal_rows_in_middle_fail(): assert [f for f in d.failures if "像合计行" in f] == [ "工作表「门店」:第 4 行像合计行(B 列「小计」),但合计字样不在第一列,无法按合计行核对", "工作表「门店」:第 7 行像合计行(B 列「小计」),但合计字样不在第一列,无法按合计行核对"] - # 小计写在第一列、但下面还有明细:不是末尾的合计行,整块照样收进来,不另报「数据区外的数字」 + # 期 5:小计写在第一列、下面还有明细:提议成穿插的小计(不进数据,按上一行小计之后的明细核对),不再记失败 wb2 = Workbook() ws2 = wb2.active ws2.title = "门店" @@ -960,10 +960,21 @@ def test_list_subtotal_rows_in_middle_fail(): ws2.append(row) sc2, grids2 = build(wb2) d2 = S.draft(sc2, grids2, "x.xlsx", validate=ok_validate()) - assert not d2.complete - assert d2.failures == ["工作表「门店」:第 4 行像合计行(A 列「小计」),但不在列表末尾,无法确定它汇总的是哪几行"] + assert d2.complete and d2.failures == [] block = d2.recipe["sheets"][0]["blocks"][0] assert block["rows"]["total_row"]["pick"] == "合计" + assert block["rows"]["subtotals"] == {"label_column": "门店", "pick": "小计", "scope": "since_previous", + "group_column": None, "keep_as": None} + # 字样不一样(「小计」「合计」都夹在中间):分不清,照旧记失败 + wb3 = Workbook() + ws3 = wb3.active + ws3.title = "门店" + for row in [["门店", "金额"], ["门店一", 100], ["小计", 100], ["门店二", 200], ["合计", 200], ["门店三", 150], + ["总计", 450]]: + ws3.append(row) + sc3, grids3 = build(wb3) + d3 = S.draft(sc3, grids3, "x.xlsx", validate=ok_validate()) + assert not d3.complete and any("不在列表末尾" in f for f in d3.failures) # 末尾第一列的合计行照旧改作核对,不算失败(test_list_total_row_is_verified_and_kept) diff --git a/backend/tests/test_recipe_validate.py b/backend/tests/test_recipe_validate.py index 0d2f945e..3e9fe110 100644 --- a/backend/tests/test_recipe_validate.py +++ b/backend/tests/test_recipe_validate.py @@ -315,7 +315,7 @@ def test_accumulate_mode_follows_eligibility(): probs = run(lr, facts=None) assert [(p.code, p.path) for p in probs] == [("accumulate_unkeyed", "/tables/0"), ("accumulate_unkeyed", "/tables/1")] - assert "列表形态的表目前只支持每期替换" in probs[0].message + assert "没有指定每一行属于哪一期" in probs[0].message # 同一份配方每期替换时不报 lr["mode"] = "replace" assert run(lr, facts=None) == [] diff --git a/backend/tests/test_recipe_validate_p3.py b/backend/tests/test_recipe_validate_p3.py index a2e2f775..b4f0acca 100644 --- a/backend/tests/test_recipe_validate_p3.py +++ b/backend/tests/test_recipe_validate_p3.py @@ -79,7 +79,7 @@ def test_accumulate_list_table_message_does_not_promise_future_versions(): r["sheets"][0]["context"] = [{}] _rec, problems = validate_recipe(r) assert [p.code for p in problems] == ["accumulate_unkeyed"] - assert "列表形态的表目前只支持每期替换" in problems[0].message and "后续版本" not in problems[0].message + assert "没有指定每一行属于哪一期" in problems[0].message and "后续版本" not in problems[0].message # ========================================================================== diff --git a/backend/tests/test_records.py b/backend/tests/test_records.py index 88f435aa..c051c126 100644 --- a/backend/tests/test_records.py +++ b/backend/tests/test_records.py @@ -461,7 +461,7 @@ async def test_unknown_result_is_found_by_the_lookup_and_not_written_again(engin if (await session.get(ActionRecord, rec.id)).next_lookup_at is not None: break await asyncio.sleep(0.05) - assert await actions.tick(utcnow() + timedelta(minutes=5)) == 1 + assert rec.id in await actions.tick(utcnow() + timedelta(minutes=5)) done = await wait(run.id, ("succeeded", "failed")) assert done.status == "succeeded", done.error async with SessionLocal() as session: diff --git a/backend/tests/test_run_rejected.py b/backend/tests/test_run_rejected.py new file mode 100644 index 00000000..bf00fda6 --- /dev/null +++ b/backend/tests/test_run_rejected.py @@ -0,0 +1,239 @@ +"""驳回 ≠ 失败(docs/design/next-round/05-已知遗留.md (a)、(h);总方案 X9)。 + +人的驳回(工具调用审批、代码执行审批、审批设了驳回即终止、填写 / 编辑被驳回、动作确认、双盲裁决)以前在引擎里抛普通的 +NodeError,运行记成 failed:通知推「实例失败」,定时推「运行失败」,记录页给「继续运行」——接着跑只会读到同一个驳回答复、 +再停一次。现在统一抛 NodeRejected,runner 收成 cancelled + end_reason=rejected,run.cancelled 带上在哪一步、哪种关卡、 +结论、谁、备注;原话(Run.error)一字不变。手动停止记 end_reason=stopped。被驳回的不能接着跑,存量的 failed+驳回也不能。 + +动作确认、双盲裁决的驳回在 test_process_actions.py、test_dual_check_node.py 里(要假的外部系统、口径卡)。 +""" +from __future__ import annotations + +import asyncio +import uuid +from datetime import timedelta + +import pytest +from httpx import ASGITransport, AsyncClient +from sqlalchemy import select + +from app.db.base import SessionLocal, utcnow +from app.db.models import Run, RunEvent, Task +from app.engine.runner import run_manager + + +@pytest.fixture(autouse=True) +async def engine_up(): + await run_manager.setup() + yield + await run_manager.shutdown() + + +@pytest.fixture +async def client(): + from app.main import app + + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +def node(nid, ntype, label=None, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": label or nid, "config": config}} + + +def chain(*nodes): + return {"nodes": list(nodes), "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]} + + +START = node("start", "input", fields=[{"name": "q"}]) +OUT = node("out", "output", fields=[{"name": "结果", "value": "到了"}]) + +#: (关卡, 那一步, 驳回之后 Run.error 的开头) +GATES = { + "tool": (node("gate", "tool", label="算一下", tool="calculator", args={"expression": "1+1"}, approval="always"), + "工具 calculator 的调用已被驳回:不对"), + "code": (node("gate", "code", label="跑代码", code="print(1)", language="python", approval="always"), + "代码执行已被驳回:不对"), + "approve": (node("gate", "human", label="把关", mode="approve", title="可以吗", stop_on_reject=True), + "人工驳回:不对"), + "edit": (node("gate", "human", label="改稿", mode="edit", title="改一下", draft="初稿"), "人工驳回:不对"), + "input": (node("gate", "human", label="补充", mode="input", title="补一下"), "人工驳回:不对"), +} + + +async def _wait(run_id: str, statuses: tuple[str, ...], timeout: float = 20.0) -> Run: + deadline = asyncio.get_event_loop().time() + timeout + while asyncio.get_event_loop().time() < deadline: + async with SessionLocal() as session: + row = await session.get(Run, run_id) + if row is not None and row.status in statuses: + return row + await asyncio.sleep(0.05) + raise AssertionError(f"运行 {run_id} 没有进入 {statuses}") + + +async def _events(run_id: str, kind: str | None = None) -> list[RunEvent]: + async with SessionLocal() as session: + query = select(RunEvent).where(RunEvent.run_id == run_id).order_by(RunEvent.seq) + if kind: + query = query.where(RunEvent.type == kind) + return list((await session.execute(query)).scalars()) + + +async def _rejected_run(kind: str, *, note: str = "不对", **reply) -> str: + gate, _ = GATES[kind] + run = await run_manager.start(graph=chain(START, gate, OUT), input_payload={"q": "x"}) + await _wait(run.id, ("interrupted",)) + await run_manager.wait_idle(run.id) + await run_manager.resume(run.id, {"approved": False, "note": note, **reply}, actor="张工") + return run.id + + +@pytest.mark.parametrize("kind", list(GATES)) +async def test_a_rejection_ends_the_run_as_cancelled_with_who_and_why(kind): + run_id = await _rejected_run(kind) + final = await _wait(run_id, ("cancelled", "failed", "succeeded")) + assert (final.status, final.end_reason) == ("cancelled", "rejected"), final.error + # 原话不变:给人看的那一句照旧 + assert final.error == GATES[kind][1] and final.error_node_id == "gate" and final.finished_at is not None + assert not await _events(run_id, "run.failed"), "驳回不是失败" + [cancelled] = await _events(run_id, "run.cancelled") + data = {k: v for k, v in cancelled.data.items() if k != "timing"} + assert data == {"reason": "rejected", "node_id": "gate", "label": GATES[kind][0]["data"]["label"], "kind": kind, + "decision": "reject", "actor": "张工", "note": "不对"} + # 收尾和取消一样:封存了,在线的流收到终态 + async with SessionLocal() as session: + row = await session.get(Run, run_id) + assert row.manifest_hash and row.manifest_seq == cancelled.seq + + +async def test_a_timeout_rejection_is_recorded_as_expired_reject(): + from app.process import tasks + + gate = node("gate", "human", label="把关", mode="approve", title="可以吗", stop_on_reject=True, due="PT30M", + on_timeout={"action": "reject"}) + run = await run_manager.start(graph=chain(START, gate, OUT), input_payload={"q": "x"}) + await _wait(run.id, ("interrupted",)) + await run_manager.wait_idle(run.id) + ticked = await tasks.tick(utcnow() + timedelta(hours=1)) + async with SessionLocal() as session: + mine = (await session.execute(select(Task.id).where(Task.run_id == run.id))).scalar_one() + assert mine in ticked + final = await _wait(run.id, ("cancelled", "failed", "succeeded")) + assert (final.status, final.end_reason) == ("cancelled", "rejected") + [cancelled] = await _events(run.id, "run.cancelled") + assert cancelled.data["decision"] == "expired_reject" and cancelled.data["actor"] is None + + +async def test_the_api_carries_end_reason_and_refuses_to_continue_a_rejected_run(client): + run_id = await _rejected_run("approve") + await _wait(run_id, ("cancelled",)) + body = (await client.get(f"/api/runs/{run_id}")).json() + assert (body["status"], body["end_reason"]) == ("cancelled", "rejected") + r = await client.post(f"/api/runs/{run_id}/continue", json={}) + assert r.status_code == 409 and r.json()["code"] == "run_rejected" + assert r.json()["detail"] == "被驳回的运行不能继续运行:改好内容后重新发起" + stop = await client.post(f"/api/runs/{run_id}/cancel") + assert stop.status_code == 409 and "被驳回" in stop.json()["detail"] + + +async def test_a_legacy_failed_rejection_cannot_be_continued_either(client): + """加 end_reason 之前被驳回的运行记的是 failed(封存不改写):失败的那一步上最近一条处理完的待办是驳回,也不许接着跑。 + 别的失败照旧能接着跑(这里断点不在,说的是没有断点,不是被驳回)。""" + async with SessionLocal() as session: + old = Run(workflow_name="老的驳回", status="failed", graph=chain(START, GATES["approve"][0], OUT), input={}, + error="人工驳回:不对", error_node_id="gate") + plain = Run(workflow_name="老的失败", status="failed", graph=chain(START, OUT), input={}, error="出错了") + session.add_all([old, plain]) + await session.flush() + session.add_all([ + RunEvent(run_id=old.id, seq=1, type="node.failed", node_id="gate", ts=1.0, data={"error": "人工驳回:不对"}), + Task(run_id=old.id, node_id="gate", kind="approve", title="可以吗", status="resolved", + outcome={"decision": "reject"}), + RunEvent(run_id=plain.id, seq=1, type="node.failed", node_id="out", ts=1.0, data={"error": "出错了"}), + ]) + await session.commit() + old_id, plain_id = old.id, plain.id + r = await client.post(f"/api/runs/{old_id}/continue", json={}) + assert r.status_code == 409 and r.json()["code"] == "run_rejected", r.text + other = await client.post(f"/api/runs/{plain_id}/continue", json={}) + assert other.status_code == 409 and other.json().get("code") != "run_rejected" + + +async def test_instance_state_says_the_same_for_the_new_shape_and_the_legacy_one(): + from app.apps.resolve import _trails, instance_state + + run_id = await _rejected_run("approve") + await _wait(run_id, ("cancelled",)) + async with SessionLocal() as session: + new = await session.get(Run, run_id) + legacy = Run(workflow_name="老的驳回", status="failed", graph=new.graph, input={}, error=new.error, + error_node_id="gate") + session.add(legacy) + await session.flush() + session.add_all([ + RunEvent(run_id=legacy.id, seq=1, type="node.failed", node_id="gate", ts=1.0, data={}), + Task(run_id=legacy.id, node_id="gate", kind="approve", title="可以吗", status="resolved", + outcome={"decision": "reject"}), + ]) + await session.commit() + trails = await _trails(session, [new, legacy]) + said = {"code": "cancelled", "text": "「把关」被驳回"} + assert instance_state(new, trails[new.id]) == said + assert instance_state(legacy, trails[legacy.id]) == said + + +async def test_stopping_is_recorded_as_stopped(): + gate, _ = GATES["approve"] + run = await run_manager.start(graph=chain(START, gate, OUT), input_payload={"q": "x"}) + await _wait(run.id, ("interrupted",)) + await run_manager.wait_idle(run.id) + assert await run_manager.cancel(run.id, actor="李工") == "cancelled" + final = await _wait(run.id, ("cancelled",)) + assert final.end_reason == "stopped" + [cancelled] = await _events(run.id, "run.cancelled") + assert cancelled.data["reason"] == "stopped" and cancelled.data["actor"] == "李工" + + +async def test_a_run_still_queued_at_restart_gets_its_run_failed_event(): + """(h):重启时排队中的运行改成 failed,以前只改状态,时间线里没有失败原因这一行。""" + from app.core.bus import bus + + async with SessionLocal() as session: + run = Run(workflow_name=f"排队-{uuid.uuid4().hex[:6]}", status="queued", graph={}, input={}, last_seq=2) + session.add(run) + await session.commit() + run_id = run.id + await run_manager.shutdown() + bus._seq.pop(run_id, None) + await run_manager.setup() + async with SessionLocal() as session: + row = await session.get(Run, run_id) + assert row.status == "failed" and row.started_at is None and row.finished_at is None + [failed] = await _events(run_id, "run.failed") + assert failed.seq == 3 and failed.data == {"error": row.error} + + +@pytest.mark.parametrize("kind", list(GATES)) +@pytest.mark.parametrize("tolerance", [{"on_error": "continue"}, {"retries": 1, "retry_backoff": 0.01}]) +async def test_a_rejection_is_neither_retried_nor_swallowed_by_on_error(kind, tolerance): + """「失败重试」「出错时继续」是给故障准备的。驳回是人的决定:重试只会把同一件事再问一遍人,on_error=continue 会把 + 驳回吞掉、接着往下走、最后以 succeeded 结束——通知、定时推送、发布单都看不到这次驳回(总方案 X9)。""" + gate, said = GATES[kind] + gate = {**gate, "data": {**gate["data"], "config": {**gate["data"]["config"], **tolerance}}} + run = await run_manager.start(graph=chain(START, gate, OUT), input_payload={"q": "x"}) + await _wait(run.id, ("interrupted",)) + await run_manager.wait_idle(run.id) + await run_manager.resume(run.id, {"approved": False, "note": "不对"}, actor="张工") + final = await _wait(run.id, ("cancelled", "failed", "succeeded", "interrupted")) + if final.status == "interrupted": + # 重试会再问一遍人:又停在同一个关卡上 + await run_manager.wait_idle(run.id) + final = await _wait(run.id, ("cancelled", "failed", "succeeded", "interrupted")) + assert (final.status, final.end_reason) == ("cancelled", "rejected"), (final.status, final.error) + assert final.error == said and final.error_node_id == "gate" + events = await _events(run.id) + assert not [e for e in events if e.type == "log" and (e.data or {}).get("code") == "node_retry"], "驳回不该重试" + assert not [e for e in events if e.type == "node.finished" and e.node_id == "out"], "驳回之后不该接着往下走" + # 节点这一步照旧留一条 node.failed(实例进度、时间线按它收尾这一步),只有一条 + assert len([e for e in events if e.type == "node.failed" and e.node_id == "gate"]) == 1 diff --git a/backend/tests/test_schedule_target.py b/backend/tests/test_schedule_target.py new file mode 100644 index 00000000..ad717c8d --- /dev/null +++ b/backend/tests/test_schedule_target.py @@ -0,0 +1,135 @@ +"""定时任务的目标(docs/design/next-round/00-总方案.md X4:定时只有一套)。 + +骨架给 Schedule 加了 target_kind / target_id;workflow_id 是老表上的非空约束,只加不改的迁移放宽不了。规矩只在 +scheduler/service.py 的一处封装里:workflow 目标的 target_id 同 workflow_id;不是工作流的目标 workflow_id 存空串。 +这里守: + +- 存量任务(target_kind 是缺省的 workflow、target_id 为空)照旧按 workflow_id 发起,行为一字不变; +- 建任务、改工作流都经同一处封装写目标,两列对得上; +- fire_once 按目标种类分派。这一波只实现 workflow,别的种类照实记「未能发起」并写明原因,不发起运行、不悄悄当成 + 工作流去跑空串; +- 接口这一波不暴露新种类(接口返回的形状不变,测试见 test_schedules_api.py 的 SCHEDULE_KEYS)。 + +工作流、任务名都是虚构的。 +""" +from __future__ import annotations + +import uuid +from datetime import datetime, timezone + +import pytest +from httpx import ASGITransport, AsyncClient +from sqlalchemy import delete, func, select + +from app.db.base import SessionLocal +from app.db.models import Run, Schedule, ScheduleFire, Workflow, WorkflowVersion +from app.engine.runner import run_manager +from app.main import app +from app.scheduler import service + +UTC = timezone.utc +NINE = datetime(2026, 10, 5, 1, 0, tzinfo=UTC) + + +@pytest.fixture(autouse=True) +async def engine_up(): + await run_manager.setup() + async with SessionLocal() as session: + await session.execute(delete(Schedule)) + await session.commit() + yield + await run_manager.shutdown() + + +def _node(nid, ntype, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}} + + +def _graph() -> dict: + return {"nodes": [_node("start", "input", fields=[{"name": "stat_date", "required": True}]), + _node("out", "output", fields=[{"name": "r", "value": "完成 {{ input.stat_date }}"}])], + "edges": [{"source": "start", "target": "out"}]} + + +async def _workflow() -> str: + async with SessionLocal() as session: + wf = Workflow(name=f"门店日报-{uuid.uuid4().hex[:6]}", graph=_graph(), status="published", published_version=1) + session.add(wf) + await session.flush() + session.add(WorkflowVersion(workflow_id=wf.id, version=1, graph=_graph())) + await session.commit() + return wf.id + + +def test_target_of_a_legacy_row_reads_the_workflow(): + """加列之前建的任务:target_kind 缺省 workflow、target_id 为空,目标就是 workflow_id。""" + row = Schedule(name="老任务", workflow_id="wf-legacy", cron="0 9 * * *") + assert service.target_of(row) == ("workflow", "wf-legacy") + row.target_kind = "workflow" + row.target_id = None + assert service.target_of(row) == ("workflow", "wf-legacy") + + +def test_target_columns_keep_the_two_columns_in_step(): + assert service.target_columns("workflow", "wf-1") == {"target_kind": "workflow", "target_id": "wf-1", + "workflow_id": "wf-1"} + # 非工作流目标:workflow_id 存空串(老表的非空约束),目标 id 只在 target_id + assert service.target_columns("eval_suite", "suite-1") == {"target_kind": "eval_suite", "target_id": "suite-1", + "workflow_id": ""} + row = Schedule(name="评测", cron="0 9 * * *", **service.target_columns("eval_suite", "suite-1")) + assert service.target_of(row) == ("eval_suite", "suite-1") + with pytest.raises(ValueError): + service.target_columns("nonsense", "x") + with pytest.raises(ValueError): + service.target_columns("workflow", "") + + +async def test_creating_and_retargeting_through_the_api_write_both_columns(): + wf_a, wf_b = await _workflow(), await _workflow() + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: + made = await client.post("/api/schedules", json={ + "name": "每日早报", "workflow_id": wf_a, "cron": "0 9 * * *", "timezone": "Asia/Shanghai", + "input_template": {"stat_date": "{{ schedule.yesterday }}"}}) + assert made.status_code == 201, made.text + body = made.json() + assert "target_kind" not in body and "target_id" not in body, "这一波接口不暴露新种类" + async with SessionLocal() as session: + row = await session.get(Schedule, body["id"]) + assert (row.target_kind, row.target_id, row.workflow_id) == ("workflow", wf_a, wf_a) + moved = await client.patch(f"/api/schedules/{body['id']}", json={"workflow_id": wf_b, "version": body["version"]}) + assert moved.status_code == 200, moved.text + async with SessionLocal() as session: + row = await session.get(Schedule, body["id"]) + assert (row.target_kind, row.target_id, row.workflow_id) == ("workflow", wf_b, wf_b) + + +async def test_a_legacy_schedule_still_fires_its_workflow(): + wf = await _workflow() + async with SessionLocal() as session: + row = Schedule(name="老任务", workflow_id=wf, cron="0 9 * * *", timezone="Asia/Shanghai", run_class="formal", + input_template={"stat_date": "{{ schedule.yesterday }}"}, next_run_at=NINE) + session.add(row) + await session.commit() + schedule_id = row.id + fire = await service.fire_once(schedule_id, NINE, "manual") + assert fire is not None and fire.status == "started" and fire.run_id + async with SessionLocal() as session: + run = await session.get(Run, fire.run_id) + assert run.workflow_id == wf and run.input == {"stat_date": "2026-10-04"} + + +async def test_a_target_kind_this_wave_cannot_fire_is_recorded_not_run(): + async with SessionLocal() as session: + row = Schedule(name="题集回归", cron="0 9 * * *", timezone="Asia/Shanghai", + **service.target_columns("eval_suite", "suite-x"), next_run_at=NINE) + session.add(row) + await session.commit() + schedule_id = row.id + runs_before = await session.scalar(select(func.count()).select_from(Run)) + fire = await service.fire_once(schedule_id, NINE, "manual") + assert fire is not None and fire.status == "failed_to_start" and fire.run_id is None + assert fire.reason == service.TARGET_UNSUPPORTED.format(kind=service.TARGET_LABEL["eval_suite"]) + async with SessionLocal() as session: + assert await session.scalar(select(func.count()).select_from(Run)) == runs_before, "不能发起任何运行" + assert await session.scalar(select(func.count()).select_from(ScheduleFire) + .where(ScheduleFire.schedule_id == schedule_id)) == 1 diff --git a/backend/tests/test_scheduler_webhook.py b/backend/tests/test_scheduler_webhook.py index 635a59ab..0b6207f4 100644 --- a/backend/tests/test_scheduler_webhook.py +++ b/backend/tests/test_scheduler_webhook.py @@ -426,11 +426,33 @@ async def test_without_webhook_old_fires_are_not_pushed_later(recorder): async def test_cancelled_run_is_not_pushed(recorder): - _, fire_id, _ = await make(run_status="cancelled") + _, fire_id, run_id = await make(run_status="cancelled") + await set_run(run_id, end_reason="stopped") await webhook.notify_due() assert recorder.requests == [] and (await fire_row(fire_id)).notified_status == "cancelled" +async def test_a_rejected_run_is_pushed_as_not_issued(recorder): + """被人驳回的那一期(cancelled + end_reason=rejected):按「运行结束」推「未出具(被驳回)」,写驳回的那一步,不写驳回的 + 备注和原话(可能带着数);订阅「运行失败」的不收(05-已知遗留 (a))。""" + _, fire_id, run_id = await make(run_status="cancelled", output={}, fmt="wecom", url=WECOM) + await set_run(run_id, end_reason="rejected", error_node_id="pay", error="「付款」未确认(被驳回),没有发送:金额 0042 不对", + graph={"nodes": [{"id": "pay", "type": "tool", "data": {"label": "付款", "config": {}}}], + "edges": []}) + assert await webhook.notify_due() == 1 + content = json.loads(recorder.requests[0].content)["markdown"]["content"] + assert "定时任务「门店周报」未出具(被驳回)" in content and '状态:被驳回' in content + assert "驳回节点:「付款」" in content and "失败" not in content + assert "0042" not in content and "金额" not in content + assert (await fire_row(fire_id)).notified_status == "cancelled" + [delivery] = await deliveries(fire_id) + assert delivery.event == "finished" + + _, only_failed, other = await make(run_status="cancelled", output={}, events=("failed",)) + await set_run(other, end_reason="rejected") + assert await webhook.notify_due() == 0 and (await fire_row(only_failed)).notified_status == "cancelled" + + async def test_waits_for_the_seal_before_pushing(recorder): _, fire_id, run_id = await make(sealed=False) assert await webhook.notify_due() == 0 and (await fire_row(fire_id)).notified_status is None diff --git a/backend/tests/test_signer_verified.py b/backend/tests/test_signer_verified.py new file mode 100644 index 00000000..13e98ee0 --- /dev/null +++ b/backend/tests/test_signer_verified.py @@ -0,0 +1,65 @@ +"""导入清单、累积快照清单、作废记录、清除原件记录如实写「是否认证」(docs/design/next-round/05-已知遗留.md (c))。 + +以前四处都写死 False:开了登录、署名就是登录的人,封存的证据里却记着「未认证」,和审计表、待办派单、run.origin 的 +verified 对不上。现在由 api/deps.signer_verified 在请求那一刻定下、和署名一起往下传:开登录为真,关登录(自报的署名) +为假。已封存的旧清单不改写(关登录时的形状见 test_table_imports_api、test_source_versions_api、test_datasources_upload)。 +""" +from __future__ import annotations + +import pytest + +from app.api import deps +from app.core import artifact_store +from app.db.models import TableImport +from tests.fakes import auth_env +from tests.test_table_imports_api import first_import, get, store, unique # noqa: F401 - store 是 autouse 夹具 + + +@pytest.fixture +async def admin(): + await auth_env.reset() + c = await auth_env.turn_on_local("root") + try: + yield c + finally: + await c.aclose() + await auth_env.reset() + + +def test_signer_verified_follows_the_request_identity(): + token = deps._principal.set(deps.Principal(username="root", role="admin", verified=True, mode="local")) + try: + assert deps.signer_verified() is True + finally: + deps._principal.reset(token) + token = deps._principal.set(deps.Principal(username="张三", role="admin", verified=False, mode="off")) + try: + assert deps.signer_verified() is False, "关登录时署名是自报的" + finally: + deps._principal.reset(token) + assert deps.signer_verified() is False, "不在请求里(启动整理、调度循环)不算认证" + + +def test_the_records_write_what_they_are_given(): + from app.data import source_versions + + assert source_versions._revoked_record("理由", "root", True)["signed_by_verified"] is True + assert source_versions._revoked_record("理由", "root")["signed_by_verified"] is False + assert source_versions._revoked_record("理由", None, True)["signed_by_verified"] is False, "没有署名谈不上认证" + + +async def test_login_on_import_manifest_and_purge_record_say_verified(admin): + out = (await first_import(admin, unique("signer")))["commit"] + imp = await get(TableImport, out["import_id"]) + manifest = artifact_store.load(imp.manifest_artifact) + assert manifest["signed_by"] == {"name": "root", "verified": True} + assert imp.signed_by == "root" + + source_id = out["source"]["id"] + records = (await admin.get(f"/api/datasources/{source_id}/imports")).json() + assert [(r["signed_by"], r["signed_by_verified"]) for r in records] == [("root", True)], "导入记录按清单里记下的说" + resp = await admin.post(f"/api/datasources/{source_id}/imports/{imp.id}/purge-raw", + json={"reason": "含有不应保留的内容", "signed_by": "随手填的名字"}) + assert resp.status_code == 200, resp.text + purged = resp.json()["import"]["purged"] + assert (purged["signed_by"], purged["signed_by_verified"]) == ("root", True) diff --git a/backend/tests/test_spend_ledger.py b/backend/tests/test_spend_ledger.py new file mode 100644 index 00000000..364f9f12 --- /dev/null +++ b/backend/tests/test_spend_ledger.py @@ -0,0 +1,82 @@ +"""每日账泛化成 Ledger(key)(docs/design/next-round/01-效果度量与回归评测.md 做法 17)。 + +核查、裁判以前各写一份一模一样的「预留 → 找补 → 按本地日期滚动」,评测要是再抄一份就是第三份。现在三本账是同一个 +Ledger 的三个实例,各记各的键、各管各的上限:一本触顶不挤占另外两本。裁判、核查原有的行为(模块级的 SPEND_KEY、 +daily_spend、charge / settle)不变,已有的测试照旧守着;这里守账本本身和互不挤占。 +""" +from __future__ import annotations + +import pytest +from sqlalchemy import delete + +from app.db.base import SessionLocal +from app.db.models import Setting +from app.engine import judge, spend +from app.engine.judge import judge_spend +from app.engine.spend import Ledger, eval_spend, verify_spend + + +@pytest.fixture(autouse=True) +async def _clean(): + keys = [verify_spend.key, judge_spend.key, eval_spend.key, "test_ledger"] + async with SessionLocal() as session: + await session.execute(delete(Setting).where(Setting.key.in_(keys))) + await session.commit() + yield + async with SessionLocal() as session: + await session.execute(delete(Setting).where(Setting.key.in_(keys))) + await session.commit() + + +def test_three_books_three_keys(): + assert (verify_spend.key, judge_spend.key, eval_spend.key) == ("verify_spend", "judge_spend", "eval_spend") + # 老名字照旧指向各自那本:核查的模块函数、裁判的 SPEND_KEY / daily_spend + assert spend.SPEND_KEY == verify_spend.key + assert judge.SPEND_KEY == judge_spend.key + + +async def test_reserve_then_settle_and_the_limit(): + book = Ledger("test_ledger") + day = await book.charge(0.4, limit=1.0, priced=True) + assert day is not None + assert await book.charge(0.7, limit=1.0, priced=True) is None, "加上这笔超过上限就不记" + await book.settle(day, -0.1) + got = await book.daily() + assert got["usd"] == pytest.approx(0.3) and got["calls"] == 1 and got["unpriced_calls"] == 0 + # 估不出金额的照记次数,不比上限 + assert await book.charge(0.0, limit=None, priced=False) is not None + assert (await book.daily())["unpriced_calls"] == 1 + + +async def test_a_new_day_starts_from_zero(): + day = {"today": "2026-09-28"} + book = Ledger("test_ledger", today=lambda: day["today"]) + first = await book.charge(0.9, limit=1.0, priced=True) + assert first == "2026-09-28" + assert await book.charge(0.2, limit=1.0, priced=True) is None + day["today"] = "2026-09-29" + assert await book.charge(0.2, limit=1.0, priced=True) == "2026-09-29" + await book.settle(first, -0.5) # 跨了天的预留不找补:今天从零算 + assert (await book.daily())["usd"] == pytest.approx(0.2) + + +async def test_one_book_hitting_its_cap_does_not_squeeze_the_others(): + """核查花光了当天的额度,裁判、评测照常记:三本账互不挤占。""" + assert await verify_spend.charge(2.0, limit=2.0, priced=True) is not None + assert await verify_spend.charge(0.01, limit=2.0, priced=True) is None + assert await judge_spend.charge(1.0, limit=2.0, priced=True) is not None + assert await eval_spend.charge(1.5, limit=2.0, priced=True) is not None + assert (await judge.daily_spend())["usd"] == pytest.approx(1.0) + assert (await spend.daily_spend())["usd"] == pytest.approx(2.0) + assert (await eval_spend.daily())["usd"] == pytest.approx(1.5) + + +async def test_all_books_roll_over_on_the_local_date(monkeypatch): + """三本账按同一个本地日期滚动(spend._today);裁判那本另留一个 judge._today,换它只动裁判这一本。""" + monkeypatch.setattr(spend, "_today", lambda: "2026-01-02") + assert await judge_spend.charge(0.1, limit=None, priced=True) == "2026-01-02" + assert (await judge.daily_spend())["date"] == "2026-01-02" + assert (await eval_spend.daily())["date"] == "2026-01-02" + monkeypatch.setattr(judge, "_today", lambda: "2026-01-03") + assert (await judge.daily_spend())["date"] == "2026-01-03" + assert (await verify_spend.daily())["date"] == "2026-01-02" diff --git a/backend/tests/test_sqlcheck.py b/backend/tests/test_sqlcheck.py index 2f8bf546..8277908a 100644 --- a/backend/tests/test_sqlcheck.py +++ b/backend/tests/test_sqlcheck.py @@ -555,3 +555,15 @@ async def test_checker_builds_relation_graph_lazily_and_reuses_it(cache, monkeyp checker.check(FANOUT[0]) checker.check(FANOUT[1]) assert calls == [1] + + +def test_a_postgres_enum_whose_type_name_says_date_is_not_a_time_column(): + """PG 的枚举编译出来就是类型名本身(update_date_kind 里也有 DATE):探查记下了 enum_values 的列,SQL 检查和剖析一样 + 不把它当时间列。以前 SQL 检查认时间列时没把取值列表传进去,剖析不认、SQL 检查却认,两边对不上。""" + from app.data.sqlcheck import _Table + + meta = {"columns": [{"name": "kind", "type": "update_date_kind", "enum_values": ["a", "b"]}, + {"name": "paid_on", "type": "DATE"}]} + table = _Table(alias="t", name="t", meta=meta, notes={}, order=0) + assert not table.temporal("kind") + assert table.temporal("paid_on") diff --git a/backend/tests/test_usage_meter.py b/backend/tests/test_usage_meter.py new file mode 100644 index 00000000..c69259f6 --- /dev/null +++ b/backend/tests/test_usage_meter.py @@ -0,0 +1,283 @@ +"""用量计量(providers/usage.py;细案 01 做法 3):助手建图(含挑表、接着写、自查修正)和复核的用量也记。 + +守的是: +- 按模型累计 tokens 和调用次数,金额按价格目录算;目录外的模型、回复没带用量(而目录价又不是 0)的调用记 unpriced_calls, + 不当成 0 元; +- metered 只计生成过程自己的调用(含它在一步里起的任务);调用方在两次 yield 之间做的事(发起运行)不计——运行的用量 + 另有 llm.end,包进来就重复了;嵌套时内层并进外层; +- answer_ops 末尾的 usage 记下这一轮建图的每一次调用(起草、自查修正各一次);复核的结果带 usage。 + +模型都是剧本,不调真接口;价格用目录里登记过的模型 id,名字之外都是虚构的。 +""" +from __future__ import annotations + +import asyncio +import json +from typing import Any + +import pytest +from httpx import ASGITransport, AsyncClient +from langchain_core.language_models.chat_models import BaseChatModel +from langchain_core.messages import AIMessage, AIMessageChunk +from langchain_core.outputs import ChatGeneration, ChatGenerationChunk, ChatResult + +from app.main import app +from app.providers import usage +from app.providers.catalog import estimate_cost + +PRICED = "claude-opus-5" # 价格目录里有 +UNPRICED = "made-up-model-x" # 价格目录里没有 +FREE = "mock-fast" # 目录价是 0 + + +class Scripted(BaseChatModel): + """真的 LangChain 模型:回调照常触发。每次调用吐 replies 里的下一份(用完重复最后一份),用量按 tokens 报; + report=False 时回复不带用量(有的网关流式不给)。""" + + model: str = PRICED + replies: list[list[str]] = [["好的"]] + tokens: tuple[int, int] = (1000, 200) + report: bool = True + calls: int = 0 + + @property + def _llm_type(self) -> str: + return "scripted" + + @property + def _identifying_params(self) -> dict[str, Any]: + return {"model": self.model} + + def _next(self) -> list[str]: + lines = self.replies[min(self.calls, len(self.replies) - 1)] + self.calls += 1 + return lines + + def _usage(self) -> dict[str, int] | None: + if not self.report: + return None + inp, out = self.tokens + return {"input_tokens": inp, "output_tokens": out, "total_tokens": inp + out} + + def _generate(self, messages, stop=None, run_manager=None, **kwargs) -> ChatResult: + message = AIMessage(content="\n".join(self._next())) + if (meta := self._usage()) is not None: + message.usage_metadata = meta + return ChatResult(generations=[ChatGeneration(message=message)]) + + async def _astream(self, messages, stop=None, run_manager=None, **kwargs): + """像 Anthropic 那样报用量:第一段带输入 token,流末尾单独一段带输出 token。""" + lines = self._next() + inp, out = self.tokens + for i, line in enumerate(lines): + meta = {"input_tokens": inp, "output_tokens": 0, "total_tokens": inp} if self.report and i == 0 else None + yield ChatGenerationChunk(message=AIMessageChunk(content=line + "\n", **({"usage_metadata": meta} if meta + else {}))) + if self.report: + yield ChatGenerationChunk(message=AIMessageChunk(content="", usage_metadata={ + "input_tokens": 0, "output_tokens": out, "total_tokens": out})) + + +# -------------------------------------------------------------------------- +# 计量本身 +# -------------------------------------------------------------------------- + + +async def test_metering_counts_by_model_and_prices_from_the_catalog(): + m = usage.meter() + with usage.metering(m): + await Scripted(model=PRICED).ainvoke("你好") + await Scripted(model=PRICED).ainvoke("再来") + await Scripted(model=UNPRICED).ainvoke("你好") + await Scripted(model=PRICED + "-silent", report=False).ainvoke("你好") + await Scripted(model=FREE, report=False).ainvoke("你好") + got = m.summary() + assert got["by_model"][PRICED] == {"input_tokens": 2000, "output_tokens": 400, "calls": 2, + "cost_usd": round(estimate_cost(PRICED, 2000, 400), 6), "unpriced_calls": 0} + assert got["by_model"][UNPRICED]["unpriced_calls"] == 1 and got["by_model"][UNPRICED]["cost_usd"] == 0 + # 目录外的模型名(多了后缀)照样算估不出;目录价是 0 的本地模型没报用量也不算估不出 + assert got["by_model"][PRICED + "-silent"]["unpriced_calls"] == 1 + assert got["by_model"][FREE]["unpriced_calls"] == 0 + assert got["calls"] == 5 and got["unpriced_calls"] == 2 + assert got["cost_usd"] == pytest.approx(estimate_cost(PRICED, 2000, 400)) + + +async def test_outside_the_meter_nothing_is_counted(): + m = usage.meter() + with usage.metering(m): + pass + await Scripted().ainvoke("你好") + assert m.summary()["calls"] == 0 + assert usage.current() is None + + +async def test_metered_counts_the_generator_and_its_tasks_but_not_the_caller(): + """生成过程里的调用(含它起的任务)计进来;调用方在两次 yield 之间发起的「运行」不计。""" + m = usage.meter() + leaked: list[asyncio.Task] = [] + + async def generation(): + await Scripted(model=PRICED).ainvoke("起草") + # 心跳那一层每一步起一个任务:任务在步里创建,拷走的上下文带着计量 + await asyncio.ensure_future(Scripted(model=PRICED).ainvoke("挑表")) + yield "first" + await Scripted(model=PRICED).ainvoke("自查修正") + yield "second" + + seen = [] + async for item in usage.metered(generation(), m): + seen.append(item) + # 调用方拿到一条操作后自己调模型(相当于发起运行):它的上下文里没有计量 + await Scripted(model=UNPRICED).ainvoke("运行里的调用") + leaked.append(asyncio.ensure_future(Scripted(model=UNPRICED).ainvoke("运行任务"))) + await asyncio.gather(*leaked) + got = m.summary() + assert seen == ["first", "second"] + assert got["by_model"] == {PRICED: got["by_model"][PRICED]} and got["calls"] == 3 + assert usage.current() is None + + +async def test_nested_meters_fold_into_the_outer_one(): + outer, inner = usage.meter(), usage.meter() + + async def generation(): + await Scripted(model=PRICED).ainvoke("起草") + yield 1 + + with usage.metering(outer): + await Scripted(model=PRICED).ainvoke("外层自己的") + async for _ in usage.metered(generation(), inner): + pass + with usage.metering(usage.meter()): + await Scripted(model=PRICED).ainvoke("复核") + assert inner.summary()["calls"] == 1 + assert outer.summary()["calls"] == 3 + + +# -------------------------------------------------------------------------- +# answer_ops 末尾的 usage、复核的 usage +# -------------------------------------------------------------------------- + + +def _ops(*ops: dict) -> list[str]: + return [json.dumps(o, ensure_ascii=False) for o in ops] + + +GRAPH = _ops( + {"op": "plan", "summary": "数一数"}, + {"op": "add_node", "node": {"id": "start", "type": "input", "label": "输入", + "config": {"fields": [{"name": "question"}]}}}, + {"op": "add_node", "node": {"id": "lp", "type": "loop", "label": "循环", + "config": {"mode": "while", "condition": "foo(vars.x)", "max_iterations": 2}}}, + {"op": "add_edge", "edge": {"source": "start", "target": "lp"}}, + {"op": "add_node", "node": {"id": "body", "type": "transform", "label": "体", + "config": {"mode": "expression", "expression": "1"}}}, + {"op": "add_edge", "edge": {"source": "lp", "target": "body", "sourceHandle": "body"}}, + {"op": "add_edge", "edge": {"source": "body", "target": "lp"}}, + {"op": "add_node", "node": {"id": "out", "type": "output", "label": "成果", + "config": {"fields": [{"name": "r", "value": "1"}]}}}, + {"op": "add_edge", "edge": {"source": "lp", "target": "out", "sourceHandle": "done"}}, + {"op": "done", "explanation": "数一数", "run": True}, +) +FIX = _ops({"op": "update_node", "id": "lp", "config": {"mode": "while", "condition": "len(vars.x) > 1", + "max_iterations": 2}}, + {"op": "done", "explanation": "改好了"}) + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +async def test_answer_ops_ends_with_the_usage_of_every_build_call(client, monkeypatch): + """起草、自查修正各一次。生成过程读到 done 就关流(后面的闲话不要),流末尾才报的输出 token 拿不到:这两次照样计次数、 + 计已报的输入 token,记成估不出(unpriced_calls),金额只是估得出的那部分——不悄悄少算。""" + import app.api.copilot as copilot + + model = Scripted(model=PRICED, replies=[GRAPH, FIX], tokens=(3000, 500)) + + async def _model(*_a, **_k): + return model, PRICED + + monkeypatch.setattr(copilot, "get_chat_model", _model) + events = [] + async with client.stream("POST", "/api/copilot/generate-stream", json={"instruction": "数一数", "intent": "answer"}) as r: + async for line in r.aiter_lines(): + if line.startswith("data: "): + events.append(json.loads(line[6:])) + assert [e["op"] for e in events if e["op"] == "check"] and events[-2]["op"] == "final" + spent = events[-1] + assert spent["op"] == "usage", spent + assert model.calls == 2, spent # 起草一次、自查修正一次 + assert spent["by_model"][PRICED]["calls"] == 2, spent + assert spent["by_model"][PRICED]["input_tokens"] == 6000 and spent["by_model"][PRICED]["output_tokens"] == 0 + assert spent["cost_usd"] == pytest.approx(estimate_cost(PRICED, 6000, 0), abs=1e-6) + assert spent["unpriced_calls"] == 2, "关流前没报全的两次记成估不出" + + +async def test_a_build_on_an_unpriced_model_says_it_cannot_be_priced(client, monkeypatch): + import app.api.copilot as copilot + + model = Scripted(model=UNPRICED, replies=[_ops({"op": "reply", "text": "已经查过了"})]) + + async def _model(*_a, **_k): + return model, UNPRICED + + monkeypatch.setattr(copilot, "get_chat_model", _model) + events = [] + async with client.stream("POST", "/api/copilot/generate-stream", json={"instruction": "刚才那个数是多少"}) as r: + async for line in r.aiter_lines(): + if line.startswith("data: "): + events.append(json.loads(line[6:])) + assert [e["op"] for e in events] == ["model", "reply", "usage"] + assert events[-1]["calls"] == 1 and events[-1]["unpriced_calls"] == 1 and events[-1]["cost_usd"] == 0 + + +async def test_a_stream_read_to_the_end_is_fully_priced(): + """读完的流(直接回答那一行之后模型自己停了):输入、输出 token 都报全了,金额估得出。""" + m = usage.meter() + + async def generation(): + async for _ in Scripted(model=PRICED, tokens=(400, 60)).astream("问"): + pass + yield 1 + + async for _ in usage.metered(generation(), m): + pass + row = m.summary()["by_model"][PRICED] + assert (row["input_tokens"], row["output_tokens"], row["unpriced_calls"]) == (400, 60, 0) + assert row["cost_usd"] == round(estimate_cost(PRICED, 400, 60), 6) + + +async def test_the_review_carries_its_own_usage(client, monkeypatch): + import app.api.copilot as copilot + from app.core.events import EventType + from app.db.base import SessionLocal + from app.db.models import Run, RunEvent + + reply = json.dumps({"note": "agent 用满了步数,结论不完整。", "answer": "", "retry": True}, ensure_ascii=False) + model = Scripted(model=PRICED, replies=[[reply]], tokens=(800, 100)) + + async def _model(*_a, **_k): + return model, PRICED + + monkeypatch.setattr(copilot, "get_chat_model", _model) + async with SessionLocal() as session: + run = Run(status="succeeded", output={"answer": "大概是 3 家"}, graph={}) + session.add(run) + await session.flush() + session.add(RunEvent(run_id=run.id, seq=1, type=EventType.LOG, node_id="a", ts=0.0, + data={"level": "warn", "code": "step_limit", "message": "用满了步数"})) + await session.commit() + run_id = run.id + body = (await client.post("/api/copilot/review", json={"run_id": run_id, "question": "有几家门店"})).json() + assert body["usage"]["calls"] == 1 and body["usage"]["by_model"][PRICED]["input_tokens"] == 800 + # 干净的运行不调模型:usage 全 0 + async with SessionLocal() as session: + clean = Run(status="succeeded", output={"answer": "3 家"}, graph={}) + session.add(clean) + await session.commit() + clean_id = clean.id + body = (await client.post("/api/copilot/review", json={"run_id": clean_id, "question": "有几家门店"})).json() + assert body["verdict"] == "ok" and body["usage"]["calls"] == 0 and body["usage"]["by_model"] == {} diff --git a/backend/tests/test_web_security.py b/backend/tests/test_web_security.py new file mode 100644 index 00000000..cc74f582 --- /dev/null +++ b/backend/tests/test_web_security.py @@ -0,0 +1,235 @@ +"""安全响应头(04 细案 C1):app/web_security.py 的纯 ASGI 中间件,加上 mount_web 给页面算内联脚本的哈希。 + +要守住的: +- 页面、/api 的响应(连同登录检查自己回的 401)都带 nosniff、Referrer-Policy 和 CSP frame-ancestors; +- frame-ancestors 缺省 'self',只有 'self' 时另加 X-Frame-Options: SAMEORIGIN;配了 AGENTLAB_FRAME_ANCESTORS 白名单时 + 以 CSP 为准、不发 XFO(XFO 写不了白名单,两个都发的话老浏览器按 XFO 把白名单里的门户也拦了); +- 分享页 /s 用 AGENTLAB_EMBED_ORIGINS、Referrer-Policy: no-referrer(路由第 3 波才加,头的规则先守住); +- 严格的 script-src 只发 Content-Security-Policy-Report-Only,只在页面(index.html)上发,哈希就是构建产物里内联 + 脚本的 sha256;接口返回的 HTML(导出的报告自带 meta CSP)不带它; +- 接口自己设过的同名头不覆盖、不重复;WebSocket 不经过它; +- 白名单写错了启动时就报错。 +""" +from __future__ import annotations + +import base64 +import hashlib +import json +import os +import subprocess +import sys +from pathlib import Path + +import httpx +import pytest +from fastapi import APIRouter, FastAPI, WebSocket +from fastapi.responses import HTMLResponse, Response +from fastapi.testclient import TestClient +from pydantic import ValidationError + +from app.core.config import BACKEND_DIR, PROJECT_DIR, Settings +from app.main import app as main_app +from app.web_security import SecurityHeaders +from tests.fakes import auth_env + +THEME = "\n try { document.documentElement.setAttribute('data-theme', 'light') } catch (e) {}\n " +INDEX = ("" + "" + "
") +HASH = base64.b64encode(hashlib.sha256(THEME.encode()).digest()).decode() +REPORT = ("" + "") + + +@pytest.fixture +def dist(tmp_path: Path) -> Path: + root = tmp_path / "dist" + (root / "assets").mkdir(parents=True) + (root / "index.html").write_text(INDEX) + (root / "assets" / "index-AbC123.js").write_text("console.log('bundle')") + return root + + +def _app(dist: Path | None = None, **config) -> FastAPI: + app = FastAPI() + api = APIRouter(prefix="/api") + + @api.get("/ping") + async def ping() -> dict[str, str]: + return {"pong": "1"} + + @api.get("/report.html") + async def report() -> HTMLResponse: + return HTMLResponse(REPORT) + + @api.get("/download") + async def download() -> Response: + # 文件下载自己设了 nosniff(api/files.py):不能再加一份 + return Response(b"x", headers={"X-Content-Type-Options": "nosniff", "Referrer-Policy": "no-referrer"}) + + @api.websocket("/echo") + async def echo(ws: WebSocket) -> None: + await ws.accept() + await ws.send_text(await ws.receive_text()) + await ws.close() + + app.include_router(api) + if dist is not None: + from app.web import mount_web + + mount_web(app, dist) + app.add_middleware(SecurityHeaders, **config) + return app + + +async def _get(app: FastAPI, path: str) -> httpx.Response: + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://t") as client: + return await client.get(path) + + +def _single(r: httpx.Response, name: str) -> str | None: + values = r.headers.get_list(name) + assert len(values) <= 1, f"{name} 发了 {len(values)} 份:{values}" + return values[0] if values else None + + +async def test_api_and_page_carry_the_defaults(dist: Path) -> None: + app = _app(dist, frame_ancestors=[], embed_origins=[]) + for path in ("/api/ping", "/api/nope", "/", "/studio", "/assets/index-AbC123.js"): + r = await _get(app, path) + assert _single(r, "x-content-type-options") == "nosniff", path + assert _single(r, "referrer-policy") == "strict-origin-when-cross-origin", path + assert _single(r, "content-security-policy") == "frame-ancestors 'self'", path + assert _single(r, "x-frame-options") == "SAMEORIGIN", path + + +async def test_report_only_script_policy_is_only_on_the_page(dist: Path) -> None: + app = _app(dist, frame_ancestors=[], embed_origins=[]) + for path in ("/", "/studio", "/runs/abc/evidence"): + r = await _get(app, path) + assert r.text == INDEX + assert _single(r, "content-security-policy-report-only") == f"script-src 'self' 'sha256-{HASH}'", path + # 脚本文件、接口不带;接口返回的 HTML(导出的报告)自带 meta CSP,不加一份对不上哈希的上报策略,正文原样 + for path in ("/assets/index-AbC123.js", "/api/ping", "/api/report.html"): + r = await _get(app, path) + assert "content-security-policy-report-only" not in r.headers, path + r = await _get(app, "/api/report.html") + assert r.text == REPORT + + +async def test_whitelist_wins_and_drops_x_frame_options() -> None: + app = _app(frame_ancestors=["https://portal.example.com", "https://*.corp.example"], embed_origins=[]) + r = await _get(app, "/api/ping") + assert _single(r, "content-security-policy") == \ + "frame-ancestors 'self' https://portal.example.com https://*.corp.example" + assert "x-frame-options" not in r.headers, "有白名单时以 CSP 为准:XFO 写不了白名单" + + +async def test_share_page_uses_the_embed_origins(dist: Path) -> None: + app = _app(dist, frame_ancestors=["https://portal.example.com"], embed_origins=["https://bi.example.org"]) + for path in ("/s", "/s/", "/s/anything"): + r = await _get(app, path) + assert _single(r, "content-security-policy") == "frame-ancestors 'self' https://bi.example.org", path + assert _single(r, "referrer-policy") == "no-referrer", path + assert "x-frame-options" not in r.headers, path + # 只认 /s 这一段:/settings、/studio 照整站的规则 + for path in ("/settings", "/studio", "/share"): + r = await _get(app, path) + assert _single(r, "content-security-policy") == "frame-ancestors 'self' https://portal.example.com", path + assert _single(r, "referrer-policy") == "strict-origin-when-cross-origin", path + # 没配嵌入白名单:分享页只能被本站嵌,同样另加 XFO + r = await _get(_app(dist, frame_ancestors=[], embed_origins=[]), "/s") + assert _single(r, "content-security-policy") == "frame-ancestors 'self'" + assert _single(r, "x-frame-options") == "SAMEORIGIN" + + +async def test_headers_already_set_are_kept() -> None: + r = await _get(_app(frame_ancestors=[], embed_origins=[]), "/api/download") + assert _single(r, "x-content-type-options") == "nosniff" + assert _single(r, "referrer-policy") == "no-referrer" + assert _single(r, "content-security-policy") == "frame-ancestors 'self'" + + +def test_websocket_is_not_touched() -> None: + with TestClient(_app(frame_ancestors=[], embed_origins=[])) as client: + with client.websocket_connect("/api/echo") as ws: + ws.send_text("hi") + assert ws.receive_text() == "hi" + + +async def test_main_app_and_its_own_refusals_carry_the_headers() -> None: + """真实的 app.main:加在最外层,登录检查自己回的 401、CORS 预检也带。""" + app = main_app + r = await _get(app, "/api/health") + assert _single(r, "content-security-policy") == "frame-ancestors 'self'" + assert _single(r, "x-frame-options") == "SAMEORIGIN" and _single(r, "x-content-type-options") == "nosniff" + await auth_env.reset() + try: + await auth_env.turn_on_local() + r = await _get(app, "/api/workflows") + assert r.status_code == 401 and r.json()["code"] == "auth_required" + assert _single(r, "x-content-type-options") == "nosniff" + assert _single(r, "content-security-policy") == "frame-ancestors 'self'" + finally: + await auth_env.reset() + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://t") as client: + r = await client.options("/api/health", headers={"Origin": "http://localhost:5273", + "Access-Control-Request-Method": "GET"}) + assert r.status_code == 200 and _single(r, "x-content-type-options") == "nosniff" + + +def test_origin_lists_parse_and_bad_entries_fail_at_startup(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("AGENTLAB_FRAME_ANCESTORS", raising=False) + monkeypatch.delenv("AGENTLAB_EMBED_ORIGINS", raising=False) + assert Settings().frame_ancestors == [] and Settings().embed_origins == [] + monkeypatch.setenv("AGENTLAB_FRAME_ANCESTORS", " https://portal.example.com/ , https://*.corp.example,'self'") + monkeypatch.setenv("AGENTLAB_EMBED_ORIGINS", '["http://bi.example.org:8080"]') + got = Settings() + assert got.frame_ancestors == ["https://portal.example.com", "https://*.corp.example"], "末尾的 / 去掉,'self' 本来就有" + assert got.embed_origins == ["http://bi.example.org:8080"] + monkeypatch.delenv("AGENTLAB_EMBED_ORIGINS") + # 只收「协议://主机[:端口]」:路径、通配一切、没写协议、'none' 这类写法启动时就报出来,别到嵌入失败才发现 + for bad in ("*", "portal.example.com", "https://portal.example.com/app", "ftp://portal.example.com", "'none'", + "https://", "https://portal.example.com?x=1"): + monkeypatch.setenv("AGENTLAB_FRAME_ANCESTORS", bad) + with pytest.raises(ValidationError, match="AGENTLAB_FRAME_ANCESTORS"): + Settings() + + +def test_deploy_env_documents_both_lists() -> None: + text = (PROJECT_DIR / "deploy" / "agentlab.env").read_text() + assert "# AGENTLAB_FRAME_ANCESTORS=" in text and "# AGENTLAB_EMBED_ORIGINS=" in text + + +_PROBE = r""" +import asyncio, json, httpx +from app.main import app + +async def main(): + t = httpx.ASGITransport(app=app) + async with httpx.AsyncClient(transport=t, base_url="http://t") as c: + out = {} + for p in ("/", "/api/health"): + r = await c.get(p) + out[p] = {k: r.headers.get(k) for k in ("content-security-policy", "x-frame-options", + "content-security-policy-report-only")} + print(json.dumps(out)) + +asyncio.run(main()) +""" + + +def test_main_app_reads_the_whitelist_and_hashes_the_page(dist: Path, tmp_path: Path) -> None: + """真实的 app.main:环境变量里的白名单生效、不发 XFO;托管页面时 index.html 带上报策略。settings、app 都是模块级 + 单例,起一个子进程导入。""" + env = {**os.environ, "AGENTLAB_DATA_DIR": str(tmp_path / "data"), "AGENTLAB_WEB_DIST": str(dist), + "AGENTLAB_SECRET_KEY": "test-only-not-a-real-key", "AGENTLAB_FRAME_ANCESTORS": "https://portal.example.com"} + proc = subprocess.run([sys.executable, "-c", _PROBE], cwd=BACKEND_DIR, env=env, capture_output=True, text=True, + timeout=120) + assert proc.returncode == 0, proc.stderr[-2000:] + out = json.loads(proc.stdout.strip().splitlines()[-1]) + for path in ("/", "/api/health"): + assert out[path]["content-security-policy"] == "frame-ancestors 'self' https://portal.example.com", path + assert out[path]["x-frame-options"] is None, path + assert out["/"]["content-security-policy-report-only"] == f"script-src 'self' 'sha256-{HASH}'" + assert out["/api/health"]["content-security-policy-report-only"] is None diff --git a/deploy/agentlab.env b/deploy/agentlab.env index 15b615d8..d121040b 100644 --- a/deploy/agentlab.env +++ b/deploy/agentlab.env @@ -23,7 +23,9 @@ BIND_ADDR=0.0.0.0 # 不能再改登录方式。local:第一次打开登录页时建第一个管理员(只在还没有管理员时可用);oidc 的身份提供方地址、 # 客户端、管理员映射在「设置 → 身份」里配。忘了密码:docker exec -it agentlab python -m app.auth reset-admin # AGENTLAB_AUTH_MODE=local -# 对外的地址(在反向代理后面时填):OIDC 的回调地址按它拼,非 GET 请求的 Origin 核对也认它 +# 对外访问地址(在反向代理后面时填):通知和定时任务 Webhook 里的链接、OIDC 的回调地址都按它拼,非 GET 请求的 +# Origin 核对也认它。设了这一项就以它为准,界面上「设置 → 通用」里的「对外访问地址」显示「由环境变量指定」、不能改; +# 不设时用界面上填的那一项。写错了(没有 http:// 或 https://、带 ? 或 #)服务启动时报错 # AGENTLAB_PUBLIC_URL=https://agentlab.example.internal # 会话:闲置多少小时、最长多少天过期 # AGENTLAB_AUTH_SESSION_IDLE_HOURS=12 @@ -31,6 +33,17 @@ BIND_ADDR=0.0.0.0 # 会话 cookie 是否带 Secure:留空按请求是不是 https 决定;反向代理终结了 https 时填 true # AGENTLAB_AUTH_COOKIE_SECURE= +# ---- 嵌入(可选)---- +# 默认只有 AgentLab 自己能把页面放进 iframe(响应头 Content-Security-Policy: frame-ancestors 'self',另加 +# X-Frame-Options: SAMEORIGIN),别的站点嵌进去会被浏览器拦下,防点击劫持。HSTS 不在这里加,交给终结 https 的反向代理。 +# 同站的门户(和 AgentLab 在同一个可注册域名下,比如 portal.example.internal 和 agentlab.example.internal)要嵌整站时, +# 把门户的地址写在下面,逗号分隔,只写 协议://主机[:端口],主机可以写 *.example.internal。填了之后只发 +# frame-ancestors、不再发 X-Frame-Options。登录用的 cookie 只随同站的请求带上,跨站的门户嵌整站登录不了。 +# 写错了(带路径、没写协议、只写 *)服务启动时报错。 +# AGENTLAB_FRAME_ANCESTORS=https://portal.example.internal +# 分享页(/s)允许被哪些站点嵌,写法同上。分享页只认这一项、不认上面那一项。分享链接还没有上线,现在填了不起作用 +# AGENTLAB_EMBED_ORIGINS=https://bi.example.internal + # ---- 数据 ---- # 数据目录:数据库、执行断点、上传的文件、沙箱工作区,还有加密密钥 .secret_key 都在这里。 # 要备份的就是这个目录(丢了 .secret_key,存进去的 API Key 就解不开了)。 diff --git a/frontend/src/api/auth.ts b/frontend/src/api/auth.ts index 51990394..b20eb648 100644 --- a/frontend/src/api/auth.ts +++ b/frontend/src/api/auth.ts @@ -2,7 +2,7 @@ * 身份的接口(/api/auth、/api/users、/api/groups、/api/audit),挂在 api.auth / api.identity 上。形状见 typesApp.ts。 * 开登录时:没登录的请求回 401(前端跳 /login),角色不够回 403。关登录时 /me 恒为 mode='off'。 */ -import type { AuditRow, AuthMode, Directory, GroupRow, Principal, Role, UserRow } from '../typesApp' +import type { AuditRow, AuthMode, CandidateSource, Candidates, Directory, GroupRow, Principal, Role, UserRow } from '../typesApp' import { del, get, patch, post, put, qs, type RequestOptions } from './http' export const auth = { @@ -29,6 +29,9 @@ export const identity = { groups: () => get<{ items: GroupRow[] }>('/groups'), /** 选人用的名录:搭建者及以上只读(/users、/groups 只给管理员) */ directory: () => get('/directory'), + /** 使用者选人的候选:按用途给的池,只有名字(任何登录的人都能调;用不了那个应用的 403)。q 按开头筛 */ + candidates: (source: CandidateSource, q?: string, limit = 20) => + get(`/directory/candidates${qs({ ...source, q: q || undefined, limit })}`), createGroup: (body: { name: string; label?: string }) => post('/groups', body), setMembers: (id: string, members: string[]) => put(`/groups/${encodeURIComponent(id)}/members`, { members }), deleteGroup: (id: string) => del(`/groups/${encodeURIComponent(id)}`), diff --git a/frontend/src/api/capabilities.ts b/frontend/src/api/capabilities.ts index 3f2af1bd..08fde911 100644 --- a/frontend/src/api/capabilities.ts +++ b/frontend/src/api/capabilities.ts @@ -5,6 +5,7 @@ import type { CatalogReviewAction, CatalogSource, CatalogStatus } from '../types' import type { CapabilityEffect, CapabilityReviewOutcome, CapabilityReviewRow, CapabilityRow, EndpointRow, NotifyChannelRow, + NotifyEventRow, } from '../typesApp' import { del, get, patch, post, put, qs, type RequestOptions } from './http' @@ -128,4 +129,6 @@ export const notify = { removeChannel: (id: string) => del(`/notify/channels/${encodeURIComponent(id)}`), testChannel: (id: string) => post<{ ok: boolean; message: string }>(`/notify/channels/${encodeURIComponent(id)}/test`), deliveries: (params?: { channel_id?: string; limit?: number }) => get<{ items: any[] }>(`/notify/deliveries${qs(params)}`), + /** 登记的事件和推给哪几条路:哪种渠道能订阅哪些由服务端说,界面不自己猜 */ + events: () => get('/notify/events'), } diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index 832089de..079342e4 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -13,6 +13,7 @@ import type { RunEvent, RunStatus, Skill, ToolChange, ToolInfo, ToolTrust, UpgradeResult, UploadDecision, UploadMixedColumn, UploadResult, UploadShapeReason, ValidationIssue, VarIssue, Variable, Workflow, WorkflowVersion, } from '../types' +import type { UnlistedRuns } from '../typesApp' import { ApiError, BASE, NETWORK_MESSAGE, actorHeader, codeOf, del, describeDetail, failure, get, isGatewayFailure, parseBody, patch, post, put, qs, report, request, upload, type RequestOptions, type UploadOptions, @@ -563,6 +564,14 @@ export const api = { /** 按运行来源筛(X3);none 是来源未记录的老运行 */ origin_kind?: string }) => get(`/runs${qs(params)}`), + /** + * 同样的筛选下,列表因为「系统派生的运行默认不列」而没列出的次数(评测、口径对账、口径召回,next-round X1)。按来源 + * 筛了的什么都不藏,total 为 0。老后端没有这个接口(404),调用方当作 0 + */ + unlisted: (params?: { + workflow_id?: string; status?: RunStatus | RunStatus[] | string; run_class?: 'formal' | 'exploratory' + q?: string; origin_kind?: string + }) => get(`/runs/unlisted${qs(params)}`), get: (id: string) => get(`/runs/${id}`), /** 这次运行实际跑的那张图(运行时的快照),不是工作流现在的样子 */ graph: (id: string) => get<{ graph: GraphSpec; workflow_id: string | null; version: number | null }>( @@ -871,7 +880,8 @@ export const api = { setModel: (body: { provider?: string | null; model?: string | null }) => put('/copilot/model', body), /** 跑完之后复核一次。干净的运行后端直接返回 verdict='ok',不调模型 */ - review: (body: { run_id: string; question: string }) => + /** attempt:这一轮已经自动试了几次(从 0 数),服务端据此决定给不给 retry_instruction。老服务端不认它,忽略 */ + review: (body: { run_id: string; question: string; attempt?: number }) => post('/copilot/review', body), /** * 一键升级为可追溯结构:先按确定的规则改写(llm 换成报告撰写、问数据的图插入报告节点……), diff --git a/frontend/src/components/PrincipalPicker.tsx b/frontend/src/components/PrincipalPicker.tsx index 229061d9..c1b68c6e 100644 --- a/frontend/src/components/PrincipalPicker.tsx +++ b/frontend/src/components/PrincipalPicker.tsx @@ -5,15 +5,18 @@ * 名录里没有的、停用了的不让添加,说清是哪一个(账号密码登录时人和组都在本系统里建,名录里没有就是填错了);单点登录的 * 人和组第一次登录后才进名录,名录里没有的照实提醒,点「仍然添加」才加上。已选的里头名录里没有的、停用了的在标签上 * 标出来,不静默放着。 - * 使用者读不到名录:照填,服务端按名字认。关登录时没有用户表,只能手填名字——签批记的也是各自填的署名,这里不假装能核对。 + * 使用者读不到名录(只读与最小可见):给了 source 的(表单里的选人控件、个人应用提交审核)边输入边按开头取候选 + * (GET /api/directory/candidates,05-已知遗留 (b)),只有这件事能选的人、只有名字;加的时候按同一个池问一次服务端,不在池里的 + * 当场说清。提交时服务端照同一个池核对(不在池里 422),这里只是提前说。没给 source 的照填,服务端按名字认。 + * 关登录时没有用户表,只能手填名字——签批记的也是各自填的署名,这里不假装能核对。 */ -import { useEffect, useId, useState } from 'react' +import { useEffect, useId, useRef, useState } from 'react' import { Plus, Users, UserRound, X } from 'lucide-react' import clsx from 'clsx' import { api } from '../api/client' import { PRINCIPAL_TEXT } from '../lib/termsApp' import { useAtLeast, useAuth, useLoginOn } from '../store/auth' -import type { Directory } from '../typesApp' +import type { CandidateSource, Candidates, Directory } from '../typesApp' import { isComposing } from './ui' export interface Principal { @@ -29,8 +32,16 @@ export interface PrincipalPickerProps { /** 只能选一个 */ single?: boolean label?: string + /** + * 使用者取候选的池(服务端按用途算):表单里的选人控件(入口、待办)、个人应用提交审核。搭建者及以上照旧用整份名录, + * 不看它 + */ + source?: CandidateSource } +/** 边输入边取候选:停手这么久再去问,不是每敲一个字问一次 */ +const ASK_DELAY_MS = 250 + /** 名录只取一次,同一页上几个选择器共用;没取到(断线)下次打开再取 */ let directory: Promise | null = null function loadDirectory(): Promise { @@ -46,15 +57,21 @@ function problemOf(p: Principal, known: Directory | null): 'missing' | 'disabled return !user ? 'missing' : user.disabled ? 'disabled' : null } -export function PrincipalPicker({ value, onChange, allowEveryone = false, single = false, label }: PrincipalPickerProps) { +export function PrincipalPicker({ value, onChange, allowEveryone = false, single = false, label, source }: PrincipalPickerProps) { const uid = useId().replace(/[^a-zA-Z0-9_-]/g, '') const loginOn = useLoginOn() // 名录给搭建者及以上(使用者读不到,不去取:取了也是 403) const listable = useAtLeast('builder') && loginOn + // 使用者:有 source 就按池取候选 + const asking = !listable && loginOn && !!source + const sourceKey = source ? JSON.stringify(source) : '' const oidc = useAuth((s) => s.me?.mode === 'oidc') const [kind, setKind] = useState<'user' | 'group'>('user') const [text, setText] = useState('') const [known, setKnown] = useState(null) + const [pool, setPool] = useState(null) + /** 取到过的候选:用户名 → 显示名(标签上写显示名,悬停给用户名) */ + const shown = useRef(new Map()) /** 刚才要加、名录里对不上的那一项:soft 是单点登录下名录里还没有(可以「仍然添加」),其余不让加 */ const [refused, setRefused] = useState<{ item: Principal; message: string; soft: boolean } | null>(null) useEffect(() => { @@ -63,12 +80,38 @@ export function PrincipalPicker({ value, onChange, allowEveryone = false, single void loadDirectory().then((d) => { if (alive) setKnown(d) }) return () => { alive = false } }, [listable]) + useEffect(() => { + if (!asking || !source) return + let alive = true + const timer = window.setTimeout(() => { + api.identity.candidates(source, text.trim()) + .then((c) => { + if (!alive) return + for (const u of c.users) shown.current.set(u.username, u.display_name) + setPool(c) + }) + .catch(() => { if (alive) setPool(null) }) + }, ASK_DELAY_MS) + return () => { alive = false; window.clearTimeout(timer) } + // sourceKey 代替 source:调用方每次渲染都给一个新对象 + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [asking, sourceKey, text]) const has = (p: Principal) => value.some((x) => x.kind === p.kind && x.name === p.name) - const add = (p: Principal, anyway = false) => { + const add = async (p: Principal, anyway = false) => { const item = { ...p, name: p.name.trim() } if (!item.name) return if (has(item)) { setText(''); setRefused(null); return } + if (asking && source && item.kind !== 'everyone') { + // 按同一个池问一次服务端:不在池里的当场说清(提交时服务端照样拒)。问不到(断线)就先收下,提交时由服务端判 + const found = await api.identity.candidates(source, item.name, 50).catch(() => null) + const listed = found && (item.kind === 'group' ? found.groups.some((g) => g.name === item.name) + : found.users.some((u) => u.username === item.name)) + if (found && !listed) { + setRefused({ item, soft: false, message: PRINCIPAL_TEXT.notCandidate(item.name) }) + return + } + } const problem = problemOf(item, known) const soft = problem === 'missing' && oidc if (problem && !(soft && anyway)) { @@ -87,9 +130,16 @@ export function PrincipalPicker({ value, onChange, allowEveryone = false, single const remove = (p: Principal) => onChange(value.filter((x) => !(x.kind === p.kind && x.name === p.name))) const listId = `${uid}-list` const errorId = `${uid}-error` - const options = !known ? [] : kind === 'user' - ? known.users.filter((u) => !u.disabled).map((u) => ({ value: u.username, label: u.display_name })) - : known.groups.map((g) => ({ value: g.name, label: g.label })) + const options = asking + ? (!pool ? [] : kind === 'user' ? pool.users.map((u) => ({ value: u.username, label: u.display_name })) + : pool.groups.map((g) => ({ value: g.name, label: g.label }))) + : !known ? [] : kind === 'user' + ? known.users.filter((u) => !u.disabled).map((u) => ({ value: u.username, label: u.display_name })) + : known.groups.map((g) => ({ value: g.name, label: g.label })) + /** 候选池要先输入开头才列(应用授权给了所有人)、列不全(超过上限) */ + const poolHint = asking && pool + ? (pool.needs_query && !text.trim() ? PRINCIPAL_TEXT.typeToSearch : pool.truncated ? PRINCIPAL_TEXT.truncated : null) + : null return (
@@ -102,7 +152,9 @@ export function PrincipalPicker({ value, onChange, allowEveryone = false, single data-principal-problem={problem ?? undefined} style={problem ? { color: 'var(--st-waiting)', borderColor: 'var(--st-waiting)' } : undefined}> {p.kind === 'group' ? : p.kind === 'user' ? : null} - {p.kind === 'everyone' ? PRINCIPAL_TEXT.everyone : p.name} + + {p.kind === 'everyone' ? PRINCIPAL_TEXT.everyone : shown.current.get(p.name) ?? p.name} + ({PRINCIPAL_TEXT[p.kind]}) {problem && · {problem === 'disabled' ? PRINCIPAL_TEXT.disabledTag : PRINCIPAL_TEXT.missingTag}} {allowEveryone && ( )} @@ -151,12 +203,13 @@ export function PrincipalPicker({ value, onChange, allowEveryone = false, single style={{ color: refused.soft ? 'var(--st-waiting)' : 'var(--err)' }} data-principal-refused={refused.soft ? 'soft' : 'hard'}> {refused.message} {refused.soft && ( - )}
)} + {poolHint &&
{poolHint}
} {!loginOn &&
{PRINCIPAL_TEXT.manualHint}
} ) diff --git a/frontend/src/components/SchemaForm.tsx b/frontend/src/components/SchemaForm.tsx index ffa52aaa..b2b0780f 100644 --- a/frontend/src/components/SchemaForm.tsx +++ b/frontend/src/components/SchemaForm.tsx @@ -12,7 +12,7 @@ import { useEffect, useId, useState } from 'react' import type { CSSProperties, KeyboardEvent as ReactKeyboardEvent, ReactNode } from 'react' import clsx from 'clsx' -import type { FormError, FormField, FormSchema } from '../typesApp' +import type { CandidateSource, FormError, FormField, FormSchema } from '../typesApp' import { FORM_TEXT } from '../lib/termsApp' import { fieldOfPath, fieldOrder, fileLimitsOf, localToRfc3339, optionsOf, rfc3339ToLocal, widgetOf, @@ -88,6 +88,16 @@ export function SchemaForm({ schema, value, onChange, errors, readOnly, onSubmit ) } +/** + * 选人控件的候选从哪个池取:表单用在入口(app_id + entry)、待办(task_id)上时,按这个字段问服务端(05-已知遗留 (b)); + * 蓝图试运行之类没有应用的地方不给(搭建者用整份名录) + */ +function principalSource(context: Record | undefined, field: string): CandidateSource | undefined { + if (context?.task_id) return { purpose: 'form', task_id: context.task_id, field } + if (context?.app_id && context.entry) return { purpose: 'form', app_id: context.app_id, entry: context.entry, field } + return undefined +} + function FieldRow({ id, name, field, required, value, error, readOnly, onChange, fileContext }: { id: string; name: string; field: FormField; required: boolean; value: unknown; error?: FormError readOnly: boolean; onChange: (v: unknown) => void; fileContext?: Record @@ -143,7 +153,7 @@ function FieldRow({ id, name, field, required, value, error, readOnly, onChange,
{label}
{readOnly ?
{typeof value === 'string' && value ? value : '—'}
- : onChange(list[0]?.name || undefined)} />} {foot} diff --git a/frontend/src/lib/explain.ts b/frontend/src/lib/explain.ts index 4e6e06e5..d9c992fe 100644 --- a/frontend/src/lib/explain.ts +++ b/frontend/src/lib/explain.ts @@ -295,11 +295,15 @@ export function explainRunError(error: string | null | undefined, detail?: strin raw, } } - // 后端有一条路径以前写「人工拒绝:」(现在统一为「人工驳回:」),历史运行里两种都有 - if (/人工(?:驳回|拒绝)/.test(text)) { + // 只留给存量:加 end_reason 之前被驳回的运行记的是 failed,报错原文是这几种说法。新的驳回结束在 cancelled + + // end_reason=rejected,界面按那个字段说(lib/status、REJECTED_TEXT),不走这里(05-已知遗留 (a))。后端有一条路径 + // 以前写「人工拒绝:」(现在统一为「人工驳回:」);动作确认、工具调用审批、代码执行审批的驳回各有一种原话。 + // 接着跑只会读到同一个驳回答复、再停一次,所以都不给「继续运行」 + if (/人工(?:驳回|拒绝)|未确认(被驳回)|的调用已被驳回|代码执行已被驳回/.test(text)) { + const human = /^人工(?:驳回|拒绝)/.test(plain) return { title: '人工审批驳回,运行终止', - reason: plain.replace(/^人工(?:驳回|拒绝)[::]?\s*/, '') || undefined, + reason: (human ? plain.replace(/^人工(?:驳回|拒绝)[::]?\s*/, '') : plain) || undefined, action: '这是审批人的决定,不是故障。如有需要,请修改内容后重新发起运行。', continuable: false, raw, diff --git a/frontend/src/lib/publicUrl.ts b/frontend/src/lib/publicUrl.ts new file mode 100644 index 00000000..781500a9 --- /dev/null +++ b/frontend/src/lib/publicUrl.ts @@ -0,0 +1,31 @@ +/** + * 对外访问地址:别人打开 AgentLab 用的那个地址。服务端全站只认一个(backend/app/core/public_url.py):环境变量 + * AGENTLAB_PUBLIC_URL 优先,其次设置里 notify 一组的 public_base_url(界面上在「设置 → 通用」改,定时任务的 Webhook 一节 + * 有快捷入口)。通知和 Webhook 里的链接、单点登录的回调、来源核对都按它。 + * + * GET /api/settings 另给只读的 public_url 一组 {value, env_locked}:环境变量指定了的话显示它、写明「由环境变量指定」、 + * 不让改(改了服务端回 409)。老服务端没有这一组时按设置里存的那一项、可以改。 + */ + +export interface PublicAddress { + /** 生效的地址;null 是没设置(通知里不附链接) */ + value: string | null + /** 环境变量指定的:设置里的那一项不生效、不能改 */ + env: boolean +} + +export function publicAddressOf(settings: Record | null | undefined): PublicAddress { + const env = settings?.public_url?.env_locked === true + const raw = env ? settings?.public_url?.value : settings?.notify?.public_base_url + return { value: typeof raw === 'string' && raw ? raw : null, env } +} + +/** 填的地址去掉首尾空白和末尾的 /(服务端存的也是这样) */ +export function cleanPublicUrl(text: string): string { + return text.trim().replace(/\/+$/, '') +} + +/** http:// 或 https:// 开头、有主机名、中间没有空白、不带 ? 或 # 之后的部分(同服务端 public_url.url_problem) */ +export function isPublicUrl(text: string): boolean { + return /^https?:\/\/[^\s/?#]+(?:\/[^\s?#]*)?$/i.test(text.trim()) +} diff --git a/frontend/src/lib/status.ts b/frontend/src/lib/status.ts index c817fbf3..b91c04fa 100644 --- a/frontend/src/lib/status.ts +++ b/frontend/src/lib/status.ts @@ -22,6 +22,7 @@ export type StatusCode = | 'succeeded' // 运行跑完 | 'failed' | 'cancelled' + | 'rejected' // 运行:被人驳回结束(后端是 cancelled + end_reason=rejected,05-已知遗留 (a)) | 'suspended' // 服务重启后挂起,可续跑 | 'held' // 后端的 interrupted 但没有待审批:已挂起 · 可续跑 | 'skipped' // 被 skip_if 跳过 @@ -104,6 +105,11 @@ export const STATUS: Record = { code: 'cancelled', label: '已取消', short: '已取消', shape: 'stop', alert: false, terminal: true, ...tok('cancelled', 'var(--text-faint)'), }, + // 人的决定,不是故障:不画红、不给继续运行;和手动停止同一个剪影,叫法分开 + rejected: { + code: 'rejected', label: '被驳回', short: '被驳回', shape: 'stop', alert: false, terminal: true, + hint: '审批人驳回了这次运行,运行到这里结束;改好内容后重新发起', ...tok('cancelled', 'var(--text-faint)'), + }, suspended: { code: 'suspended', label: '已中断(服务重启)', short: '已中断', shape: 'pause', alert: true, terminal: true, hint: '服务重启时本次运行尚未结束,已在断点处挂起,可继续运行', ...tok('suspended', 'var(--warn)'), @@ -141,15 +147,25 @@ const ALIAS: Record = { * 才分得开——传 pendingApproval。不知道时(没传)按等待审批处理,这是 * interrupted 最常见的来源,也是改造前的说法,不会比原来更错。 */ -export function resolveStatus(status: string | null | undefined, opts?: { pendingApproval?: boolean }): StatusCode { +export function resolveStatus(status: string | null | undefined, opts?: StatusOpts): StatusCode { if (!status) return 'idle' if (status === 'interrupted') return opts?.pendingApproval === false ? 'held' : 'waiting' + // 被驳回只看服务端给的 end_reason,不解析报错原文(界面层不放逻辑) + if (status === 'cancelled' && opts?.endReason === 'rejected') return 'rejected' if (status in STATUS) return status as StatusCode return ALIAS[status] ?? 'idle' } +/** + * 落状态要的旁证:pendingApproval 分 interrupted 的两种;endReason 是运行的 end_reason(cancelled 时分被驳回和手动停止) + */ +export interface StatusOpts { + pendingApproval?: boolean + endReason?: string | null +} + /** 状态的完整外观。未知状态码回落到 idle 的外观,文字写原码,免得静默吞掉 */ -export function statusMeta(status: string | null | undefined, opts?: { pendingApproval?: boolean }): StatusMeta { +export function statusMeta(status: string | null | undefined, opts?: StatusOpts): StatusMeta { const code = resolveStatus(status, opts) const meta = STATUS[code] if (status && code === 'idle' && status !== 'idle' && !(status in ALIAS)) { @@ -159,7 +175,7 @@ export function statusMeta(status: string | null | undefined, opts?: { pendingAp } /** 只要文字 */ -export function statusLabel(status: string | null | undefined, opts?: { pendingApproval?: boolean; short?: boolean }): string { +export function statusLabel(status: string | null | undefined, opts?: StatusOpts & { short?: boolean }): string { const meta = statusMeta(status, opts) return opts?.short ? meta.short : meta.label } @@ -167,10 +183,11 @@ export function statusLabel(status: string | null | undefined, opts?: { pendingA /** * 运行列表的状态筛选用:按显示顺序排好的运行状态。 * 这是显示用的码,不能直接当查询参数——waiting / held 在后端都是 interrupted, - * 查询前用 serverStatusOf 换回去。 + * rejected 在后端是 cancelled(+ end_reason=rejected),查询前用 serverStatusOf 换回去。 + * 被驳回单独一项、紧跟在已取消后面:不在这里的码,分段、计数、搜索词表里都没有它,被驳回的运行只剩「全部」里看得到 */ export const RUN_STATUS_ORDER: StatusCode[] = [ - 'running', 'waiting', 'held', 'failed', 'succeeded', 'cancelled', 'queued', + 'running', 'waiting', 'held', 'failed', 'succeeded', 'cancelled', 'rejected', 'queued', ] /** @@ -178,7 +195,7 @@ export const RUN_STATUS_ORDER: StatusCode[] = [ * * waiting、held、suspended 在后端都是 interrupted:按它查回来的是两种混在一起, * 要再用 resolveStatus(run.status, {pendingApproval: hasPendingApproval(approvals, - * run.id)}) 在前端分开。节点才有的状态(idle / skipped / blocked / unreached) + * run.id)}) 在前端分开。rejected 同理查 cancelled,回来按 end_reason 分开(被驳回 / 手动停止)。节点才有的状态(idle / skipped / blocked / unreached) * 不是运行状态,返回 null。 */ export function serverStatusOf(code: StatusCode): RunStatus | null { @@ -186,6 +203,7 @@ export function serverStatusOf(code: StatusCode): RunStatus | null { case 'waiting': case 'held': case 'suspended': return 'interrupted' case 'done': case 'succeeded': return 'succeeded' case 'queued': case 'running': case 'failed': case 'cancelled': return code + case 'rejected': return 'cancelled' default: return null } } diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index 6e4a5e4d..d90db81e 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -759,6 +759,21 @@ export function runCostText(usd: number | null | undefined, unpriced: number, ap return `${(approx || unpriced > 0) && usd > 0 ? '≥ ' : ''}${formatCost(usd)}` } +/** + * 问数据一轮的花费:助手建图(含挑表、接着写、自查修正)+ 运行 + 复核(next-round 细案 01 做法 3)。以前只算运行那份, + * 这一轮的花费是低报的 + */ +export const TURN_COST_TEXT = { + copilot: '助手建图', + run: '运行', + review: '复核', + /** 悬停说明的第一行,parts 是「助手建图 $0.012 · 运行 $0.03 · 复核 $0」 */ + title: (parts: string) => `这一轮的花费:${parts}`, + /** n 次调用的金额估不出来:金额只是下限 */ + unpriced: (n: number) => + `其中 ${formatNumber(n)} 次调用的金额无法估算(模型不在价格目录中,或用量没有报全),未计入,实际花费不低于此数`, +} as const + export const JUDGE_SETTING_TEXT = { label: '证据裁判', hint: '报告中的结论句(如「增长主要来自新客首单」)由另一个模型按证据逐句判断:正式运行由报告撰写节点当场裁判,探索运行在点击时逐句裁判。判断由模型给出,不是系统核对,界面上一律标注「非确定」。', @@ -1231,8 +1246,6 @@ export const UPLOAD_TEXT = { shapeTitle: '该表格不是一行一条记录的明细表', shapeLead: '检测到以下结构,默认不导入:', shapeRecipe: '这类表格可以按配方导入:指定表头、分段和合计行的处理方式,试运行核对、逐条确认后再启用。', - /** CSV 没有配方这条路(配方只认 Excel 的工作表结构) */ - shapeRecipeCsv: '按配方导入目前只支持 Excel(.xlsx)文件:可另存为 .xlsx 后再按配方导入。', shapeHeader: (row: number) => `如果表头不在第 ${row} 行,请取消后修改表头行号,再重新上传。`, /** 结构问题之外,预告的混合列:选按原样导入以后还要再选一次 */ shapeMixedLead: '另外,以下各列以数字为主,但混有非数字的值。选择按原样导入后,还需要选择这些值的处理方式:', @@ -1500,9 +1513,9 @@ export const RECIPE_TEXT = { descriptionLabel: '说明(供助手参考)', pickDrop: '拖入文件,或点击选择', /** 与选择框的 accept 一致 */ - pickFormats: '支持 Excel(.xlsx、.xlsm)', + pickFormats: '支持 Excel(.xlsx、.xlsm)和 CSV / TSV', pickAria: '选择表格文件', - pickWrongType: (name: string) => `「${name}」不是 Excel 文件:按配方导入只支持 .xlsx、.xlsm`, + pickWrongType: (name: string) => `「${name}」不是 Excel 或 CSV 文件:按配方导入支持 .xlsx、.xlsm、.csv、.tsv`, cancelUpload: '取消上传', cancel: '取消', close: '关闭', @@ -1637,6 +1650,20 @@ export const RECIPE_TEXT = { totalLabelColumn: '合计标签所在的列', totalPick: '合计行以此开头', totalKeep: '原表合计另存为表(留空则只核对)', + // 期 5:穿插的小计行(ListRows.subtotals)、列表每期怎么更新(ListBlock.period) + subtotals: '小计行', + subtotalsOn: '有小计行', + subtotalsHint: '夹在明细中间的小计不进数据,按它所管的那组明细逐列求和核对', + subtotalLabelColumn: '小计标签所在的列', + subtotalPick: '小计行以此开头', + subtotalScope: '一行小计管哪些明细', + subtotalGroupColumn: '按哪一列分组', + subtotalKeep: '原表小计另存为表(留空则只核对)', + /** 人确认过列表里没有小计(ListRows.no_subtotals):像小计的行照常是数据,不再每期报「像小计的行」要人确认 */ + noSubtotals: '没有小计行(以合计字样开头的行也是数据,不再提醒)', + listPeriod: '按期累积时每一行属于哪一期', + listPeriodColumn: '按哪个日期列核对', + listPeriodHint: '列表要按期累积,每一行都要说得出属于哪一期;所在工作表要配置统计期(表内没有时,每期试运行时录入)。导入模式另在「导入模式」中选', hiddenRows: '隐藏行', hiddenCols: '隐藏列', fallback: '工作表改名时', @@ -1849,6 +1876,7 @@ export const FIX_KIND_LABEL: Record = { edit_members: '更新关系成员', rename_title: '改分段标题', declare_total: '改作合计核对', + declare_subtotals: '改作小计核对', ignore_cells: '按文字忽略', declare_placeholder: '声明占位符', declare_hidden: '设置隐藏行处理', @@ -1994,6 +2022,7 @@ export const RECEIPT_BLOCK_TEXT = { ignored_outside: '按配方忽略了区域外数字的行', after_stop: '列表结束之后的文字行', total_not_kept: '只核对、不另存的合计行', + subtotal: '穿插的小计行(核对后不进数据)', } as Record, excludedOther: '其他原因', excludedTitle: '排除的行', @@ -2107,6 +2136,8 @@ export const RECIPE_CHOICE_LABEL = { colType: { TEXT: '文字', INTEGER: '整数', REAL: '小数', DATE: '日期' } as Record, colStore: { '': '默认', canonical: '规范写法', raw: '原文' } as Record, axisChecks: { contiguous: '逐日连续', covers_context: '恰好覆盖统计期' } as Record, + subtotalScope: { since_previous: '上一行小计之后的全部明细', group_column: '紧挨在上面、分组列取值相同的明细' } as Record, + listPeriod: { none: '不区分(只能每期替换)', column: '按日期列核对每行都在本期统计期内', synthesize: '每行加上本期统计期的起止两列' } as Record, } // =========================================================================== @@ -2771,7 +2802,7 @@ export const SCHEMA_PARTIAL_TEXT = { // - 剖析是后台任务:逐张表进行,每张表剖析完就写进目录;查询一条一条地发,可以随时停止。关掉对话框照常进行。 // 目录只在整张表结束时写一次:停止、关停服务时正在做的那张来得及收尾,做完的项照常写进去;进程被杀时那张表没来得及收尾, // 这次做完的项一项都没写进去。说停下以后写进了什么,按逐表的状态照实说(cancelledNote / interruptedNote),不一律说「已写进」。 -// - 预算按表算:「每张表最多查询数」「每张表时长上限」用完只停这张表,下一张表重新计数;另有单条查询时限、抽样键值数、 +// - 预算按表算:「每张表最多查询数」「每张表时长上限」用完只停这张表,下一张表重新计数;另有单条查询时限、核对键值数、 // 整表统计行数上限。标签和服务端校验报错里的叫法一致(catalog_profile._NUMBER_FIELDS),保存被拒时按机读码认出是哪一项。 // - 没有整次总量上限:任务自己在连续失败、多数超时时停下(止损),写明原因。 // - 「没查成」≠ 结论错了:因为次数、时长、超时、出错没查成的项保留剖析以前的结论,可以「重试没做完的表」。 @@ -2783,7 +2814,7 @@ export const PROFILE_FIELD_LABEL: Record<'enabled' | CatalogProfileNumberKey, st table_max_queries: '每张表最多查询数', table_max_s: '每张表时长上限', query_timeout_s: '单条查询时限', - sample_size: '抽样键值数', + sample_size: '核对键值数', max_scan_rows: '整表统计行数上限', } @@ -2799,7 +2830,7 @@ export const PROFILE_FIELD_HINT: Record = { table_max_queries: '剖析一张表最多发出的查询条数,用完只停这张表,下一张表重新计数', table_max_s: '剖析一张表最多用的时长,到时只停这张表,下一张表重新计时', query_timeout_s: '超过即中断这条查询;不超过数据源自身的查询时限,也不超过这张表剩下的时长', - sample_size: '核对一条关联关系时,从本表抽取的不同键值个数', + sample_size: '核对一条关联关系时,从本表取的不同键值个数:取的是数据库先给出的那些,不是随机抽样', max_scan_rows: '不超过此行数的表才做去重计数、最小值和最大值等整表统计', } @@ -2863,7 +2894,7 @@ export const PROFILE_TEXT = { budgetPerTable: (queries: number, secs: number) => `每张表最多 ${formatNumber(queries)} 条查询、${formatNumber(secs)} 秒,用完只停这张表,下一张表重新计数`, budgetQuery: (secs: number) => `单条查询不超过 ${formatNumber(secs)} 秒`, - budgetSample: (n: number) => `每条关联关系抽样 ${formatNumber(n)} 个键值`, + budgetSample: (n: number) => `每条关联关系取 ${formatNumber(n)} 个不同键值核对`, budgetScan: (n: number) => (n > 0 ? `行数超过 ${formatNumber(n)} 的表不做整表统计` : '不做整表统计'), /** 上限的估算:表数 × 每张表的上限 */ budgetTotal: (tables: number, queries: number, span: string) => @@ -3002,7 +3033,7 @@ export const PROFILE_TEXT = { relationLowCoverage: '覆盖率不足,可能不是这条关系', relationNotUnique: '被指向列未核实唯一', coverage: (pct: string) => `覆盖率 ${pct}`, - sampled: (sample: number, matched: number) => `抽样 ${formatNumber(sample)} 个键值,对上 ${formatNumber(matched)} 个`, + sampled: (sample: number, matched: number) => `取了 ${formatNumber(sample)} 个不同键值,对上 ${formatNumber(matched)} 个`, cardinalityUnknown: '基数未核实', codeRows: (n: number) => `${formatNumber(n)} 行`, codesSummary: (values: number, rows: number) => `${formatNumber(values)} 个取值,共 ${formatNumber(rows)} 行`, @@ -3012,10 +3043,12 @@ export const PROFILE_TEXT = { dateProposal: (col: string) => `提议按 ${col} 作为业务日期`, dateSpan: (min: string, max: string) => `取值从 ${min} 到 ${max}`, dateRanges: '日期列的取值范围', + /** 日期列的最小值、最大值不计入的值(catalog_profile.VALID_DATE_FROM):零日期、0001-01-01、9999-12-31、无穷大 */ + dateInvalid: (n: number) => `另有 ${formatNumber(n)} 个无效日期未计入(零日期或占位日期)`, skippedTitle: (n: number) => `跳过 ${formatNumber(n)} 项`, moreSkipped: (n: number) => `…另有 ${formatNumber(n)} 项`, queriesTitle: (n: number) => `看发过的查询(${formatNumber(n)} 条)`, - queriesHint: 'SQL 里的取值都换成了「?」:抽样到的键值不保存在 AgentLab 中;遮罩的列不出现在任何查询里', + queriesHint: 'SQL 里的取值都换成了「?」:取到的键值不保存在 AgentLab 中;遮罩的列不出现在任何查询里', queryKind: { row_estimate: '行数估算', relation: '关系', codes: '码值', date: '日期' } as Record, queryOutcome: { ok: '成功', timeout: '超时', error: '失败', rejected: '未通过只读守卫' } as Record, queryRows: (n: number) => `${formatNumber(n)} 行`, @@ -3889,19 +3922,20 @@ export const SCHEDULE_TEXT = { testResult: (head: string, status: number | null, detail: string) => `${head}${status != null ? `(HTTP ${status})` : ''}${detail ? `:${detail}` : ''}`, - // ---- 通知里的链接:运行、对比页都按这个地址拼(设置 notify 组,所有定时任务共用) + // ---- 通知里的链接:运行、对比页都按全站的对外访问地址拼(GENERAL_SETTING_TEXT,这里是它的快捷入口) linkBase: '通知中的链接', linkBaseValue: (url: string) => `按 ${url} 拼接`, - linkBaseUnset: '未设置服务访问地址,通知中不附带运行和对比页链接', + linkBaseUnset: '未设置对外访问地址,通知中不附带运行和对比页链接', linkBaseEdit: '修改', linkBaseSet: '设置', - linkBaseLabel: '服务访问地址', - linkBaseHint: '收件人打开运行、对比页链接用的地址,例如 https://agentlab.example.com。所有定时任务共用', + linkBaseLabel: '对外访问地址', + linkBaseHint: '收件人打开运行、对比页链接用的地址,例如 https://agentlab.example.com。全站共用一个,也可以在「设置 → 通用」中修改', linkBasePlaceholder: 'https://agentlab.example.com', - linkBaseInvalid: '请输入以 http:// 或 https:// 开头的地址', - linkBaseSaved: '已保存服务访问地址', - linkBaseCleared: '已清除服务访问地址', - linkBaseFailed: '无法读取服务访问地址', + linkBaseInvalid: '请输入以 http:// 或 https:// 开头的完整地址,不带 ? 或 # 之后的部分', + linkBaseSaved: '已保存对外访问地址', + linkBaseCleared: '已清除对外访问地址', + linkBaseFailed: '无法读取对外访问地址', + linkBaseEnv: '由环境变量 AGENTLAB_PUBLIC_URL 指定,不能在这里修改', // ---- 表单:保存 save: '保存', @@ -5114,13 +5148,18 @@ export const ORIGIN_LABEL: Record = { api: '接口', trial: '蓝图试运行', personal_app: '个人应用刷新', + eval: '评测', + ledger_issue: '按口径正式出具', + reconcile: '口径对账', + recall: '口径召回', } export const ORIGIN_TEXT = { unknown: '来源未记录', unknownHint: '这次运行发起时还没有记录来源', hint: '这次运行从哪里发起。发起时写进运行记录,在封存范围内', - filterAll: '全部来源', + // 不点名来源时系统派生的运行(评测、口径对账、口径召回)不列,「全部」得照实说不含它们 + filterAll: '全部来源(不含系统运行)', filterLabel: '按运行来源筛选', /** 时间线里 run.origin 那一行 */ step: (label: string) => `来源:${label}`, @@ -5131,6 +5170,37 @@ export const ORIGIN_TEXT = { actor: (who: string, verified: boolean) => `${verified ? '已登录用户' : '署名'}「${who}」`, } +/** + * 记录页没列出的系统派生运行(评测、口径对账、口径召回:服务端 engine/origin.SYSTEM_ORIGINS,next-round X1)。它们是完整的 + * 运行、不进使用统计,默认不列;照实写有几次没列出、在哪儿能看 + */ +export const SYSTEM_RUNS_TEXT = { + /** parts 是「评测 2 次、口径对账 1 次」 */ + unlisted: (total: number, parts: string) => + `另有 ${total} 次系统运行未列出(${parts}),在来源筛选中选择对应来源即可查看`, + part: (label: string, n: number) => `${label} ${n} 次`, + hint: '评测、口径对账、口径召回是系统代为发起的运行:证据仍可查看,但不计入使用统计,记录页默认不列', +} + +/** + * 驳回 ≠ 失败(docs/design/next-round/05-已知遗留.md (a)):人的驳回让运行结束在 cancelled + end_reason=rejected, + * run.cancelled 带着在哪一步(label)、谁(actor)、结论(decision)、备注(note)。时间线、问数据的终态、记录页都照这一份说, + * 不再按报错原文猜 + */ +export const REJECTED_TEXT = { + title: '被驳回', + /** 副标题:谁在哪一步驳回、备注;超时驳回写按设置驳回(备注是系统写的那句,不重复) */ + sub: (d: { actor?: unknown; label?: unknown; note?: unknown; decision?: unknown }): string => { + const who = typeof d.actor === 'string' ? d.actor.trim() : '' + const where = typeof d.label === 'string' && d.label.trim() ? `「${d.label.trim()}」` : '这一步' + const note = typeof d.note === 'string' ? d.note.trim() : '' + if (d.decision === 'expired_reject') return `${where}超过时限,按设置驳回` + return `${who ? `${who} 驳回了${where}` : `${where}被驳回`}${note ? `:${note}` : ''}` + }, + /** 问数据:这一轮的运行被驳回结束时,补救区的那一句 */ + chatReason: '这一轮的运行被审批人驳回,已经结束。这是人的决定,不是故障;改好内容后可以重新运行。', +} + /** 来源标签:老运行没有来源时写「来源未记录」 */ export const originLabel = (kind: string | null | undefined): string => kind && kind in ORIGIN_LABEL ? ORIGIN_LABEL[kind as OriginKind] : ORIGIN_TEXT.unknown @@ -5364,13 +5434,26 @@ export const NOTIFY_TEXT = { scope: '范围', scopes: { global: '全局', app: '应用', user: '个人' } as Record, events: '订阅的事件', + /** + * 通知事件的叫法:和服务端 notify/service.EVENTS 一一对应、逐字一致(test_notify_events 对照),事件只在那里登记(总方案 + * X6)。哪种渠道能订阅哪些、推没推,由服务端说(GET /api/notify/events 的 scopes):本轮登记了、还没有推送的几个这里 + * 先有叫法,界面上不列,实现推送的那一块不用再改这张表 + */ eventLabel: { - 'task.opened': '待办打开', 'task.escalated': '待办升级', 'task.expired': '待办超时', 'task.resolved': '待办处理完', - 'action.unknown': '动作结果未知', 'action.failed': '动作失败', 'action.confirmed': '动作已生效', - 'run.failed': '运行失败', 'run.finished': '运行结束', + 'task.opened': '待办打开', 'run.finished': '运行结束', 'run.failed': '运行失败', 'action.unknown': '动作结果未知', // 个人应用提交审核有了结果(发布了或退回了),通知提交人:只对个人渠道、全局渠道有意义(服务端 notify/service.notify_review) 'review.done': '我提交审核的应用有结果', + 'source.changed': '数据源有变化要处理', 'ledger.recall': '口径升版,答案的数变了', + 'eval.regressed': '评测有退步', 'eval.failed': '评测失败', + 'task.digest': '每天的待办汇总', 'key.expiring': '访问密钥快到期', } as Record, + /** 投递记录里才有的事件:改派、升级、催办按「待办打开」的订阅推,记录里照实写是哪一种(服务端 notify/service._AS) */ + deliveryEventLabel: { + 'task.reassigned': '待办改派', 'task.escalated': '待办升级', 'task.reminded': '待办催办', test: '测试消息', + } as Record, + /** 渠道编辑框、应用的通知:可订阅的事件还没取到 / 没取到 */ + eventsLoading: '正在读取可订阅的事件…', + eventsError: '没能读取可订阅的事件,请刷新页面后重试', enabled: '启用', signed: '已签名', test: '发送测试消息', @@ -5821,6 +5904,31 @@ export const APP_SETTING_TEXT = { personalAppsHint: '开启后,使用者可以从说需求、场景模板或问数据的回答建一个只有自己看得见的应用:只读、结果遮罩,每人每天的刷新次数按上面的每日上限算;想给别人用要提交给搭建者审核发布。搭建者及以上不受这一项限制', } +/** + * 设置 → 通用:全站共用的部署设置。对外访问地址全站只有一个(服务端 core/public_url.py):环境变量 AGENTLAB_PUBLIC_URL + * 优先,其次这里填的;定时任务 Webhook 一节里是它的快捷入口(SCHEDULE_TEXT.linkBase*) + */ +export const GENERAL_SETTING_TEXT = { + tab: '通用', + publicUrl: '对外访问地址', + publicUrlHint: '别人打开 AgentLab 用的地址。通知和定时任务 Webhook 里的链接、单点登录的回调地址都按它拼,提交操作时也按它核对请求来源。留空时通知里不附链接,回调和来源核对按浏览器访问的地址认', + address: '地址', + addressHint: '以 http:// 或 https:// 开头的完整地址,例如 https://agentlab.example.com;放在反向代理的子路径下时带上路径', + placeholder: 'https://agentlab.example.com', + invalid: '请输入以 http:// 或 https:// 开头的完整地址,不带 ? 或 # 之后的部分', + unset: '未设置', + envLocked: '由环境变量 AGENTLAB_PUBLIC_URL 指定', + envLockedHint: '部署时用环境变量定了这个地址,这里不能修改。要改请修改部署配置后重启服务', + readOnly: '全站共用,只有管理员可以修改', + save: '保存', + saved: '已保存对外访问地址', + cleared: '已清除对外访问地址', + leaveTitle: '对外访问地址还没保存', + leaveBody: '离开此页面将丢失这次修改。', + leaveConfirm: '放弃修改', + leaveCancel: '返回保存', +} + /** 自定义工具绑定登记的端点(§3.9):只填路径,地址和认证头由端点提供 */ export const TOOL_ENDPOINT_TEXT = { bind: '端点', diff --git a/frontend/src/lib/termsApp.ts b/frontend/src/lib/termsApp.ts index 13c5b33e..3c24e994 100644 --- a/frontend/src/lib/termsApp.ts +++ b/frontend/src/lib/termsApp.ts @@ -368,6 +368,10 @@ export const PRINCIPAL_TEXT = { /** 已选的那一项不在名录里、或已停用:标在标签上 */ missingTag: '名录里没有', disabledTag: '已停用', + /** 使用者选人:候选按这件事能选的人给(服务端 auth/directory 的候选池,05-已知遗留 (b)) */ + notCandidate: (name: string) => `「${name}」不在可选的人里,请从候选中选择`, + typeToSearch: '能选的人较多:输入名字的开头查找', + truncated: '只列出了一部分:多输入几个字缩小范围', } /** 身份:登录、账户菜单、角色(§4.4 AUTH_TEXT) */ @@ -445,6 +449,15 @@ export const SIGNER_TEXT = { record: '记入留痕', } +/** + * 个人渠道(设置 → 通知,范围是「仅自己」)里实例结束的两个事件:只推自己发起、而且是登录身份发起的实例(服务端 + * notify/service.notify_run,04-对外集成与触达 B1)。以前叫「实例完成 / 失败」,勾了却永远收不到 + */ +export const NOTIFY_PERSONAL_EVENT_LABEL: Record = { + 'run.finished': '我发起的实例完成', + 'run.failed': '我发起的实例失败', +} + /** 发布弹窗按角色收起的档位(Q8:受管级发布只给管理员,服务端 api/workflows 同样拒) */ export const PUBLISH_ROLE_TEXT = { governedAdminOnly: '需要管理员:以受管级发布只给管理员,请联系管理员发布', diff --git a/frontend/src/lib/termsBlueprint.ts b/frontend/src/lib/termsBlueprint.ts index 6970c069..bfa98369 100644 --- a/frontend/src/lib/termsBlueprint.ts +++ b/frontend/src/lib/termsBlueprint.ts @@ -356,6 +356,13 @@ export const BP_EDIT_TEXT = { reconcilerHint: '这项写入能力没登记查询:写回的结果不明时(请求发出去了、没拿到答复),平台查不了,也不会自动重发,' + '要有人去目标系统里看写进去没有。只能由你来选。', reconcilerUnused: '这项写入能力已经登记了查询,结果不明时平台自己去查:这一项用不上了,发布时不会派给他们。', + /** + * 自由形的「谁来做」(05-已知遗留 (d)):按导入的图列出,只定由谁做;行名、会签、时限、超时都来自图,要改回画布 + */ + freeformPeopleHint: '按导入的图列出:这里只定每一步由谁做,会签、时限、超时和步骤本身在画布上改。', + freeformPeopleEmpty: '导入的图里没有要指定由谁处理的步骤。', + freeformWho: '谁来做', + freeformReadOnly: '图上已经写定,这里不用选', removeTitle: (where: string) => `删掉${where}?`, removeConfirm: '删掉', template: '换一个场景', diff --git a/frontend/src/pages/ChatPage.tsx b/frontend/src/pages/ChatPage.tsx index 8bae6511..36579bf9 100644 --- a/frontend/src/pages/ChatPage.tsx +++ b/frontend/src/pages/ChatPage.tsx @@ -17,14 +17,15 @@ import { humanizeError } from '../lib/errors' import { explainRunError } from '../lib/explain' import { canLeave, leavePass } from '../lib/leave' import { formatNumber, formatTime } from '../lib/format' -import { CHAT_BLUEPRINT_TEXT } from '../lib/terms' +import { CHAT_BLUEPRINT_TEXT, REJECTED_TEXT, TURN_COST_TEXT, runCostText } from '../lib/terms' import { useCatalog, useCatalogListState, useDatasources, useOnReconnect, type CatalogListState } from '../store/catalog' import { useAtLeast } from '../store/auth' import { useBlueprint } from '../store/blueprint' import type { BlueprintScope } from '../typesBlueprint' import { blueprintPath, useCreateScope } from './blueprints/shared' import { - agentSteps, busyTurn, isBusy, stepsVisible, useChat, type Attempt, type ChatTurn, type Failure, type ScopeSource, + agentSteps, busyTurn, isBusy, stepsVisible, turnCost, usageOfTurn, useChat, type Attempt, type ChatTurn, type Failure, + type ScopeSource, } from '../store/chat' import { lastVisited, useConversations } from '../store/conversations' import { @@ -383,7 +384,8 @@ function toStreamTurn(turn: ChatTurn): StreamTurn { : turn.phase === 'error' ? 'error' : live || turn.phase === 'checking' ? 'running' : 'done', // 没核对上的不知道结局,用「未运行」那个空心点,不画成完成也不画成失败 - statusCode: turn.phase === 'cancelled' ? 'cancelled' + // 被人驳回的(turn.rejected,服务端 end_reason=rejected)照 lib/status 写「被驳回」,不写「已取消」 + statusCode: turn.phase === 'cancelled' ? (turn.rejected ? 'rejected' : 'cancelled') : turn.phase === 'suspended' ? 'suspended' // 助手反问:这一轮结束了,在等用户回答 : turn.asked && turn.phase === 'done' ? 'waiting' @@ -422,11 +424,34 @@ function toStreamTurn(turn: ChatTurn): StreamTurn { // 可能先到一两条事件,流会拿第一步的起点顶上,计时器从提问那天跑起 startedAt: live ? turn.startedAt : null, elapsedMs: !live && turn.endedAt ? turn.endedAt - turn.startedAt : turn.meta?.ms ?? undefined, + cost: costOf(turn), } streamCache.set(turn, out) return out } +/** + * 这一轮的花费(助手建图 + 运行 + 复核,store 的 usageOfTurn):没花钱、也没有估不出的(演示模型、老轮次)不写。 + * 悬停写各部分,估不出的照实说金额只是下限 + */ +function costOf(turn: ChatTurn): StreamTurn['cost'] { + const usage = usageOfTurn(turn) + const total = turnCost(usage) + if (!usage || !total || (!total.usd && !total.unpriced)) return undefined + const text = runCostText(total.usd, total.unpriced) + if (!text) return undefined + const part = (label: string, u: { cost_usd?: number | null; unpriced_calls?: number } | undefined) => + `${label} ${runCostText(u?.cost_usd ?? 0, u?.unpriced_calls ?? 0) ?? '—'}` + const run = turnCost({ earlier: usage.earlier, run: usage.run }) + const parts = [ + part(TURN_COST_TEXT.copilot, usage.copilot), + part(TURN_COST_TEXT.run, run ? { cost_usd: run.usd, unpriced_calls: run.unpriced } : undefined), + part(TURN_COST_TEXT.review, usage.review), + ].join(' · ') + return { text, title: [TURN_COST_TEXT.title(parts), total.unpriced ? TURN_COST_TEXT.unpriced(total.unpriced) : ''] + .filter(Boolean).join('\n') } +} + /** * 交给流里报错块的那份:store 已经拆好的标题、原因、怎么办,照原样画。运行的失败在 * store 里就按 lib/explain 讲过了(和记录页同一套说法),再把原话交过去会被翻第二遍; @@ -729,6 +754,7 @@ function Remedies({ conversationId, turnId, last, onRephrase, readOnly }: { // 中断、取消、没核对上在流里不出红框(它们不是故障),为什么停、停了意味着什么在这里说 const note = kind === 'suspended' || kind === 'unknown' ? turn.failure + : kind === 'cancelled' && turn.rejected ? { title: '', reason: REJECTED_TEXT.chatReason, hint: turn.rejected.sub } : kind === 'cancelled' ? { title: '', reason: turn.run ? '你已停止本轮,对应的运行也已一并取消。' : '你在搭建工作流时停止了本轮。' } : null @@ -800,7 +826,7 @@ function remedyOf(t: ChatTurn): Remedy | null { } const OUTCOME: Record = { - failed: 'failed', cancelled: 'cancelled', suspended: 'suspended', + failed: 'failed', cancelled: 'cancelled', rejected: 'rejected', suspended: 'suspended', unusable: 'failed', partial: 'waiting', done: 'succeeded', } diff --git a/frontend/src/pages/ConversationList.tsx b/frontend/src/pages/ConversationList.tsx index b26fab62..df648109 100644 --- a/frontend/src/pages/ConversationList.tsx +++ b/frontend/src/pages/ConversationList.tsx @@ -473,6 +473,9 @@ export async function purgeConversation(c: Conversation, navigate: NavigateFunct /** * 这个会话此刻值得标出来的状态(已经取回内存的)。正常完成的不标:正常态安静, * 异常态才醒目。 + * + * 被驳回的标(不画异常色):是审批人的决定,发问的人不一定知道——在等审批的那一问被驳回之后,不标就像是答完了。以前 + * 驳回记成失败,标的是「失败」 */ function liveStatus(turns: ChatTurn[]): string | null { if (!turns.length) return null @@ -481,12 +484,13 @@ function liveStatus(turns: ChatTurn[]): string | null { const last = turns[turns.length - 1] if (last.phase === 'error') return 'failed' if (last.phase === 'suspended') return 'suspended' + if (last.phase === 'cancelled' && last.rejected) return 'rejected' return null } -/** 没打开过的会话:列表接口带来的最后一轮状态。完成、取消的不标,同上 */ +/** 没打开过的会话:列表接口带来的最后一轮状态。完成、取消的不标,被驳回的标,同上 */ const LIST_STATUS: Partial, string>> = { - running: 'running', waiting: 'waiting', error: 'failed', suspended: 'suspended', + running: 'running', waiting: 'waiting', error: 'failed', suspended: 'suspended', rejected: 'rejected', } const listStatus = (c: Conversation): string | null => (c.last_status && LIST_STATUS[c.last_status]) || null diff --git a/frontend/src/pages/DataSourcesTab.tsx b/frontend/src/pages/DataSourcesTab.tsx index a7585768..cfd0c97e 100644 --- a/frontend/src/pages/DataSourcesTab.tsx +++ b/frontend/src/pages/DataSourcesTab.tsx @@ -1773,7 +1773,7 @@ function UploadDecisionBody({ decision, choices, headerRow, sheetNoun, onRecipe )}
-

{onRecipe ? UPLOAD_TEXT.shapeRecipe : UPLOAD_TEXT.shapeRecipeCsv}

+

{UPLOAD_TEXT.shapeRecipe}

{onRecipe && (
diff --git a/frontend/src/pages/SettingsPage.tsx b/frontend/src/pages/SettingsPage.tsx index 68314672..4eb66a44 100644 --- a/frontend/src/pages/SettingsPage.tsx +++ b/frontend/src/pages/SettingsPage.tsx @@ -25,6 +25,7 @@ import { localActor, setLocalActor } from '../lib/actor' import { AGENT_GUARD_TEXT, APP_SETTING_TEXT, DRAFT_MODEL_SETTING_TEXT, IDENTITY_TEXT, JUDGE_SETTING_TEXT, NOTIFY_TEXT, PROVIDER_REF_TEXT, SETTINGS_ROLE_TEXT, TOOL_GATE_TEXT, VERIFY_SETTING_TEXT, judgeByModelText, judgeUnlimitedText, + GENERAL_SETTING_TEXT, } from '../lib/terms' import { atLeast, loginOn, prefsLocalOnly, usePrincipal } from '../store/auth' import type { ThemePref } from '../lib/theme' @@ -32,6 +33,7 @@ import { ProviderOptions, VerifySection, verifyChanged, verifyDraftOf, verifyErrors, verifyLabels, verifyPayload, type VerifyDraft, } from './VerifySettings' import { DraftModelSection } from './DraftModelSettings' +import { GeneralTab } from './settings/GeneralTab' import { IdentityTab } from './settings/IdentityTab' import { NotifyTab } from './settings/NotifyTab' import { AuditTab } from './settings/AuditTab' @@ -41,6 +43,8 @@ import { AuditTab } from './settings/AuditTab' const TABS = [ { key: 'providers', label: '模型接入' }, { key: 'prefs', label: '偏好设置' }, + // 全站共用的部署设置:对外访问地址(环境变量指定时只读) + { key: 'general', label: GENERAL_SETTING_TEXT.tab }, { key: 'system', label: '运行环境' }, // 应用平台(A7):登录方式和人、通知渠道、管理类操作的留痕 { key: 'identity', label: IDENTITY_TEXT.tab }, @@ -74,12 +78,13 @@ export function SettingsPage() { return (
} title="设置" - subtitle={builder ? '模型接入、偏好、运行环境、身份和通知' : SETTINGS_ROLE_TEXT.subtitleUser} /> + subtitle={builder ? '模型接入、偏好、对外访问地址、运行环境、身份和通知' : SETTINGS_ROLE_TEXT.subtitleUser} />
{tab === 'providers' && } {tab === 'prefs' && (builder ? : )} + {tab === 'general' && } {tab === 'system' && } {tab === 'identity' && } {tab === 'notify' && } diff --git a/frontend/src/pages/apps/edit/AccessSections.tsx b/frontend/src/pages/apps/edit/AccessSections.tsx index d29cf615..9791ee04 100644 --- a/frontend/src/pages/apps/edit/AccessSections.tsx +++ b/frontend/src/pages/apps/edit/AccessSections.tsx @@ -8,7 +8,7 @@ import { import { PrincipalPicker, type Principal as Picked } from '../../../components/PrincipalPicker' import { APP_EDITOR_TEXT as T, GRANT_ROLE_HINT, GRANT_ROLE_LABEL, NOTIFY_TEXT } from '../../../lib/terms' import { useLoginOn } from '../../../store/auth' -import type { AppGrant, GrantRole, NotifyChannelRow } from '../../../typesApp' +import type { AppGrant, GrantRole, NotifyChannelRow, NotifyEventRow } from '../../../typesApp' import { applyAppOp } from './appOps' import { useEditor } from './context' @@ -343,9 +343,16 @@ export function NotifySection() { const { draft, setDraft } = useEditor() const [channels, setChannels] = useState(null) useEffect(() => { api.notify.channels().then(({ items }) => setChannels(items)).catch(() => setChannels([])) }, []) + const [registry, setRegistry] = useState(null) + const [registryFailed, setRegistryFailed] = useState(false) + useEffect(() => { api.notify.events().then(setRegistry).catch(() => setRegistryFailed(true)) }, []) const set = (notify: typeof draft.notify) => setDraft((s) => ({ ...s, notify })) - // 「我提交审核的应用有结果」是发给提交人的,不归哪个应用:应用的通知里不列 - const events = Object.keys(NOTIFY_TEXT.eventLabel).filter((e) => e !== 'review.done') + // 应用能订阅哪些由服务端说(notify/service.EVENTS 推给「应用的订阅」这条路的):「我提交审核的应用有结果」发给提交人、 + // 登记了还没推送的都不列。这条订阅里已经勾着的照样列出来(以前界面列过、其实推不到的也在内),看得到、能取消 + const appEvents = new Set((registry ?? []).filter((r) => r.scopes.includes('app')).map((r) => r.event)) + const known = Object.keys(NOTIFY_TEXT.eventLabel) + const eventsOf = (subscribed: string[]) => [...known.filter((e) => appEvents.has(e) || subscribed.includes(e)), + ...subscribed.filter((e) => !known.includes(e))] return (
@@ -372,12 +379,12 @@ export function NotifySection() { set(draft.notify.filter((_, j) => j !== i))} />
- {events.map((ev) => ( + {eventsOf(n.events).map((ev) => ( ))}
@@ -385,6 +392,7 @@ export function NotifySection() { ))}
)} + {registryFailed &&

{NOTIFY_TEXT.eventsError}

}
) } diff --git a/frontend/src/pages/apps/personal.tsx b/frontend/src/pages/apps/personal.tsx index c5298c15..2e10036d 100644 --- a/frontend/src/pages/apps/personal.tsx +++ b/frontend/src/pages/apps/personal.tsx @@ -171,7 +171,7 @@ function ReviewDialog({ appId, onClose, onSent }: {

{T.reviewIntro}

{T.reviewTo}
- +

{T.reviewToHint}

diff --git a/frontend/src/pages/blueprints/BlueprintCards.tsx b/frontend/src/pages/blueprints/BlueprintCards.tsx index be72d9ed..86659199 100644 --- a/frontend/src/pages/blueprints/BlueprintCards.tsx +++ b/frontend/src/pages/blueprints/BlueprintCards.tsx @@ -7,6 +7,7 @@ * 哪些项必填(P7)。 * * 写回卡片上「结果不明时谁去核实」只在目标能力没登记查询时摆(服务端 Blueprint.reconciler_needed),只能由人选(同「谁能看」)。 + * 自由形的「谁来做」按服务端给的 derived_people 摆(FreeformPeople):骨架是导入的图,卡片只定每一步由谁做。 * * 每一项可以点「修改」:改完发一个 bp_set(值是纯值,服务端记成来源「你」、已确认);待确认的可以直接点「确认」(只改状态, * 来源不变),一张卡片上待确认的不止一项时卡头另有「确认全部」,一次修改带上这几项的状态(服务端整批落或整批不落)。 @@ -26,7 +27,7 @@ import { } from '../../lib/terms' import { chatBusy, useBlueprint } from '../../store/blueprint' import type { - Blueprint, BpDataItem, BpField, BpItem, BpOp, BpStep, ItemStatus, JointState, PlainIssue, TemplateInfo, + Blueprint, BpDataItem, BpField, BpItem, BpOp, BpStep, DerivedPerson, ItemStatus, JointState, PlainIssue, TemplateInfo, } from '../../typesBlueprint' import { SELF_RECORDS, ValueEditor, valueText, type EditorKind, type EditorScope } from './editors' import { isItem, missingPaths, visibleCards, type CardKey } from './shared' @@ -175,6 +176,7 @@ function CardFor({ card, ctx, template }: { card: CardKey; ctx: CardCtx; templat ) case 'people': { + if (ctx.bp.derived_people) return const items = spec.people.flatMap((s, i) => STEP_KEYS.map((k) => [`/people/${i}/${k}`, s[k] as BpItem] as [string, BpItem])) return ( {BLUEPRINT_TEXT.empty}

} +/** + * 自由形的「谁来做」(05-已知遗留 (d)):按服务端给的 derived_people 一行一行摆,不自己从图里推。岗位行、核实人那一行只定 + * 由谁做(改了发 bp_set 到那一行的 path);发起人、表达式、外部系统回调的步骤只读。不能加减步骤、不能改会签和时限:骨架是 + * 导入的图,要改回画布 + */ +function FreeformPeople({ ctx, rows, note }: { ctx: CardCtx; rows: DerivedPerson[]; note?: string }) { + const spec = ctx.bp.spec + const itemOf = (row: DerivedPerson): BpItem | null => { + if (row.kind === 'reconciler') return spec.writeback?.reconciler ?? UNSET + if (row.kind !== 'slot') return null + return spec.people.find((s) => s.id === row.slot)?.who ?? UNSET + } + const items = rows.flatMap((r): [string, BpItem][] => { + const item = itemOf(r) + return r.path && item ? [[r.path, item]] : [] + }) + const paths = new Set(items.map(([p]) => p)) + const issues = (ctx.bp.issues ?? []).filter((i) => i.path === '/people' || (i.path != null && paths.has(i.path))) + return ( + +

{BP_EDIT_TEXT.freeformPeopleHint}

+ {rows.length === 0 &&

{BP_EDIT_TEXT.freeformPeopleEmpty}

} + {rows.map((row, i) => { + const item = itemOf(row) + return ( +
+
{row.label}
+

{row.detail}

+ {row.path && item + ? + :

{BP_EDIT_TEXT.freeformReadOnly}

} +
+ ) + })} +
+ ) +} + function Card({ card, status, note, actions, issues, children }: { card: CardKey; status: ItemStatus | null; note?: string; actions?: ReactNode; issues?: PlainIssue[]; children: ReactNode }) { diff --git a/frontend/src/pages/catalog/ProfileDialog.tsx b/frontend/src/pages/catalog/ProfileDialog.tsx index 321ad9c3..b14641fc 100644 --- a/frontend/src/pages/catalog/ProfileDialog.tsx +++ b/frontend/src/pages/catalog/ProfileDialog.tsx @@ -780,6 +780,10 @@ export function ProfileTaskBody({ detail: d, mode, filled = {}, focusAfter = nul {PT.dateProposal(date.column)} ,{PT.dateSpan(date.min.slice(0, 19), date.max.slice(0, 19))} + {(() => { + const invalid = d.date_ranges.find((r) => r.column === date.column)?.invalid + return invalid ? ,{PT.dateInvalid(invalid)} : null + })()} )} @@ -788,6 +792,7 @@ export function ProfileTaskBody({ detail: d, mode, filled = {}, focusAfter = nul {d.date_ranges.map((r) => (
  • {r.column}:{PT.dateSpan(r.min.slice(0, 19), r.max.slice(0, 19))} + {r.invalid ? `,${PT.dateInvalid(r.invalid)}` : null}
  • ))} diff --git a/frontend/src/pages/catalog/ProfileSettings.tsx b/frontend/src/pages/catalog/ProfileSettings.tsx index ddd9a3ec..91f13ee1 100644 --- a/frontend/src/pages/catalog/ProfileSettings.tsx +++ b/frontend/src/pages/catalog/ProfileSettings.tsx @@ -14,7 +14,7 @@ import type { ProfileForm } from './profile' // 数据剖析的设置:开关、五项预算、风险说明。数据源编辑框里是其中一段;导入表格的源没有编辑框,数据目录页在 // 剖析被拒(未开启)时也要就地打开它,所以另有一个只改这一项的弹窗(options 的其余键原样带上)。 // 剖析会对业务库发查询,开关默认关着;开启后五项预算才展开,风险说明写的是此刻填的数。 -// 预算按表算:每张表最多查询数、每张表时长上限(用完只停这张表),另有单条查询时限、抽样键值数、整表统计行数上限; +// 预算按表算:每张表最多查询数、每张表时长上限(用完只停这张表),另有单条查询时限、核对键值数、整表统计行数上限; // 没有整次剖析的总量上限(任务自己在连续失败、多数超时时停下)。 // =========================================================================== diff --git a/frontend/src/pages/import/ImportWizard.tsx b/frontend/src/pages/import/ImportWizard.tsx index a2b23b80..2a28ac11 100644 --- a/frontend/src/pages/import/ImportWizard.tsx +++ b/frontend/src/pages/import/ImportWizard.tsx @@ -55,7 +55,8 @@ const NAME_RE = /^[a-z][a-z0-9_]{0,40}$/ /** 提交时遇到这些:试运行库已被消耗、作废或被改过,回到配方步骤重新试运行(期 3 加了并集、各期被改过的两种) */ const RERUN_CODES = new Set(['trial_required', 'trial_tampered', 'base_changed', 'name_taken', 'union_tampered', 'part_tampered']) /** 选择框收的扩展名,与 RECIPE_TEXT.pickFormats 的说法一致 */ -const PICK_ACCEPT = '.xlsx,.xlsm' +// CSV / TSV 也走配方(服务端 csv_grid 把它适配成一张「工作表」,名字固定为 CSV) +const PICK_ACCEPT = '.xlsx,.xlsm,.csv,.tsv' const ENDED = new Set(['committed', 'discarded', 'expired']) /** 文件名里的两个日期(「月报导出_2026-09-01_2026-09-30.xlsx」):只作统计期录入框的建议,服务端会再核对 */ diff --git a/frontend/src/pages/import/ManifestBody.tsx b/frontend/src/pages/import/ManifestBody.tsx index c19a028e..b2665b84 100644 --- a/frontend/src/pages/import/ManifestBody.tsx +++ b/frontend/src/pages/import/ManifestBody.tsx @@ -3,7 +3,7 @@ import clsx from 'clsx' import type { CheckResult, ImportAcceptance, ManifestOut } from '../../types' import { CopyButton, Notice } from '../../components/ui' import { formatBytes, formatCost, formatDateTime, formatTokens } from '../../lib/format' -import { RAW_STATE_LABEL, RECIPE_ORIGIN_LABEL, RECIPE_TEXT, VERSIONS_TEXT as VT } from '../../lib/terms' +import { RAW_STATE_LABEL, RECIPE_ORIGIN_LABEL, RECIPE_TEXT, SIGNER_TEXT, VERSIONS_TEXT as VT } from '../../lib/terms' import { CheckStatusChip } from './ImportReceipt' import { ReceiptSummary } from './ReceiptBlocks' @@ -79,7 +79,11 @@ export function ManifestBody({ m, fileName, rawState, recipeSeq }: { {short(c.file?.raw_sha256) && {VT.sha(short(c.file?.raw_sha256))}} {rawState != null && {RAW_STATE_LABEL[rawState] ?? rawState}}
    - {c.signed_by?.name &&
    {VT.signedBy(c.signed_by.name)}
    } + {c.signed_by?.name && ( +
    + {signerText(c.signed_by.name, !!c.signed_by.verified)} +
    + )} @@ -146,7 +150,9 @@ export function ManifestBody({ m, fileName, rawState, recipeSeq }: {
    - {VT.signedBy(a.signed_by || VT.unsigned)} + {/* 接受理由由提交这一期的人写,署名同清单的 signed_by:是不是登录身份按清单里记下的说 */} + {a.signed_by ? signerText(a.signed_by, !!c.signed_by?.verified && a.signed_by === c.signed_by.name) + : VT.signedBy(VT.unsigned)} {a.at && {formatDateTime(a.at)}}
    @@ -235,3 +241,11 @@ function RawJson({ summary, value, attr }: { summary: string; value: unknown; at ) } + +/** + * 留痕里的署名怎么说:按记下的「是否认证」写「登录身份:x」或「署名(未认证):x」(05-已知遗留 (c)),和确认框里那一行 + * (VersionsDialog 的 signerLine)同一套说法。旧清单记的是未认证,照记下的说 + */ +export function signerText(name: string, verified: boolean): string { + return verified ? SIGNER_TEXT.signedIn(name) : VT.signedBy(name) +} diff --git a/frontend/src/pages/import/RecipePanel.tsx b/frontend/src/pages/import/RecipePanel.tsx index 3b2f0487..003f029b 100644 --- a/frontend/src/pages/import/RecipePanel.tsx +++ b/frontend/src/pages/import/RecipePanel.tsx @@ -72,8 +72,13 @@ export function fillRecipeDefaults(input: Recipe | null | undefined): Recipe | n block.extra_columns ??= 'reject' block.ignore_columns ??= [] block.merged_data ??= 'reject' - block.rows = { blank_rows: 'stop', total_row: null, ...(isObj(block.rows) ? block.rows : {}) } + block.rows = { blank_rows: 'stop', total_row: null, subtotals: null, no_subtotals: false, ...(isObj(block.rows) ? block.rows : {}) } if (isObj(block.rows.total_row)) block.rows.total_row.keep_as ??= null + if (isObj(block.rows.subtotals)) { + block.rows.subtotals = { scope: 'since_previous', group_column: null, keep_as: null, ...block.rows.subtotals } + } + block.period ??= null + if (isObj(block.period)) block.period = { column: null, check: 'within_context', synthesize: null, ...block.period } block.values = { blank: 'null', text_number: 'reject', formula: 'accept_cached', ...(isObj(block.values) ? block.values : {}), } @@ -97,6 +102,10 @@ export interface RecipeColumn { name: string; type: string; role: string; header /** 列表合计行另存的表固定有这一列(契约 LIST_TOTAL_DIM) */ const LIST_TOTAL_DIM = '合计项' +/** 期 5:列表小计另存的表固定有这一列(契约 LIST_SUBTOTAL_DIM) */ +const LIST_SUBTOTAL_DIM = '小计项' +/** 期 5:列表每行加统计期时的两列(契约 LIST_PERIOD_START / LIST_PERIOD_END) */ +const LIST_PERIOD_COLUMNS = ['统计期起', '统计期止'] /** * 由配方推出每张表的列,照服务端 derive_tables 的顺序:宽表 = 轴、各指标;长表 = 轴、维度、常量(按列名排序)、 @@ -140,11 +149,18 @@ export function tableColumns(recipe: Recipe | null): Map name: String(c?.name ?? ''), type: c?.type === 'DATE' ? 'TEXT' : String(c?.type ?? 'TEXT'), role: c?.type === 'INTEGER' || c?.type === 'REAL' ? 'measure' : 'text', header: c?.header, })) - put(block.table, cols) + const synth = isObj(block.period) && block.period.synthesize + ? LIST_PERIOD_COLUMNS.map((name) => ({ name, type: 'TEXT', role: 'period' })) : [] + put(block.table, [...cols, ...synth]) + const numeric = cols.filter((c) => c.type === 'INTEGER' || c.type === 'REAL').map((c) => ({ ...c, role: 'value' })) const total = block.rows?.total_row if (isObj(total) && total.keep_as) { - put(total.keep_as, [{ name: LIST_TOTAL_DIM, type: 'TEXT', role: 'dim' }, - ...cols.filter((c) => c.type === 'INTEGER' || c.type === 'REAL').map((c) => ({ ...c, role: 'value' }))]) + put(total.keep_as, [{ name: LIST_TOTAL_DIM, type: 'TEXT', role: 'dim' }, ...numeric]) + } + const sub = block.rows?.subtotals + if (isObj(sub) && sub.keep_as) { + const group = sub.scope === 'group_column' && sub.group_column ? [{ name: String(sub.group_column), type: 'TEXT', role: 'dim' }] : [] + put(sub.keep_as, [{ name: LIST_SUBTOTAL_DIM, type: 'TEXT', role: 'dim' }, ...group, ...numeric]) } } } @@ -164,6 +180,7 @@ export function renameTable(r: Recipe, old: string, nu: string) { if (!isObj(block)) continue if (block.table === old) block.table = nu if (isObj(block.rows?.total_row) && block.rows.total_row.keep_as === old) block.rows.total_row.keep_as = nu + if (isObj(block.rows?.subtotals) && block.rows.subtotals.keep_as === old) block.rows.subtotals.keep_as = nu for (const seg of arr(block.segments)) { if (!isObj(seg)) continue if (seg.table === old) seg.table = nu @@ -205,6 +222,11 @@ function renameRefs(r: Recipe, table: string, old: string, nu: string) { if (block.table === table && isObj(block.rows?.total_row) && block.rows.total_row.label_column === old) { block.rows.total_row.label_column = nu } + if (block.table === table && isObj(block.rows?.subtotals)) { + if (block.rows.subtotals.label_column === old) block.rows.subtotals.label_column = nu + if (block.rows.subtotals.group_column === old) block.rows.subtotals.group_column = nu + } + if (block.table === table && isObj(block.period) && block.period.column === old) block.period.column = nu for (const seg of arr(block.segments)) { if (isObj(seg?.verify) && seg.verify.against_table === table && seg.verify.value === old) seg.verify.value = nu } @@ -736,6 +758,10 @@ function ListEditor({ s, b, block, edit }: { s: number; b: number; block: any; e const at = (r: Recipe) => r.sheets[s].blocks[b] const columns: any[] = arr(block.columns) const total = isObj(block.rows?.total_row) ? block.rows.total_row : null + const subtotals = isObj(block.rows?.subtotals) ? block.rows.subtotals : null + const period = isObj(block.period) ? block.period : null + const periodKind = period?.synthesize ? 'synthesize' : period?.column ? 'column' : 'none' + const dateColumns = columns.filter((c) => c?.type === 'DATE').map((c) => String(c?.name ?? '')) return (
    @@ -857,6 +883,87 @@ function ListEditor({ s, b, block, edit }: { s: number; b: number; block: any; e
    )} + + edit((r) => { + const x = at(r) + x.rows = isObj(x.rows) ? x.rows : {} + x.rows.subtotals = on + ? { label_column: arr(x.columns)[0]?.name ?? '', pick: '小计', scope: 'since_previous', group_column: null, keep_as: null } + : null + // 有小计和「没有小计行」只能二选一(后端静态校验 subtotals_conflict) + if (on) x.rows.no_subtotals = false + })}> + {RECIPE_TEXT.subtotalsOn} + + {!subtotals && ( +
    + edit((r) => { + const x = at(r) + x.rows = isObj(x.rows) ? x.rows : {} + x.rows.no_subtotals = on + })}> + {RECIPE_TEXT.noSubtotals} + +
    + )} + {subtotals && ( +
    + + String(c?.name ?? ''))} + onChange={(v) => edit((r) => { at(r).rows.subtotals.group_column = v })} /> + + )} +
    + )} +
    + + edit((r) => { + const x = at(r) + x.period = v === 'none' ? null + : v === 'synthesize' ? { column: null, check: 'within_context', synthesize: '统计期' } + : { column: dateColumns[0] ?? '', check: 'within_context', synthesize: null } + })} /> + {periodKind === 'column' && ( +
    + + setDraft(e.target.value)} + onKeyDown={(e) => { if (e.key === 'Enter' && !isComposing(e)) { e.preventDefault(); void save() } }} + data-public-url-input="" /> + +
    + + ) : ( +
    +
    + {address.value ?? T.unset} +
    +

    + + {address.env ? `${T.envLocked}。${T.envLockedHint}` : T.readOnly} +

    +
    + )} + + + ) +} diff --git a/frontend/src/pages/settings/NotifyTab.tsx b/frontend/src/pages/settings/NotifyTab.tsx index d30ffda5..297c6955 100644 --- a/frontend/src/pages/settings/NotifyTab.tsx +++ b/frontend/src/pages/settings/NotifyTab.tsx @@ -6,8 +6,9 @@ import { } from '../../components/ui' import { formatDateTime } from '../../lib/format' import { NOTIFY_TEXT as T, SETTINGS_ROLE_TEXT } from '../../lib/terms' +import { NOTIFY_PERSONAL_EVENT_LABEL } from '../../lib/termsApp' import { useAtLeast } from '../../store/auth' -import type { NotifyChannelRow } from '../../typesApp' +import type { NotifyChannelRow, NotifyEventRow } from '../../typesApp' /* * 设置 → 通知(A4 第 7 条):待办、动作的事件推送到哪些地址。格式只有 Webhook 的四种(Q6),发送核心和定时任务的 @@ -18,6 +19,9 @@ import type { NotifyChannelRow } from '../../typesApp' const FORMATS = ['generic', 'feishu', 'wecom', 'dingtalk'] as const const EVENTS = Object.keys(T.eventLabel) +/** 事件在这个范围的渠道上怎么叫:个人渠道的实例结束只推自己发起的,叫「我发起的实例完成 / 失败」 */ +const eventLabel = (event: string, scope: string): string => + (scope === 'user' ? NOTIFY_PERSONAL_EVENT_LABEL[event] : undefined) ?? T.eventLabel[event] ?? event export function NotifyTab() { const builder = useAtLeast('builder') @@ -86,7 +90,7 @@ export function NotifyTab() {
    {row.url_host} - {row.events.length > 0 && ` · ${row.events.map((e) => T.eventLabel[e] ?? e).join('、')}`} + {row.events.length > 0 && ` · ${row.events.map((e) => eventLabel(e, row.scope)).join('、')}`}