diff --git a/backend/app/api/ledger.py b/backend/app/api/ledger.py index 5c8baa41..5917c529 100644 --- a/backend/app/api/ledger.py +++ b/backend/app/api/ledger.py @@ -11,7 +11,10 @@ L1、L5：参数只从比对工件取，使用者只能按他有权使用的应用固定着的版本出具）； - POST /api/ledger/answers/{run_id}/reconcile（同上）：口径对账，点开才跑一次确定性派生的只读图（app/ledger/reconcile.py）； - GET /api/ledger/certify-queue、POST /api/ledger/certify-queue/dismiss（router，搭建者，第 3 波）：可沉淀的问题、「不是口径」 - （app/ledger/certify.py）。这几个新接口都不放给访问密钥（auth/keys.KEY_ROUTES 不登记）。 + （app/ledger/certify.py）。 +- GET /api/ledger/recall/{workflow_id}/{version}、POST …/continue（router，搭建者，第 4 波）：口径升到这一版之后的召回批次—— + 每组的结论、「另有 M 组未重算」、手动接着算（app/ledger/recall.py）。版本用量（workflow-versions）另加问数回答、导出、召回。 +这几个新接口都不放给访问密钥（auth/keys.KEY_ROUTES 不登记）。 """ from __future__ import annotations @@ -24,7 +27,7 @@ from pydantic import BaseModel, Field from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession -from app.db.base import get_session +from app.db.base import SessionLocal, get_session from app.db.models import Workflow from app.db.models_process import ActionRecord @@ -40,12 +43,18 @@ answers_router = APIRouter(prefix="/api/ledger/answers", tags=["ledger"]) @router.get("/workflow-versions/{workflow_id}/{version}") async def workflow_version(workflow_id: str, version: int, session: AsyncSession = Depends(get_session) ) -> dict[str, Any]: - """一个工作流版本被哪些应用固定、生效记录数、正式运行数。""" + """一个工作流版本被哪些应用固定、生效记录数、正式运行数；另加口径账本的用量（next-round 02 做法 8）：被几次问数回答用过 + （answers：比对里等价已证明的探索运行）、导出过几次（exports：这些答案和这一版的正式运行上的报告导出审计）、定时任务几个、 + 召回（recall：以这一版为新版本召回过几组、其中数变了几组，作为旧版本被召回过几组）。""" from app.apps.versions import usage + from app.ledger.uses import version_usage if await session.get(Workflow, workflow_id) is None: raise HTTPException(404, "工作流不存在，可能已被删除") - return await usage(session, workflow_id, version) + base = await usage(session, workflow_id, version) + extra = await version_usage(session, workflow_id, version) + return {**base, "answers": extra["answers"], "exports": extra["exports"], "schedules": extra["schedules"], + "recall": extra["recall"]} def effect_row(r: ActionRecord, *, receipt: bool = True) -> dict[str, Any]: @@ -163,6 +172,38 @@ async def reconcile_answer(run_id: str, payload: ReconcileIn) -> dict[str, Any]: return {"run_id": run.id, "reconciled": ledger_reconcile.reconciled_view(run)} +@router.get("/recall/{workflow_id}/{version}") +async def recall_batches(workflow_id: str, version: int, session: AsyncSession = Depends(get_session) + ) -> dict[str, Any]: + """口径升到这一版之后的召回（app/ledger/recall.batches）：每组（旧版本、参数）的结论——数变了 / 没变 / 无法重算：原因 / + 重算失败 / 还在算 / 未重算——和「另有 M 组未重算」、能不能接着算。计划从已有的召回运行现算，不另存。""" + from app.ledger import recall + + if await session.get(Workflow, workflow_id) is None: + raise HTTPException(404, "工作流不存在，可能已被删除") + return await recall.batches(session, workflow_id, version) + + +@router.post("/recall/{workflow_id}/{version}/continue") +async def recall_continue(workflow_id: str, version: int, session: AsyncSession = Depends(get_session) + ) -> dict[str, Any]: + """接着算：把还没重算的组再发起一批（每批 recall_limit 组，上限是 0 时按缺省的组数），交回发起了的召回运行和新的批次 + 情况。都算过了的回 409。""" + from app.api.coded import CodedHTTPException + from app.api.deps import current_principal + from app.ledger import recall + + if await session.get(Workflow, workflow_id) is None: + raise HTTPException(404, "工作流不存在，可能已被删除") + current = await recall.batches(session, workflow_id, version) + if not current["can_continue"]: + raise CodedHTTPException(409, current.get("problem") or "没有还没重算的组", recall.REFUSED) + launched = await recall.launch(workflow_id, version, batch="manual", actor=current_principal().username) + async with SessionLocal() as fresh: + view = await recall.batches(fresh, workflow_id, version) + return {"launched": launched, **view} + + @router.get("/certify-queue") async def certify_queue(session: AsyncSession = Depends(get_session)) -> dict[str, Any]: """可沉淀的问题（认证队列，app/ledger/certify.py）：确定性重复的 SQL 形状、问题聚类、被反复用的答案。每项给代表问题、 diff --git a/backend/app/api/workflows.py b/backend/app/api/workflows.py index 62dee247..6355d69f 100644 --- a/backend/app/api/workflows.py +++ b/backend/app/api/workflows.py @@ -577,9 +577,12 @@ async def commit_publish(session: AsyncSession, workflow: Workflow, snapshot: Wo workflow.published_by = actor await session.commit() # 口径账本（next-round 02 做法 8）：从问数据固化出的口径发布了，来源运行在后台按它补比一次（不挡发布） + from app.ledger import recall as ledger_recall from app.ledger.certify import after_publish after_publish(workflow.id, workflow.origin) + # 口径召回（next-round 02 做法 7、拍板 L6）：用过旧版本的答案在后台按计划排队重算（每次最多 recall_limit 组），失败不影响发布 + ledger_recall.after_publish(workflow.id, snapshot.version, actor) return { "ok": True, "level": level, @@ -590,6 +593,20 @@ async def commit_publish(session: AsyncSession, workflow: Workflow, snapshot: Wo } +@router.get("/{workflow_id}/recall-preview") +async def recall_preview(workflow_id: str, version: int | None = None, + session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """发布弹窗的召回预告（next-round 02 做法 7、8）：把 version（缺省是最新保存的那一版）发布成新版本之后，旧版本被多少问数 + 回答、导出、定时任务、应用用过，预计自动重算几组，超出上限的几组（「另有 M 组未重算」），不能重算的原因各几组。和发布之后 + 真的召回走同一份计划（ledger/recall.build_plan），规模一致。只读，不发起。""" + from app.ledger import recall as ledger_recall + + workflow = await session.get(Workflow, workflow_id) + if not workflow: + raise HTTPException(404, _GONE) + return await ledger_recall.preview(session, workflow_id, int(version or workflow.version)) + + # -------------------------------------------------------------------------- # 发布前检查与自动修复：两个接口都只读，不改库、不发布 # -------------------------------------------------------------------------- diff --git a/backend/app/bundle/build.py b/backend/app/bundle/build.py index caada4ab..9dd73f2e 100644 --- a/backend/app/bundle/build.py +++ b/backend/app/bundle/build.py @@ -1184,6 +1184,19 @@ def _summary(etype: str, data: Any) -> str: ("none", "没有相关口径"), ("skipped", "没有比完")) if isinstance(counts.get(k), int) and counts[k]] return "封存后比对已发布口径" + (f"：{'、'.join(parts)}" if parts else "") + if etype == EventType.LEDGER_RECALLED: + # 口径召回（next-round 02 做法 7）：只说按哪一版核对、数变没变，不带新旧值和差值 + to = data.get("to_version") + head = f"已按 v{to} 核对" if isinstance(to, int) else "口径召回核对" + changed = data.get("changed") + return head + ("：数变了" if changed is True else "，不影响" if changed is False else "") + if etype == EventType.LEDGER_BASIS_CHANGED: + # 答案依据的目录事实、表结构变了：只说哪一类、几项，不带表名、列名、取值 + kind = {"catalog": "目录事实", "structure": "表结构", "values": "取值", "catalog_edit": "目录"}.get( + str(data.get("kind") or ""), "") + items = data.get("items") + count = len(items) if isinstance(items, list) else 0 + return f"依据的{kind or '东西'}有变" + (f"（{count} 项）" if count else "") if etype == EventType.LOG and isinstance(data.get("message"), str): return _clip(data["message"]) return "" diff --git a/backend/app/bundle/verifier.js b/backend/app/bundle/verifier.js index 48edbc12..f4ae077d 100644 --- a/backend/app/bundle/verifier.js +++ b/backend/app/bundle/verifier.js @@ -65,6 +65,7 @@ 'human.requested': '请求人工审批', 'human.resolved': '人工审批结果', 'approval.inquiry': '审批质询', 'issuance': '出具判定', 'caliber.upgrade': '口径卡升版处置', 'catalog.drift': '数据目录变化', 'schema.drift': '表结构变化', 'report.checked': '报告自查', 'evidence.judged': '按需裁判', 'file.extracted': '读入附件', + 'ledger.matched': '比对已发布口径', 'ledger.recalled': '口径召回核对', 'ledger.basis_changed': '答案的依据有变', 'log': '日志', 'usage': '用量', } const T = { diff --git a/backend/app/core/events.py b/backend/app/core/events.py index 3a39eb14..4cbd7c7a 100644 --- a/backend/app/core/events.py +++ b/backend/app/core/events.py @@ -92,6 +92,16 @@ class EventType(StrEnum): # algo, counts, trigger}。追加在封存之后（post_seal），是批注不是证据：比对依据的已发布口径在运行之外，不能混进运行 # 自身的封存事实；封存核对照旧一致 LEDGER_MATCHED = "ledger.matched" + # 口径召回（02 做法 7）：口径发布新版本后，在同一份数据上把新旧两版各算一次（召回运行，origin_kind=recall）。召回运行 + # 结束时在它自己身上记一条结论 {role: recall, changed, workflow_id, from_version, to_version, metrics, answers, tasks}； + # 数没变的那几个原答案的运行上各记一条 {role: answer, changed: false, recall_run, workflow_id, from_version, + # to_version}，界面上写「已按 vN 核对，不影响」（数变了的开 recall 待办，不在原答案上记）。同 ledger.matched：追加在 + # 封存之后（post_seal），是批注不是证据，不进封存范围 + LEDGER_RECALLED = "ledger.recalled" + # 答案依据的东西变了（02 做法 7 二、三）：kind=catalog 目录里影响数字的一项被驳回或改了（冻结的目录里看得到这一项）； + # structure 用到的表或列被删、列的类型变了；values 触及有效记录条件的新码值；catalog_edit 处理变化时改了目录。带数据源、 + # 表、哪几项（路径和叫法），按新目录跑 SQL 检查新出现的错误级问题条数。同上，封存之后追加，不进封存范围 + LEDGER_BASIS_CHANGED = "ledger.basis_changed" # 应用平台（docs/design/app-platform-and-look.md §3.11） # 运行来源：发起时写进 launch_run 的 notes，在封存范围内（engine/origin.py） diff --git a/backend/app/data/catalog.py b/backend/app/data/catalog.py index a1144e39..fb39a3e9 100644 --- a/backend/app/data/catalog.py +++ b/backend/app/data/catalog.py @@ -919,7 +919,17 @@ async def review_entry(session: AsyncSession, source_id: str, table: str, path: if if_version != current: raise CatalogConflict(table, current) notes = review_item(entry.notes if entry else {}, path, action) - return await write_entry(session, source_id, table, notes, if_version=current, actor=actor) + written = await write_entry(session, source_id, table, notes, if_version=current, actor=actor) + after_review(source_id, table, entry.notes if entry else None, written) + return written + + +def after_review(source_id: str, table: str, before: Mapping[str, Any] | None, written: CatalogEntry) -> None: + """审阅、人工编辑、保存提案写进目录之后（已提交）：影响数字的项被驳回或改了的，口径账本在后台找出依据了它的答案、标 + 「依据有变」（ledger/recall.after_review，docs/design/next-round/02-口径账本.md 做法 7 二）。不挡这次写入，出错只记日志。""" + from app.ledger import recall + + recall.after_review(source_id, table, before, written.notes, written.version) async def save_human_edit(session: AsyncSession, source_id: str, table: str, submitted: Mapping[str, Any], *, @@ -939,7 +949,9 @@ async def save_human_edit(session: AsyncSession, source_id: str, table: str, sub schema_cache = getattr(row, "schema_cache", None) notes = apply_human_edit(entry.notes if entry else {}, submitted, table=table, inferable=structure_relation_ids(schema_cache, table)) - return await write_entry(session, source_id, table, notes, if_version=current, actor=actor) + written = await write_entry(session, source_id, table, notes, if_version=current, actor=actor) + after_review(source_id, table, entry.notes if entry else None, written) + return written # ========================================================================== @@ -1251,7 +1263,9 @@ async def save_patch(session: AsyncSession, source_id: str, table: str, changes: if if_version != current: raise CatalogConflict(table, current) notes = apply_patch(entry.notes if entry else {}, changes, table=table, tables=tables) - return await write_entry(session, source_id, table, notes, if_version=current, actor=actor) + written = await write_entry(session, source_id, table, notes, if_version=current, actor=actor) + after_review(source_id, table, entry.notes if entry else None, written) + return written # ========================================================================== diff --git a/backend/app/data/catalog_review.py b/backend/app/data/catalog_review.py index 5e7fba68..43e27594 100644 --- a/backend/app/data/catalog_review.py +++ b/backend/app/data/catalog_review.py @@ -22,7 +22,8 @@ 使用次数比运行次数多时两个都写（「被 2 次运行用到，使用 5 次」），和清单、运行列表上的数都对得上。 - **批量审阅**（review_batch，POST /catalog/review/batch）：逐表按 if_version 乐观锁，同一张表的几条在一次写入里完成， 一张表冲突、写不进去不影响别的表；路径不对、表不存在逐条报（results 和请求一一对应）。审阅的规则同单项审阅 - （catalog.review_item）。 + （catalog.review_item）。写进去之后影响数字的项被驳回了的，口径账本在后台标依据了它的答案「依据有变」 + （catalog.after_review → ledger/recall，next-round 02 做法 7 二）。 - **被哪些运行用过**（table_runs，GET /catalog/{table}/usage）：取法同使用次数（catalog.query_snapshot_tables），按运行 去重，新的在前；total 是运行次数。 - **进度**（review_progress，GET /catalog/review-progress）：常用表（使用次数前 TOP_N 张，和起草对话框「使用次数最多的 @@ -379,6 +380,8 @@ async def review_batch(session: AsyncSession, source: Any, items: Iterable[Batch continue for i in applied: results[i] = _result(items[i], ok=True, version=written.version) + # 驳回了影响数字的项：口径账本在后台标依据了它的答案「依据有变」（catalog.after_review，同单项审阅） + catalog.after_review(source.id, key, entry.notes if entry else None, written) return [r for r in results if r is not None] diff --git a/backend/app/data/source_changes.py b/backend/app/data/source_changes.py index 876bcdf2..fe95b0af 100644 --- a/backend/app/data/source_changes.py +++ b/backend/app/data/source_changes.py @@ -11,7 +11,9 @@ 起草（建起草任务）、重新剖析（建剖析任务）、知道了、误报（必须写理由）。每个动作写 resolution，动了目录的记下目录版本， 建了任务的记下任务编号。 - **拉取**（list_changes / affecting）：给召回按 seq 游标拉；seq 自增、不复用。触发召回的只有结构类、处理时改了目录的、 - 以及 rules_touched 的新码值（triggers_recall）；纯取值漂移只算数据质量提醒。 + 以及 rules_touched 的新码值（triggers_recall）；纯取值漂移只算数据质量提醒。探查、剖析任务结束时（notify_after_job） + 口径账本在后台按游标接着拉（ledger/recall.consume_source_changes）；处理变化时改了目录的那一条当场交过去 + （ledger/recall.process_change：它记下来的时候游标早已经过去了）。 - **说法**（summary）：一条变化给人看的一句（接口、日志用）；通知里不放它（不放表名、列名、取值，D9），见 notify/service.notify_source_changed。 @@ -340,10 +342,17 @@ async def notify_after_job(source_id: str, job_id: str) -> int: from app.notify import service service.spawn(service.notify_source_changed(source_id)) - return n except Exception: # noqa: BLE001 logger.exception("任务 %s 结束后推送数据源变化的通知时出错", job_id) - return 0 + n = 0 + try: + # 口径账本的召回（next-round 02 做法 7 三）：这次记下的变化里表或列被删、类型变了的，标用过它的答案「依据的表结构已变」 + from app.ledger import recall + + recall.spawn(recall.consume_source_changes()) + except Exception: # noqa: BLE001 - 辅助步骤 + logger.exception("任务 %s 结束后排队召回时出错", job_id) + return n # ========================================================================== @@ -469,6 +478,11 @@ async def handle(session: AsyncSession, source: Any, row: SourceChange, action: if version is not None: out["catalog_version"] = version out["change"] = change_out(row) + if version is not None and triggers_recall(out["change"]): + # 处理时改了目录：用过这张表的答案「依据有变」（口径账本，后台；游标早已过了这一条，当场交过去） + from app.ledger import recall + + recall.spawn(recall.process_change(out["change"])) return out diff --git a/backend/app/db/models_process.py b/backend/app/db/models_process.py index 100ebdf6..db828902 100644 --- a/backend/app/db/models_process.py +++ b/backend/app/db/models_process.py @@ -17,9 +17,10 @@ from sqlalchemy.orm import Mapped, mapped_column from app.db.base import Base, TimestampMixin, UTCDateTime, new_id, utcnow #: 待办的种类。人工节点三种 + 现有的几种中断 + 动作确认、核实、等外部系统 + 个人应用提交审核（review， -#: docs/design/normal-user-apps.md N7，app/process/tasks.REVIEW_KIND） +#: docs/design/normal-user-apps.md N7，app/process/tasks.REVIEW_KIND）+ 口径召回（recall，docs/design/next-round/02-口径账本.md +#: 做法 7：口径升版后答案的数变了、答案依据的目录或表结构变了；挂在召回运行或答案的运行上，没有 Approval，app/ledger/recall） TASK_KINDS = ("approve", "input", "edit", "tool", "code", "dual_check", "action_confirm", "reconcile", "signal", - "review") + "review", "recall") #: pending 等人；resolving 已凑齐、正在恢复运行（条件 UPDATE 抢到的那个请求负责）；resolved / cancelled / expired 终态 TASK_STATUSES = ("pending", "resolving", "resolved", "cancelled", "expired") #: 动作状态。simulated：探索运行里端点没有 test 环境时的演练，不发送 diff --git a/backend/app/engine/runner.py b/backend/app/engine/runner.py index 714fa680..ac31b861 100644 --- a/backend/app/engine/runner.py +++ b/backend/app/engine/runner.py @@ -133,15 +133,20 @@ class RunManager: # 口径账本：最近几天封存了、一次都没比过的探索运行，在后台补比一次（ledger/match.py）。只读工件和版本表，不查业务库、 # 不调模型；和调度循环同一个开关，拷来的数据目录、测试不补 from app.ledger import match as ledger_match + from app.ledger import recall as ledger_recall ledger_match.start_backfill() + # 口径召回（ledger/recall.py）：最近发布、自动那一批没发完的接着发，结束了还没收尾的召回运行补上收尾。同一个开关 + ledger_recall.start_resume() async def shutdown(self) -> None: self._closing = True - # 后台的口径比对先停：没做完的不记，下次启动补比会再比 + # 后台的口径比对、召回先停：没做完的不记，下次启动补比、接着召回会再做 from app.ledger import match as ledger_match + from app.ledger import recall as ledger_recall await ledger_match.shutdown() + await ledger_recall.shutdown() # 只等还没结束的：结束了的没什么可等，而且它可能属于另一个事件循环， # gather 会直接抛 "future belongs to a different loop" pending = [t for t in self._tasks.values() if not t.done()] @@ -1084,6 +1089,11 @@ class RunManager: if ledger_match.wanted(run_class, origin_kind): ledger_match.schedule(run_id) + if origin_kind == "recall" and status in ("succeeded", "failed", "cancelled"): + # 口径召回运行结束、封存完（ledger/recall.py）：在后台收尾——数变了开 recall 待办，没变的在原答案上记账 + from app.ledger import recall as ledger_recall + + ledger_recall.schedule_settle(run_id) async def _seal(self, run_id: str) -> None: """对到此为止的全部事件算清单哈希，记下封到了哪一条。""" diff --git a/backend/app/ledger/derive.py b/backend/app/ledger/derive.py index 4d22ff4c..8d37d063 100644 --- a/backend/app/ledger/derive.py +++ b/backend/app/ledger/derive.py @@ -17,16 +17,16 @@ cell(nodes.X, 0, '列')、X 是查库的调用工具节点（look/calibers.card_ 拒收、给原因（NotDerivable）。工作流、版本、节点不在了也照实说。 **已实现的**：caliber_probe —— 一份口径的取数 + 口径卡（评测口径题的期望）；reconcile —— 口径对账（第 3 波， -reconcile_graph）；组合式问数用的 caliber_block(card_id, inputs)（第 3 波）。 +reconcile_graph）；组合式问数用的 caliber_block(card_id, inputs)（第 3 波）；recall —— 口径召回（第 4 波，recall_graph）。 **扩展点**（第 3 波的账本按这里扩，不另写生成器）： - 口径对账（02 做法 6）：assemble("reconcile", [ours, caliber_block(pinned)], tail=[对账卡])。ours 是「这次的取数」 （执行过的 SQL 原文，节点 id ours_q1…，build 一个 Block 即可，不经 caliber_block），对账卡是本地 metrics 节点（差值、 相对差，式子引用 cell(nodes.ours_q1, 0, '列') 和口径卡的指标，on_missing=null），出口的契约写 metrics_from。 -- 口径召回（02 做法 7）：assemble("recall", [caliber_block(old, prefix="old_"), caliber_block(new, prefix="new_")], - tail=[召回卡])。同一张图里放两份口径，取数节点 id 会撞：caliber_block 带 prefix 时把拷出来的节点改名，口径卡的 - caliber_from 带上改名映射 nodes={原 id: 新 id}，运行时 metrics._pinned_caliber 按语法树改写式子里的 cell(nodes.X…) - 和 nodes.X（第 3 波起）。 +- 口径召回（02 做法 7，recall_graph）：assemble("recall", [caliber_block(old, prefix="old_"), caliber_block(new, + prefix="new_")], tail=[召回卡])。同一张图里放两份口径，取数节点 id 会撞：caliber_block 带 prefix 时把拷出来的节点改名， + 口径卡的 caliber_from 带上改名映射 nodes={原 id: 新 id}，运行时 metrics._pinned_caliber 按语法树改写式子里的 + cell(nodes.X…) 和 nodes.X（第 3 波起）。 - 组合式问数（02 做法 4，ledger/compose.py）：不经 assemble（拷进的是问数据那张图），只用 caliber_block(prefix=…, card_id=…, inputs=…)：取数节点改名、模板按助手给的参数渲染成字面 SQL，口径卡带改名映射。 - 新的派生种类登记进 KINDS，入口字段、出口、首尾相接都走 assemble，不在调用方拼节点。 @@ -44,7 +44,7 @@ from typing import Any from sqlalchemy.ext.asyncio import AsyncSession -#: 派生图的种类。caliber_probe、reconcile 已实现；recall 归第 4 波（见模块说明的扩展点） +#: 派生图的种类（见模块说明） KINDS = ("caliber_probe", "reconcile", "recall") #: 不能派生的原因的机器码 @@ -380,6 +380,53 @@ def reconcile_graph(pinned: Pinned, ours: Block, tail: Sequence[dict[str, Any]], contract={"metrics_from": RECONCILE_CARD} if card is not None else None) +#: 召回图里的召回卡（细案 02 做法 7） +RECALL_CARD = "recall" + + +def recall_prefixes(i: int) -> tuple[str, str]: + """召回图里第 i 对口径卡（旧版、新版）各自的前缀：第一对 old_ / new_，之后 old2_ / new2_……（同一版里两张卡读同一个 + 取数节点时，各拷一份也不撞）。""" + n = "" if i == 0 else str(i + 1) + return f"old{n}_", f"new{n}_" + + +def recall_fields(old: Pinned, new: Pinned, inputs: Mapping[str, Any]) -> list[dict[str, Any]]: + """召回图的入口：新版的入口字段在前（类型、必填照新版），旧版有、新版没有的接在后面；缺省值都换成这一组的参数。""" + fields: list[dict[str, Any]] = [] + seen: set[str] = set() + for f in [*new.entry_fields, *old.entry_fields]: + name = str(f.get("name") or "") + if not name or name in seen: + continue + seen.add(name) + item = copy.deepcopy(dict(f)) + if name in inputs: + item["default"] = inputs[name] + fields.append(item) + return fields + + +def recall_graph(pairs: Sequence[tuple[Pinned, Pinned]], tail: Sequence[dict[str, Any]], *, + inputs: Mapping[str, Any]) -> DerivedGraph: + """口径召回（细案 02 做法 7）：入口（这一组的参数）→ 每一对口径卡：旧版的取数 + 旧版口径卡（caliber_from 固定旧版）、 + 新版的取数 + 新版口径卡（固定新版），两版的取数节点各带前缀、口径卡的 caliber_from 带改名映射（不撞 id）→ tail（召回卡： + 两版之差在这里算，ledger/recall.recall_card）→ 出口（契约 metrics_from 指向召回卡；不加报告、不调模型）。 + + 两版在同一次运行里先后查同一份数据：差值只反映口径的变化（数据随时间的变化另拿原答案当时的值对照着说）。入口字段按 + 第一对的新版、旧版合起来（recall_fields）。同样的输入永远得到同一张图。""" + if not pairs: + raise ValueError("召回图至少要一对口径卡") + blocks: list[Block] = [] + for i, (old, new) in enumerate(pairs): + old_prefix, new_prefix = recall_prefixes(i) + blocks += [caliber_block(old, prefix=old_prefix), caliber_block(new, prefix=new_prefix)] + old, new = pairs[0] + card = next((n for n in tail if n.get("type") == "metrics" and n.get("id") == RECALL_CARD), None) + return assemble("recall", blocks, fields=recall_fields(old, new, inputs), defaults=new.defaults or old.defaults, + tail=tail, contract={"metrics_from": RECALL_CARD} if card is not None else None) + + def caliber_probe(pinned: Pinned, *, upgrade_policy: str | None = None) -> DerivedGraph: """口径探针：一份口径的取数 + 口径卡（评测口径题的期望，细案 01 做法 8）。节点 id 和原图一样，入口字段照抄。 同样的 pinned 永远得到同一张图（确定性）。""" @@ -388,6 +435,6 @@ def caliber_probe(pinned: Pinned, *, upgrade_policy: str | None = None) -> Deriv __all__ = ["ALIGN_ID", "Block", "DerivedGraph", "ENTRY_ID", "EXIT_ID", "GONE", "KINDS", "NOT_CARD", "NOT_REUSABLE", - "NotDerivable", "OURS_ID", "Pinned", "RECONCILE_CARD", "TEMPLATE", "assemble", "caliber_block", - "caliber_probe", "entry_fields", "load_pinned", "ours_block", "reconcile_graph", "reconcile_prefix", "render_inputs", - "template_problem"] + "NotDerivable", "OURS_ID", "Pinned", "RECALL_CARD", "RECONCILE_CARD", "TEMPLATE", "assemble", "caliber_block", + "caliber_probe", "entry_fields", "load_pinned", "ours_block", "recall_fields", "recall_graph", + "recall_prefixes", "reconcile_graph", "reconcile_prefix", "render_inputs", "template_problem"] diff --git a/backend/app/ledger/issue.py b/backend/app/ledger/issue.py index a05c81d8..66f701bd 100644 --- a/backend/app/ledger/issue.py +++ b/backend/app/ledger/issue.py @@ -208,12 +208,7 @@ async def plan_issue(session: Any, run: Run | None, content: dict[str, Any] | No hide 判一个绑定值是不是这次运行查到过的遮罩值。只读，不发起。""" from app.api.deps import can_read_run from app.api.settings import ledger_settings - from app.apps import versions as app_versions - from app.apps.backfill import graph_side_effect - from app.apps.forms import input_schema, validate_form - from app.apps.resolve import input_key from app.ledger import match as ledger_match - from app.ledger.derive import entry_fields from app.ledger.labels import caliber_key plan = Plan(run_id=run.id if run else "", workflow_id=workflow_id, version=int(version), @@ -237,43 +232,9 @@ async def plan_issue(session: Any, run: Run | None, content: dict[str, Any] | No plan.binding = dict(whole.get("binding") or {}) if hide is not None and any(hide(v) for v in plan.binding.values()): return plan.refuse(MASKED_BINDING) - if workflow is None: - return plan.refuse(GONE, 404) - row = await app_versions.version_row(session, workflow_id, int(version)) - if row is None: - return plan.refuse(VERSION_GONE.format(name=workflow.name, version=version), 404) - if row.retired_at is not None: - return plan.refuse(RETIRED.format(name=workflow.name, version=version)) - if not app_versions.is_published(row, current=workflow.published_version): - return plan.refuse(UNPUBLISHED.format(name=workflow.name, version=version)) - plan.level = row.level or plan.level - plan.expect_hash = row.graph_hash - if reason := await graph_side_effect(session, row.graph, level=row.level): - return plan.refuse(SIDE_EFFECT.format(name=workflow.name, version=version, reason=reason)) - fields = list(entry_fields(row.graph or {})) - values = {str(f["name"]): f.get("default") for f in fields if f.get("default") not in (None, "")} - values.update({k: v for k, v in plan.binding.items() if v not in (None, "")}) - plan.inputs = values - if errors := validate_form(input_schema(fields), values): - return plan.refuse(INPUTS.format(problems="；".join(e["message"] for e in errors))) - # 参数运行时要渲染进口径的 SQL 模板：写在引号里的不能带引号、裸写的只能是数（ledger/compose 同一个判断） - from app.ledger.compose import slot_contexts, value_problem - - contexts = slot_contexts([n for n in (row.graph or {}).get("nodes") or [] if n.get("type") == "tool"], - strict=False) - if problem := next((p for k, v in values.items() if (p := value_problem(k, v, contexts.get(k, set())))), None): - return plan.refuse(f"{problem}，不代填出具") - if principal.mode != "off" and not principal.at_least("builder"): - plan.granted_by_app = await _granting_app(session, principal, workflow_id, int(version)) - if plan.granted_by_app is None: - return plan.refuse(ASK_BUILDER, 403, FORBIDDEN) - key = input_key(values) - plan.existing = next((r.id for r in await issued_runs(session, run.id) - if (r.workflow_id, r.version) == (workflow_id, int(version)) and r.status in _LIVE - and input_key(r.input) == key), None) + if await check_target(session, plan, workflow, principal=principal, source_run=run.id) is None: + return plan settings = await ledger_settings(session) - if plan.existing is None and await issued_today(session) >= int(settings["issue_daily_limit"]): - return plan.refuse(OVER_LIMIT.format(limit=settings["issue_daily_limit"]), 429, DAILY_LIMIT) if not settings["auto_issue"]: plan.auto_reason = AUTO_OFF elif run.origin_kind != "chat": @@ -287,6 +248,68 @@ async def plan_issue(session: Any, run: Run | None, content: dict[str, Any] | No return plan +async def check_target(session: Any, plan: Plan, workflow: Any, *, principal: Any, source_run: str) -> Plan | None: + """按口径出具的那一版能不能发起（plan_issue 和召回待办的「按新版重新出具」共用，ledger/recall.reissue）：那一版已发布、 + 未退役；图里没有人工节点、不会写外部系统；入口字段的缺省值加上 plan.binding 过表单校验、能渲染进取数模板；使用者只能按 + 他有权使用的应用固定着的这一版（拍板 L5）；同一个来源（source_run）、同一版、同一组参数发起过的交回那一次；当天没超 + 上限。填好 plan 的 level、expect_hash、inputs、granted_by_app、existing；不能出具的写进 plan.refuse，交回 None。""" + from app.api.settings import ledger_settings + from app.apps import versions as app_versions + from app.apps.backfill import graph_side_effect + from app.apps.forms import input_schema, validate_form + from app.apps.resolve import input_key + from app.ledger.derive import entry_fields + + workflow_id, version = plan.workflow_id, plan.version + if workflow is None: + plan.refuse(GONE, 404) + return None + row = await app_versions.version_row(session, workflow_id, int(version)) + if row is None: + plan.refuse(VERSION_GONE.format(name=workflow.name, version=version), 404) + return None + if row.retired_at is not None: + plan.refuse(RETIRED.format(name=workflow.name, version=version)) + return None + if not app_versions.is_published(row, current=workflow.published_version): + plan.refuse(UNPUBLISHED.format(name=workflow.name, version=version)) + return None + plan.level = row.level or plan.level + plan.expect_hash = row.graph_hash + if reason := await graph_side_effect(session, row.graph, level=row.level): + plan.refuse(SIDE_EFFECT.format(name=workflow.name, version=version, reason=reason)) + return None + fields = list(entry_fields(row.graph or {})) + values = {str(f["name"]): f.get("default") for f in fields if f.get("default") not in (None, "")} + values.update({k: v for k, v in plan.binding.items() if v not in (None, "")}) + plan.inputs = values + if errors := validate_form(input_schema(fields), values): + plan.refuse(INPUTS.format(problems="；".join(e["message"] for e in errors))) + return None + # 参数运行时要渲染进口径的 SQL 模板：写在引号里的不能带引号、裸写的只能是数（ledger/compose 同一个判断） + from app.ledger.compose import slot_contexts, value_problem + + contexts = slot_contexts([n for n in (row.graph or {}).get("nodes") or [] if n.get("type") == "tool"], + strict=False) + if problem := next((p for k, v in values.items() if (p := value_problem(k, v, contexts.get(k, set())))), None): + plan.refuse(f"{problem}，不代填出具") + return None + if principal.mode != "off" and not principal.at_least("builder"): + plan.granted_by_app = await _granting_app(session, principal, workflow_id, int(version)) + if plan.granted_by_app is None: + plan.refuse(ASK_BUILDER, 403, FORBIDDEN) + return None + key = input_key(values) + plan.existing = next((r.id for r in await issued_runs(session, source_run) + if (r.workflow_id, r.version) == (workflow_id, int(version)) and r.status in _LIVE + and input_key(r.input) == key), None) + settings = await ledger_settings(session) + if plan.existing is None and await issued_today(session) >= int(settings["issue_daily_limit"]): + plan.refuse(OVER_LIMIT.format(limit=settings["issue_daily_limit"]), 429, DAILY_LIMIT) + return None + return plan + + async def _conversation_of(session: Any, run: Run) -> dict[str, str]: """这次运行出自哪段对话的哪一轮：运行来源里封存的（origin_ref，第 2 波）；老运行退回 conversation_turns.run_id。""" from app.db.models import ConversationTurn @@ -392,4 +415,4 @@ async def auto_issue(run_id: str) -> str | None: __all__ = ["AUTO_NOT_CHAT", "DAILY_LIMIT", "FORBIDDEN", "ORIGIN", "REFUSED", "IssueRefused", "Plan", "auto_issue", - "issue", "issued_runs", "issued_today", "issued_view", "plan_issue", "plans_for"] + "check_target", "issue", "issued_runs", "issued_today", "issued_view", "plan_issue", "plans_for"] diff --git a/backend/app/ledger/recall.py b/backend/app/ledger/recall.py new file mode 100644 index 00000000..a9e736a0 --- /dev/null +++ b/backend/app/ledger/recall.py @@ -0,0 +1,1463 @@ +"""口径召回：口径升版、目录事实被驳回或修改、表结构变动之后，追回已经交付的答案（docs/design/next-round/02-口径账本.md +做法 7，拍板 L6、L7；总方案 X1、X3、X6）。 + +**一、口径升版。** 工作流 W 发布了新版本 N（api/workflows.commit_publish 提交之后在后台排队，失败不影响发布）： + +- 反查（ledger/uses）：W 的旧版本 V < N 被哪些问数回答、正式运行、固定了它的工作流、定时任务、应用、导出用过。旧版本里 + 没有源头口径卡（metrics 节点、不是转抄别处的 caliber_from）的不是口径，不召回。 +- 分组：按（旧版本、新版本、input_key(参数)）去重成组。参数先按旧版本入口字段整理（只留入口有的字段、空的补旧版的缺省值）， + 这样问数回答绑定出的 {start, end} 和定时任务的 {start, end, store: all} 是同一组。组按最近一次用到的先后排。 +- 每组一次召回运行（来源 recall，探索运行，系统派生的运行不进使用统计）：派生图经 ledger/derive.recall_graph（全站唯一的 + 派生图生成器）——入口（这一组的参数）→ 旧版取数 + 旧版口径卡、新版取数 + 新版口径卡（caliber_from.nodes 改名，不撞 id） + → 召回卡（recall_card：每个两版都有的指标三个数：旧版、新版、两版之差；差值在卡里算，叙述层无算术）→ 出口（契约 + metrics_from 指向召回卡；不加报告、不调模型）。两版在同一次运行里查同一份数据，差值只反映口径；数据随时间的变化另外拿 + 原答案当时的值对照着说（task_payload 的 original）。 +- 不能重算的照实记「无法重算：原因」，不发起：口径的数据部分不能复用（derive.NotDerivable）、新版里没有这张口径卡、两版 + 没有同名的指标、新版入口要的参数这一组里没有或不合规（compose.check_inputs 同一套校验）、新版的图里有人工节点或会写外部 + 系统（apps/backfill.graph_side_effect，和按新口径补算、按口径出具同一个判断：替人重跑不代人做这些事）。 +- 上限：每次发布最多自动重算 ledger.recall_limit 组（设置，缺省 50；0 表示不自动重算，只列出受影响的答案）。超出的写 + 「另有 M 组未重算」，在版本历史里点「接着算」手动接着发起下一批（每批同样的组数，上限是 0 时按缺省值）。 +- **计划从已有的召回运行重算**（照 apps/backfill 的写法，不建任务表）：一组算没算过，看有没有 origin 指向（W、V、N、 + 参数指纹）的召回运行；不能重算的每次按同样的规则重新判出来。幂等：重复排队、服务重启之后重算一遍计划，已经算过的跳过。 + 启动时（resume）把最近 RESUME_DAYS 天发布、自动那一批没发完的接着发完，结束了还没收尾的召回运行补上收尾。同一个（W、N） + 的计划和发起在进程内串行（单进程部署，同 apps/backfill 的锁）。 +- **收尾**（settle，召回运行结束时 runner 调）：读召回卡（封存范围内的指标集），每个指标按它的单位、小数位渲染的旧值和新值 + 不同就是数变了。数变了：给这一组答案的主人开 recall 待办（挂在召回运行上；开登录时每个主人一条，只列他的答案；关登录时 + 一条派给任何人），推 ledger.recall 通知。数没变：只记账——在每个原答案的运行上追加 ledger.recalled（「已按 vN 核对， + 不影响」）。召回运行自己也追加一条 ledger.recalled 记结论（只记变没变、几个答案、几条待办，不记数），它同时是「收过尾」的 + 记号。召回运行没成功的照实记重算失败。 + +**二、目录事实被驳回或修改**（只管影响数字的那一组：有效记录条件、去重、业务日期、关联、业务主键、度量类型，同 +data/catalog_review.FIELD_GROUP 的 facts）：审阅写入之后（data/catalog 的审阅、人工编辑、保存提案，catalog_review 的批量审阅） +在后台找出查过这张表、而且查询快照冻结的目录里看得到这一项的探索运行，追加 ledger.basis_changed（kind=catalog），界面标 +「依据有变」；对这些运行执行过的 SQL 按新目录跑一遍 SQL 检查（data/sqlcheck，纯计算），比冻结目录下多出错误级问题的才开 +一条 recall 待办（reason=catalog，只有「知道了」）通知答案的主人（拍板 L7：不自动重问）。 + +**三、表结构变动**：消费 data/source_changes（list_changes(since_seq) 按 seq 游标拉，游标记在设置表里；探查、剖析任务结束、 +处理变化改了目录时触发）。只看 triggers_recall 认的：表或列被删、列的类型变了标「依据的表结构已变」（kind=structure）； +触及有效记录条件的新码值（values）、处理时改了目录的（catalog_edit）标依据有变，同二按新目录跑 SQL 检查。表结构变了的， +受影响的口径（最新的已发布版本里查这张表的口径）按最近一次正式运行的输入派生数据部分试算一次（召回运行，origin.probe； +这一版还没有正式运行的退回最近一次等价于它的问数回答的参数，再没有用入口的缺省值；参数凑不齐的不试算），算不出来才开 +recall 待办（reason=structure）通知发布人。 + +**通知**：ledger.recall 只推口径升版、数真变了的那一种待办（消息照 notify/service 的规矩：标题、状态、条数、链接，不放 +新旧值和差值）；目录和表结构的两种待办照别的待办推 task.opened。 + +**自动的那几样**（发布后自动那一批、探查剖析之后拉表结构变化、启动时接着做）和调度循环同一个开关（automatic， +AGENTLAB_SCHEDULER）：它们是在后台替人查业务库，测试、拷来的数据目录（检查脚本）不做；关着时版本历史里照样列出受影响的 +答案、能手动接着算，审阅之后标依据有变照做（只读应用库）。 + +只读：召回运行带 READ_ONLY，只查库、不写；不调模型。召回是辅助步骤：出错只记日志，不影响发布、审阅和探查。 +""" +from __future__ import annotations + +import asyncio +import logging +import weakref +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone +from typing import Any + +from sqlalchemy import func, select + +from app.core.events import EventType +from app.db.base import SessionLocal, utcnow +from app.db.models import Run, RunEvent, Workflow +from app.ledger import uses as ledger_uses + +logger = logging.getLogger(__name__) + +ORIGIN = "recall" +#: 召回待办的种类（process/tasks.RECALL_KIND）和三种来由 +TASK_KIND = "recall" +REASONS = ("caliber", "catalog", "structure") +#: 发起的批次：发布后自动的、版本历史里手动接着算的 +BATCHES = ("auto", "manual") +#: 一组的结论 +OUTCOMES = ("changed", "unchanged", "failed", "running", "pending", "not_recomputable") +#: 启动时接着做最近几天发布的 +RESUME_DAYS = 7 +#: 目录、表结构变了时最多看多少次探索运行（新的在前） +BASIS_MAX_RUNS = 200 +#: 设置表里记表结构变化游标的键（通知自己记的账同 notify_state:* 的写法，设置接口不列） +CURSOR_KEY = "ledger_state:recall_changes" +#: 被拒的机器码 +REFUSED = "ledger_recall_refused" + +#: 不能重算的原因 +NOT_CALIBER = "这一版里没有源头口径卡，不是口径，不召回" +NOT_REUSABLE = "口径的数据部分不能复用：{reason}" +CARD_GONE = "新版 v{version} 里没有口径卡「{card}」" +NO_COMMON = "新旧两版的口径卡没有同名的指标，比不了" +SIDE_EFFECT = "「{name}」v{version}{reason}，召回不代人做这些事" +NEW_INPUTS = "新版 v{version} 的入口要的参数这一组里没有或不合规：{problem}" +NEW_GONE = "新版本不存在，可能已被删除" +LAUNCH_REFUSED = "没能发起召回运行：{reason}" +RUN_FAILED = "重算失败：{reason}" +#: 收尾 +CHANGED_TITLE = "口径「{name}」升到 v{version}，答案的数变了" +CATALOG_TITLE = "答案依据的数据目录有变：按新目录检查出了新的问题" +STRUCTURE_TITLE = "口径「{name}」v{version} 依据的表结构变了，试算没有算出来" +#: 依据有变的四种（ledger.basis_changed 的 kind） +BASIS_KINDS = ("catalog", "structure", "values", "catalog_edit") +#: 目录里影响数字的那一组（同 data/catalog_review.FIELD_GROUP 的 facts） +FACT_SLOTS = (("t", "valid_filter"), ("t", "dedup"), ("t", "business_date"), ("t", "keys"), ("c", "measure")) + + +class RecallRefused(Exception): + def __init__(self, status_code: int, message: str, code: str = REFUSED) -> None: + super().__init__(message) + self.status_code = status_code + self.message = message + self.code = code + + +# ========================================================================== +# 计划 +# ========================================================================== + + +@dataclass +class Pair: + """一对口径卡：旧版那一张、新版同 id 的那一张。""" + + old: Any + new: Any + + +@dataclass +class Group: + """（旧版本、新版本、参数）一组：这一组的用法、能不能重算、算过没有。""" + + old_version: int + new_version: int + key: str + binding: dict[str, Any] + uses: list[ledger_uses.Use] = field(default_factory=list) + latest: datetime | None = None + problem: str | None = None + #: 发起召回运行用的输入（按新版入口校验过、补了缺省值；旧版独有的字段接在后面） + inputs: dict[str, Any] = field(default_factory=dict) + run: Run | None = None + refused: str | None = None + + @property + def id(self) -> str: + return f"{self.old_version}-{self.key[:16]}" + + +@dataclass +class Plan: + workflow_id: str + workflow_name: str + new_version: int + groups: list[Group] = field(default_factory=list) + #: 每个旧版本的口径卡对、旧版本自己的问题（不是口径、数据部分不能复用……） + pairs: dict[int, list[Pair]] = field(default_factory=dict) + version_problem: dict[int, str] = field(default_factory=dict) + found: ledger_uses.Found | None = None + problem: str | None = None + #: 自动那一批已经发起了几组 + auto_launched: int = 0 + + def todo(self) -> list[Group]: + """还没算、能算的组（按最近一次用到的先后）。""" + return [g for g in self.groups if g.run is None and g.problem is None] + + +def _aware(moment: datetime | None) -> datetime | None: + if moment is None: + return None + return moment if moment.tzinfo else moment.replace(tzinfo=timezone.utc) + + +def normalize(binding: Mapping[str, Any], fields: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + """参数按旧版入口整理：只留入口有的字段，没给（或给了空）的补旧版的缺省值。分组、发起都按它。""" + out: dict[str, Any] = {} + for f in fields: + name = str(f.get("name") or "") + if not name: + continue + value = binding.get(name) + if value in (None, ""): + value = f.get("default") + if value not in (None, ""): + out[name] = value + return out + + +def source_cards(graph: Any) -> list[str]: + """一版图里的源头口径卡（metrics 节点、不是转抄别处的 caliber_from），按图里的先后。""" + out: list[str] = [] + for node in (graph or {}).get("nodes") or [] if isinstance(graph, dict) else []: + if not isinstance(node, dict) or node.get("type") != "metrics": + continue + cfg = (node.get("data") or {}).get("config") if isinstance(node.get("data"), dict) else None + if isinstance(cfg, dict) and cfg.get("caliber_from") in (None, "", {}): + out.append(str(node.get("id"))) + return out + + +async def recall_runs(session: Any, workflow_id: str, new_version: int | None = None) -> list[Run]: + """以 W（新版本 N）发起过的召回运行（试算的不算），新的在前。origin_kind 有索引，先按它筛再看 origin。""" + stmt = select(Run).where(Run.origin_kind == ORIGIN, + func.json_extract(Run.origin, "$.workflow_id") == workflow_id, + func.coalesce(func.json_extract(Run.origin, "$.probe"), 0) == 0) + if new_version is not None: + stmt = stmt.where(func.json_extract(Run.origin, "$.to_version") == int(new_version)) + return list((await session.execute(stmt.order_by(Run.created_at.desc()))).scalars()) + + +async def build_plan(session: Any, workflow_id: str, new_version: int) -> Plan: + """W 升到 N 的召回计划（见模块说明）。只读，不发起。""" + from app.apps import versions as app_versions + from app.apps.backfill import graph_side_effect + from app.ledger import compose, derive + + workflow = await session.get(Workflow, workflow_id) + plan = Plan(workflow_id=workflow_id, workflow_name=workflow.name if workflow is not None else "", + new_version=int(new_version)) + new_row = await app_versions.version_row(session, workflow_id, int(new_version)) if workflow else None + if new_row is None: + plan.problem = NEW_GONE + return plan + found = await ledger_uses.find_uses(session, workflow_id, below=int(new_version)) + plan.found = found + side = await graph_side_effect(session, new_row.graph, level=new_row.level) + for version in found.versions(): + old_row = await app_versions.version_row(session, workflow_id, version) + cards = source_cards(old_row.graph if old_row is not None else None) + if not cards: + plan.version_problem[version] = NOT_CALIBER + continue + pairs: list[Pair] = [] + problems: list[str] = [] + for card in cards: + try: + old = await derive.load_pinned(session, workflow_id, version, card) + except derive.NotDerivable as e: + problems.append(NOT_REUSABLE.format(reason=e.message)) + continue + try: + new = await derive.load_pinned(session, workflow_id, int(new_version), card) + except derive.NotDerivable as e: + problems.append(CARD_GONE.format(version=new_version, card=old.card_title) + if e.code == derive.GONE else NOT_REUSABLE.format(reason=e.message)) + continue + if not {m["id"] for m in old.metrics} & {m["id"] for m in new.metrics}: + problems.append(NO_COMMON) + continue + pairs.append(Pair(old, new)) + if side: + plan.version_problem[version] = SIDE_EFFECT.format(name=plan.workflow_name, version=new_version, + reason=side) + elif not pairs: + plan.version_problem[version] = problems[0] if problems else NOT_CALIBER + plan.pairs[version] = pairs + groups: dict[tuple[int, str], Group] = {} + for use in found.uses: + if plan.version_problem.get(use.version) == NOT_CALIBER: + continue + pairs = plan.pairs.get(use.version) or [] + fields = pairs[0].old.entry_fields if pairs else () + binding = normalize(use.binding, fields) if pairs else dict(use.binding) + key = ledger_uses._key(binding) + group = groups.get((use.version, key)) + if group is None: + group = groups[(use.version, key)] = Group(old_version=use.version, new_version=int(new_version), key=key, + binding=binding, problem=plan.version_problem.get(use.version)) + group.uses.append(use) + at = _aware(use.at) + if at is not None and (group.latest is None or at > group.latest): + group.latest = at + for group in groups.values(): + pairs = plan.pairs.get(group.old_version) or [] + if group.problem is not None or not pairs: + continue + new = pairs[0].new + new_names = {str(f.get("name")) for f in new.entry_fields} + try: + checked = compose.check_inputs(new, {k: v for k, v in group.binding.items() if k in new_names}) + except compose.ComposeRefused as e: + group.problem = NEW_INPUTS.format(version=new_version, problem=e.message) + continue + group.inputs = {**checked, **{k: v for k, v in group.binding.items() if k not in new_names}} + existing = await recall_runs(session, workflow_id, int(new_version)) + by_key: dict[tuple[int, str], Run] = {} + for run in existing: + origin = run.origin if isinstance(run.origin, dict) else {} + k = (int(origin.get("from_version") or 0), str(origin.get("input_key") or "")) + by_key.setdefault(k, run) + if origin.get("batch") == "auto": + plan.auto_launched += 1 + for k, group in groups.items(): + group.run = by_key.get(k) + group.refused = _refusals.get((workflow_id, int(new_version), k[0], k[1])) + far = datetime.min.replace(tzinfo=timezone.utc) + plan.groups = sorted(groups.values(), key=lambda g: (-(g.latest or far).timestamp(), g.old_version, g.key)) + return plan + + +#: 发起时被拒的组（launch_run 拒绝了：工具不在、数据源停用……）：(W, N, V, 参数指纹) → 原因。只在进程里记（下次「接着算」 +#: 重试；重启后忘掉、照样算待算） +_refusals: dict[tuple[str, int, int, str], str] = {} + + +# ========================================================================== +# 召回卡与发起 +# ========================================================================== + + +def _metric(mid: str, name: str, expression: str, src: Mapping[str, Any]) -> dict[str, Any]: + out: dict[str, Any] = {"id": mid, "name": name, "unit": str(src.get("unit") or ""), "expression": expression} + if src.get("decimals") is not None: + out["decimals"] = src["decimals"] + if src.get("format"): + out["format"] = src["format"] + return out + + +def recall_card(pairs: Sequence[Pair]) -> tuple[dict[str, Any], list[dict[str, Any]]]: + """召回卡（derive.recall_graph 的 tail）和每个比较的说明。每个两版都有（按指标 id）的指标三个数：old_k 旧版、new_k + 新版、diff_k 两版之差（新版 − 旧版），单位、小数位、显示格式照新版的定义。缺输入记空值（on_missing=null）。纯计算。""" + from app.ledger import derive + + metrics: list[dict[str, Any]] = [] + meta: list[dict[str, Any]] = [] + k = 0 + for i, pair in enumerate(pairs): + old_prefix, new_prefix = derive.recall_prefixes(i) + new_by = {m["id"]: (j, m) for j, m in enumerate(pair.new.metrics)} + for idx, m in enumerate(pair.old.metrics): + if m["id"] not in new_by: + continue + j, nm = new_by[m["id"]] + old_expr = f"nodes.{old_prefix}{pair.old.node_id}.metrics[{idx}].value" + new_expr = f"nodes.{new_prefix}{pair.new.node_id}.metrics[{j}].value" + name = str(nm.get("name") or m["id"]) + metrics += [_metric(f"old_{k}", f"{name} · 旧版 v{pair.old.version}", old_expr, nm), + _metric(f"new_{k}", f"{name} · 新版 v{pair.new.version}", new_expr, nm), + _metric(f"diff_{k}", f"{name} · 两版之差", f"({new_expr}) - ({old_expr})", nm)] + # 旧版的式子（定义，不是数）一并记下：问数回答当时的值要按它找到答案里对应的那一格（original_values） + meta.append({"k": k, "card": pair.old.node_id, "metric": m["id"], "name": name, + "unit": str(nm.get("unit") or ""), "decimals": nm.get("decimals"), "format": nm.get("format"), + "expression": str(m.get("expression") or "")}) + k += 1 + node = derive._node(derive.RECALL_CARD, "metrics", "召回卡", caliber="口径召回", caliber_version="v1", + on_missing="null", metrics=metrics) + return node, meta + + +def build_graph(group: Group, pairs: Sequence[Pair]) -> tuple[Any, list[dict[str, Any]]]: + """一组的召回图（derive.recall_graph）和比较说明。同样的输入永远得到同一张图。""" + from app.ledger import derive + + card, meta = recall_card(pairs) + graph = derive.recall_graph([(p.old, p.new) for p in pairs], [card], inputs=group.inputs) + return graph, meta + + +async def _launch_group(plan: Plan, group: Group, *, batch: str, actor: str | None) -> Run | None: + from app.engine.launch import LaunchRefused, launch_run + from app.tools.datasource import READ_ONLY_OPTION + + pairs = plan.pairs.get(group.old_version) or [] + graph, meta = build_graph(group, pairs) + origin = {"workflow_id": plan.workflow_id, "from_version": group.old_version, "to_version": group.new_version, + "input_key": group.key, "binding": dict(group.binding), "batch": batch, + "cards": [p.old.node_id for p in pairs], "metrics": meta, + "uses": [u.describe() for u in group.uses], "derive": graph.origin} + key = (plan.workflow_id, plan.new_version, group.old_version, group.key) + async with SessionLocal() as session: + try: + run = await launch_run(session, graph=graph.graph, input_payload=dict(group.inputs), + run_class="exploratory", actor=actor, origin_kind=ORIGIN, origin=origin, + options={READ_ONLY_OPTION: True}, + workflow_name=f"口径召回：{plan.workflow_name} v{group.old_version} → v{group.new_version}") + except LaunchRefused as e: + _refusals[key] = LAUNCH_REFUSED.format(reason=e.message) + logger.info("口径召回没能发起（%s v%s → v%s）：%s", plan.workflow_id, group.old_version, group.new_version, + e.message) + return None + _refusals.pop(key, None) + return run + + +#: 同一（W、N）的计划和发起串行：{事件循环: {键: 锁}} +_locks: "weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, dict[tuple[str, int], asyncio.Lock]]" = \ + weakref.WeakKeyDictionary() + + +def _lock(key: tuple[str, int]) -> asyncio.Lock: + group = _locks.setdefault(asyncio.get_running_loop(), {}) + return group.setdefault(key, asyncio.Lock()) + + +async def launch(workflow_id: str, new_version: int, *, batch: str = "auto", actor: str | None = None) -> list[str]: + """按计划发起一批召回运行，交回发起了的运行 id。auto：发布后自动的那一批，一次发布最多 recall_limit 组（已经自动发起 + 过的扣掉，重复调幂等）；manual：「接着算」，再发起一批（上限是 0 时按缺省的组数）。""" + from app.api.settings import LEDGER_DEFAULTS, ledger_settings + + if batch not in BATCHES: + raise ValueError(f"不认识的召回批次：{batch}") + async with _lock((workflow_id, int(new_version))): + async with SessionLocal() as session: + plan = await build_plan(session, workflow_id, int(new_version)) + limit = int((await ledger_settings(session))["recall_limit"]) + if plan.problem is not None: + return [] + if batch == "auto": + size = max(0, limit - plan.auto_launched) + else: + size = limit or int(LEDGER_DEFAULTS["recall_limit"]) + launched: list[str] = [] + for group in plan.todo()[:size]: + run = await _launch_group(plan, group, batch=batch, actor=actor) + if run is not None: + launched.append(run.id) + if launched: + logger.info("口径召回：%s 升到 v%s，%s 发起了 %d 组", workflow_id, new_version, batch, len(launched)) + return launched + + +# ========================================================================== +# 预告、批次、版本用量 +# ========================================================================== + + +def _counts(plan: Plan) -> dict[str, int]: + out = dict.fromkeys(OUTCOMES, 0) + for group in plan.groups: + out[_group_state(group)[0]] += 1 + return out + + +async def preview(session: Any, workflow_id: str, version: int) -> dict[str, Any]: + """发布弹窗的召回预告：把 version 发布成新版本之后，旧版本被多少问数回答、导出、定时任务、应用用过，预计重算几组，超出 + 上限的写明。只读、不发起（和发布之后真的召回走同一份计划，规模一致）。""" + from app.api.settings import ledger_settings + + plan = await build_plan(session, workflow_id, int(version)) + limit = int((await ledger_settings(session))["recall_limit"]) + found = plan.found or ledger_uses.Found(uses=[]) + per_version = [] + for v in sorted({*found.versions(), *plan.version_problem}): + if plan.version_problem.get(v) == NOT_CALIBER: + continue + sub = ledger_uses.Found(uses=[u for u in found.uses if u.version == v], apps={v: found.apps.get(v, [])}) + counts = ledger_uses.summarize(sub) + counts["exports"] = await ledger_uses.exports_count(session, [u.run_id for u in sub.uses]) + per_version.append({"version": v, **counts, **({"problem": plan.version_problem[v]} + if v in plan.version_problem else {})}) + todo = len(plan.todo()) + done = sum(1 for g in plan.groups if g.run is not None) + auto_left = max(0, limit - plan.auto_launched) + will = min(todo, auto_left) + return { + "workflow_id": workflow_id, "version": int(version), "limit": limit, "caliber": bool(plan.pairs), + "problem": plan.problem, "versions": per_version, "groups": len(plan.groups), + "recomputable": todo + done, "not_recomputable": sum(1 for g in plan.groups if g.problem is not None), + "already": done, "will_recompute": will, "over_limit": max(0, todo - will), + "truncated": found.truncated, + "reasons": _reasons(plan), + } + + +def _reasons(plan: Plan) -> list[dict[str, Any]]: + """不能重算的原因各几组（同一句话合在一起）。""" + by: dict[str, int] = {} + for group in plan.groups: + if group.problem: + by[group.problem] = by.get(group.problem, 0) + 1 + return [{"reason": r, "groups": n} for r, n in sorted(by.items(), key=lambda x: (-x[1], x[0]))] + + +async def outcome_of(session: Any, run: Run) -> tuple[str, dict[str, Any]]: + """一次召回运行的结论：(changed / unchanged / failed / running, 收尾时记下的那一条 ledger.recalled 的内容)。""" + data = await _settled(session, run.id) + if data is not None: + if data.get("outcome") == "failed": + return "failed", data + return ("changed" if data.get("changed") else "unchanged"), data + if run.status in ("failed", "cancelled"): + return "failed", {"error": run.error} + return "running", {} + + +def _group_state(group: Group) -> tuple[str, str | None]: + """没读收尾记录之前的状态：不能重算、待算、发起被拒（照样算待算，原因另写）、算过（具体结论看 outcome_of）。""" + if group.run is not None: + return "running", None + if group.problem is not None: + return "not_recomputable", group.problem + return "pending", group.refused + + +async def batches(session: Any, workflow_id: str, version: int, *, principal: Any = None) -> dict[str, Any]: + """版本历史里「这一版的召回」：每组的结论（数变了 / 没变 / 无法重算：原因 / 重算失败 / 还在算 / 未重算），「另有 M 组 + 未重算」和能不能接着算。参数是遮过的（不给使用者看遮罩值：这个接口只给搭建者，还是照规矩遮）。""" + from app.api.settings import LEDGER_DEFAULTS, ledger_settings + + plan = await build_plan(session, workflow_id, int(version)) + limit = int((await ledger_settings(session))["recall_limit"]) + counts = dict.fromkeys(OUTCOMES, 0) + items: list[dict[str, Any]] = [] + for group in plan.groups: + state, reason = _group_state(group) + extra: dict[str, Any] = {} + if group.run is not None: + state, data = await outcome_of(session, group.run) + if state == "failed": + reason = RUN_FAILED.format(reason=str(data.get("error") or group.run.error or "运行没有成功结束")[:200]) + extra = {"run_id": group.run.id, "batch": (group.run.origin or {}).get("batch"), + "tasks": len(data.get("tasks") or []) if data else 0, + "changed_metrics": [m.get("name") for m in data.get("metrics") or [] if m.get("changed")] + if data else []} + counts[state] += 1 + kinds: dict[str, int] = {} + for use in group.uses: + kinds[use.kind] = kinds.get(use.kind, 0) + 1 + items.append({"id": group.id, "from_version": group.old_version, "to_version": group.new_version, + "binding": [{"field": k, "value": v} for k, v in sorted(group.binding.items())], + "uses": [{"kind": k, "kind_text": ledger_uses.KIND_TEXT.get(k, k), "count": n} + for k, n in sorted(kinds.items())], + "latest": group.latest.isoformat() if group.latest else None, + "state": state, **({"reason": reason} if reason else {}), **extra}) + pending = counts["pending"] + return {"workflow_id": workflow_id, "workflow_name": plan.workflow_name, "version": int(version), "limit": limit, + "batch_size": limit or int(LEDGER_DEFAULTS["recall_limit"]), "problem": plan.problem, + "caliber": bool(plan.pairs) or bool(plan.groups), "counts": counts, "groups": items, + "remaining": pending, "can_continue": pending > 0 and plan.problem is None, + "truncated": bool(plan.found and plan.found.truncated)} + + +async def version_recall_counts(session: Any, workflow_id: str, version: int) -> dict[str, Any]: + """版本用量里的召回：以这一版为新版本召回过几组（其中数变了几组），这一版作为旧版本被召回过几组。""" + as_new = await recall_runs(session, workflow_id, version) + changed = 0 + for run in as_new: + state, _ = await outcome_of(session, run) + changed += state == "changed" + as_old = int(await session.scalar(select(func.count()).select_from(Run).where( + Run.origin_kind == ORIGIN, func.json_extract(Run.origin, "$.workflow_id") == workflow_id, + func.json_extract(Run.origin, "$.from_version") == int(version), + func.coalesce(func.json_extract(Run.origin, "$.probe"), 0) == 0)) or 0) + return {"groups": len(as_new), "changed": changed, "recalled_from": as_old} + + +# ========================================================================== +# 收尾：数变没变、开待办或记账 +# ========================================================================== + + +async def _settled(session: Any, run_id: str) -> dict[str, Any] | None: + """召回运行上收尾时记的 ledger.recalled（role=recall）；没收过尾为 None。""" + rows = (await session.execute(select(RunEvent.data).where( + RunEvent.run_id == run_id, RunEvent.type == EventType.LEDGER_RECALLED).order_by(RunEvent.seq.desc()))).scalars() + for data in rows: + if isinstance(data, dict) and data.get("role") == "recall": + return data + return None + + +async def card_values(run_id: str) -> dict[str, dict[str, Any]]: + """召回运行封存范围内的召回卡：{指标 id: 指标}（value、rendered……）。读不出为空。""" + from app.api.evidence import _sealed, _sealed_cards + from app.ledger import derive + + sealed = await _sealed(run_id) + for node, _, card in _sealed_cards(sealed): + if node == derive.RECALL_CARD: + return {str(m.get("id")): m for m in card.get("metrics") or [] if isinstance(m, dict)} + return {} + + +def compare(meta: Sequence[Mapping[str, Any]], values: Mapping[str, Mapping[str, Any]]) -> list[dict[str, Any]]: + """每个比较的旧值、新值、两版之差，和数变没变：按指标的单位、小数位渲染的旧值和新值不同就是变了（召回卡上的渲染， + 同报告里显示的那个字）。只读召回卡，不在这里算差值。""" + out = [] + for item in meta: + k = item.get("k") + old, new, diff = (values.get(f"{p}_{k}") or {} for p in ("old", "new", "diff")) + changed = str(old.get("rendered")) != str(new.get("rendered")) + out.append({"k": k, "card": item.get("card"), "metric": item.get("metric"), "name": item.get("name"), + "unit": item.get("unit"), "old": old.get("value"), "new": new.get("value"), + "diff": diff.get("value"), "old_text": old.get("rendered"), "new_text": new.get("rendered"), + "diff_text": diff.get("rendered"), "changed": changed}) + return out + + +_settling: set[str] = set() +_background: set[asyncio.Task[Any]] = set() + + +def spawn(coro: Any) -> None: + """在后台做（召回是辅助步骤：出错只记日志）。""" + task = asyncio.create_task(_guarded(coro)) + _background.add(task) + task.add_done_callback(_background.discard) + + +async def _guarded(coro: Any) -> None: + try: + await coro + except asyncio.CancelledError: + raise + except Exception: # noqa: BLE001 - 辅助步骤 + logger.exception("口径召回的后台任务出错") + + +def schedule_settle(run_id: str) -> None: + """召回运行结束了（runner 收尾时调）：在后台收尾。""" + spawn(settle(run_id)) + + +async def _mark(run_id: str, data: dict[str, Any]) -> bool: + from app.engine.run_append import append_events + + return await append_events(run_id, [(EventType.LEDGER_RECALLED, None, data)], require="sealed") is not None + + +async def settle(run_id: str) -> str | None: + """一次召回运行结束之后：数变了开 recall 待办、数没变在原答案上记 ledger.recalled，最后在召回运行上记结论。返回结论 + （changed / unchanged / failed），不是召回运行、还没结束、收过尾的为 None。幂等：待办按 correlation_key 认、原答案上的 + 记账按召回运行认，中途断了再收一次不会多开。""" + if run_id in _settling: + return None + _settling.add(run_id) + try: + async with SessionLocal() as session: + run = await session.get(Run, run_id) + if run is None or run.origin_kind != ORIGIN or run.status not in ("succeeded", "failed", "cancelled") \ + or not run.manifest_hash: + return None + if await _settled(session, run_id) is not None: + return None + origin = run.origin if isinstance(run.origin, dict) else {} + if origin.get("probe"): + return await _settle_probe(run) + base = {"role": "recall", "workflow_id": origin.get("workflow_id"), "from_version": origin.get("from_version"), + "to_version": origin.get("to_version")} + if run.status != "succeeded": + await _mark(run_id, {**base, "outcome": "failed", "changed": None, + "error": str(run.error or "运行没有成功结束")[:300]}) + return "failed" + results = compare(origin.get("metrics") or [], await card_values(run_id)) + changed = any(r["changed"] for r in results) + answers = list(dict.fromkeys(str(u.get("run_id")) for u in origin.get("uses") or [] if u.get("run_id"))) + tasks: list[str] = [] + if changed: + tasks = await open_tasks(run, reason="caliber") + else: + for answer in answers: + await _mark_answer(answer, run) + await _mark(run_id, {**base, "outcome": "changed" if changed else "unchanged", "changed": changed, + "answers": len(answers), "tasks": tasks, + "metrics": [{"k": r["k"], "name": r["name"], "changed": r["changed"]} for r in results]}) + return "changed" if changed else "unchanged" + finally: + _settling.discard(run_id) + + +async def _mark_answer(answer_id: str, recall_run: Run) -> None: + """数没变：在原答案的运行上记一条 ledger.recalled（已按 vN 核对，不影响）。这次召回记过的不再记。""" + origin = recall_run.origin if isinstance(recall_run.origin, dict) else {} + async with SessionLocal() as session: + rows = (await session.execute(select(RunEvent.data).where( + RunEvent.run_id == answer_id, RunEvent.type == EventType.LEDGER_RECALLED))).scalars() + if any(isinstance(d, dict) and d.get("recall_run") == recall_run.id for d in rows): + return + await _mark(answer_id, {"role": "answer", "changed": False, "recall_run": recall_run.id, + "workflow_id": origin.get("workflow_id"), "from_version": origin.get("from_version"), + "to_version": origin.get("to_version")}) + + +# ========================================================================== +# 待办 +# ========================================================================== + + +async def _owner_targets(session: Any, owners: Iterable[tuple[str, str]]) -> list[tuple[str, str]]: + """答案的主人 → 待办派给谁：开登录时每个主人一份（组照留，开待办时展开；服务账号不派），关登录时派给任何人。""" + from app.auth import current_mode + from app.process.tasks import _service_accounts + + if current_mode() == "off": + return [("anyone", "")] + owners = list(dict.fromkeys((k, n) for k, n in owners if k in ("user", "group") and n)) + service = await _service_accounts(session, [n for k, n in owners if k == "user"]) + out = [(k, n) for k, n in owners if not (k == "user" and n in service)] + return out or [("anyone", "")] + + +def _uses_of(origin: Mapping[str, Any], owner: tuple[str, str]) -> list[dict[str, Any]]: + """这条待办列哪几个答案：派给任何人的列全部，派给某人（组）的只列他是主人的。""" + items = [u for u in origin.get("uses") or [] if isinstance(u, dict)] + if owner[0] == "anyone": + return items + return [u for u in items if [owner[0], owner[1]] in (u.get("owners") or [])] + + +def correlation(run_id: str, owner: tuple[str, str], reason: str = "caliber", extra: str = "") -> str: + """recall 待办的 correlation_key：来由:运行[:附加]:派给的种类:名字。同一条按它认，最后两段是派给谁（_owner_of_key）。""" + return ":".join([reason, run_id, *([extra] if extra else []), owner[0], owner[1]]) + + +async def open_tasks(run: Run, *, reason: str, owners: Iterable[tuple[str, str]] | None = None, + title: str | None = None, extra: str = "") -> list[str]: + """给答案的主人开 recall 待办（挂在 run 上）。owners 不给就按召回运行 origin 里各答案的主人。按 correlation_key 认同一条， + 开过的不再开。提交之后推通知。返回这次新开的待办 id。""" + from app.process import tasks as process_tasks + + origin = run.origin if isinstance(run.origin, dict) else {} + async with SessionLocal() as session: + if owners is None: + owners = [tuple(o) for u in origin.get("uses") or [] for o in u.get("owners") or [] if len(o) == 2] + targets = await _owner_targets(session, owners) + name = await ledger_uses.workflow_name(session, str(origin.get("workflow_id") or "")) + if title is None: + title = CHANGED_TITLE.format(name=name or "口径", version=origin.get("to_version")) + opened: list[str] = [] + for owner in targets: + key = correlation(run.id, owner, reason, extra) + if await session.scalar(select(process_tasks.Task.id).where(process_tasks.Task.correlation_key == key)): + continue + task = await process_tasks.open_recall(session, run=run, title=title, owner=owner, correlation_key=key) + opened.append(task.id) + await session.commit() + for task_id in opened: + await process_tasks.announce(task_id) + return opened + + +def _owner_of_key(key: str) -> tuple[str, str]: + """correlation_key 的最后两段是派给谁（见 correlation）。""" + parts = (key or "").split(":") + return (parts[-2], parts[-1]) if len(parts) >= 4 else ("anyone", "") + + +async def task_payload(session: Any, task: Any, principal: Any) -> dict[str, Any]: + """recall 待办给处理人看的内容（待办箱、待办详情的 payload，process/tasks._describe 调）：旧口径的值、新口径的值、两版之差 + 和原答案当时的值（每个都带出处：哪次运行、哪张卡、哪个指标，点得开），这一组的参数，能做的动作。使用者看到的值照样 + 遮罩（按那次运行查到过的遮罩值，同账本接口）。""" + from app.api.evidence import _sealed + from app.ledger.view import _hider + + run = await session.get(Run, task.run_id) + origin = run.origin if run is not None and isinstance(run.origin, dict) else {} + key = task.correlation_key or "" + reason = key.split(":", 1)[0] if key.split(":", 1)[0] in REASONS else "caliber" + base: dict[str, Any] = {"kind": TASK_KIND, "reason": reason, "recall_run": task.run_id, + "workflow_id": origin.get("workflow_id"), "from_version": origin.get("from_version"), + "to_version": origin.get("to_version")} + if origin.get("workflow_id"): + base["workflow_name"] = await ledger_uses.workflow_name(session, str(origin["workflow_id"])) + pending = task.status == "pending" + if reason != "caliber": + return {**base, "basis": await _basis_of(session, task, run, reason), + "actions": [{"decision": "ack", "available": pending}]} + builder = getattr(principal, "mode", "off") == "off" or principal.at_least("builder") + hide = (await _hider(await _sealed(task.run_id))).hide if not builder and run is not None else (lambda _v: False) + values = await card_values(task.run_id) if run is not None else {} + metrics = [] + for item in compare(origin.get("metrics") or [], values): + cells = {} + for side in ("old", "new", "diff"): + value, text = item[side], item[f"{side}_text"] + masked = value is not None and hide(value) + cells[side] = {"value": None if masked else value, "text": "已遮罩" if masked else text, + "metric": f"{side}_{item['k']}", "masked": masked} + metrics.append({"k": item["k"], "name": item["name"], "unit": item["unit"], "changed": item["changed"], + "card": item["card"], "metric": item["metric"], **cells}) + answers = [] + for use in _uses_of(origin, _owner_of_key(key)): + answers.append({**{k: use.get(k) for k in ("kind", "run_id", "version", "conversation_id", "turn_id", + "schedule_id", "app_id", "at", "via_workflow", "via_version") + if use.get(k) is not None}, + "kind_text": ledger_uses.KIND_TEXT.get(str(use.get("kind")), str(use.get("kind"))), + "original": await original_values(use, origin.get("metrics") or [], hide_for=not builder)}) + binding = [{"field": k, "value": None if hide(v) else v, "masked": bool(hide(v))} + for k, v in sorted((origin.get("binding") or {}).items())] + actions = [await reissue_action(session, task, principal, origin), {"decision": "ack", "available": pending}] + return {**base, "binding": binding, "metrics": metrics, "answers": answers, "actions": actions, + "outcome": task.outcome} + + +async def _basis_of(session: Any, task: Any, run: Run | None, reason: str) -> dict[str, Any]: + """目录、表结构两种 recall 待办说的是什么：目录的读答案运行上那一条 ledger.basis_changed（correlation_key 里记着它的序号）， + 表结构的读试算运行的 origin 和它为什么没算出来。""" + if reason == "structure": + origin = run.origin if run is not None and isinstance(run.origin, dict) else {} + return {"workflow_id": origin.get("workflow_id"), "version": origin.get("version"), "table": origin.get("table"), + "column": origin.get("column"), "change_kind": origin.get("change_kind"), "change": origin.get("change"), + "source_id": origin.get("source_id"), "error": (run.error if run is not None else None), + "probe_run": task.run_id} + parts = (task.correlation_key or "").split(":") + seq = int(parts[2]) if len(parts) >= 5 and parts[2].isdigit() else None + data = await session.scalar(select(RunEvent.data).where( + RunEvent.run_id == task.run_id, RunEvent.type == EventType.LEDGER_BASIS_CHANGED, RunEvent.seq == seq)) \ + if seq is not None else None + data = data if isinstance(data, dict) else {} + return {k: data.get(k) for k in ("kind", "source_id", "source", "table", "items", "new_errors", "checks", + "catalog_version") if data.get(k) is not None} + + +async def original_values(use: Mapping[str, Any], meta: Sequence[Mapping[str, Any]], *, + hide_for: bool) -> list[dict[str, Any]]: + """原答案当时的值（每个比较一项）：正式运行读它封存的口径卡（固定了这一版的读转抄来的那张）；问数回答读等价的那条查询 + 快照里那一格（口径卡的指标就是取数第一行某一列时，按列映射找这次的列）。读不出、原答案里没有单独这个数的为空，写为什么。 + hide_for：给使用者看，按那次运行查到过的遮罩值遮。""" + from app.api.evidence import _sealed, _sealed_cards + from app.ledger.view import _hider + + run_id = str(use.get("run_id") or "") + sealed = await _sealed(run_id) + hide = (await _hider(sealed)).hide if hide_for else (lambda _v: False) + out: list[dict[str, Any]] = [] + if use.get("kind") != "answer": + cards = {node: card for node, _, card in _sealed_cards(sealed)} + for item in meta: + card = _card_for(sealed, cards, str(item.get("card"))) + metric = next((m for m in (card or {}).get("metrics") or [] if m.get("id") == item.get("metric")), None) + out.append(_cell(item, metric.get("value") if metric else None, metric.get("rendered") if metric else None, + hide, node=_card_node(sealed, cards, str(item.get("card"))), metric=item.get("metric"), + missing=None if metric else "原答案里没有这个指标")) + return out + import ast + + from app.core.artifact_store import load + from app.engine.evidence import render_number + from app.engine.expressions import cell_parts + from app.ledger import store + + recorded = await store.latest(run_id) + content = recorded.content if recorded is not None else None + query = next((q for q in (content or {}).get("queries") or [] if q.get("artifact") == use.get("query")), None) + cand = next((c for c in (query or {}).get("candidates") or [] if c.get("status") == "proven" + and c.get("node_id") == use.get("node_id")), None) + try: + snap = load(str(use.get("query") or "")) if use.get("query") else None + except ValueError: + snap = None + columns = [str(c) for c in (snap or {}).get("columns") or []] if isinstance(snap, dict) else [] + rows = (snap or {}).get("rows") if isinstance(snap, dict) else None + for item in meta: + value = text = missing = None + expression = str(item.get("expression") or "") + parts = cell_parts(expression.strip()) if expression else None + column = None + if parts is not None and parts[0] == f"nodes.{use.get('node_id')}" and parts[1].strip() == "0": + try: + column = ast.literal_eval(parts[2]) + except (ValueError, SyntaxError): + column = None + ours = next((m.get("ours") for m in (cand or {}).get("columns") or [] if m.get("caliber") == column), None) \ + if isinstance(column, str) else None + if ours in columns and isinstance(rows, list) and rows and isinstance(rows[0], (list, tuple)): + value = rows[0][columns.index(ours)] + try: + text = render_number(value, unit=str(item.get("unit") or ""), decimals=item.get("decimals"), + fmt=item.get("format")) + except Exception: # noqa: BLE001 - 渲染不出就写原值 + text = str(value) + else: + missing = "原答案里没有单独的这个数" + out.append(_cell(item, value, text, hide, query=use.get("query"), column=ours, missing=missing)) + return out + + +def _card_for(sealed: Any, cards: Mapping[str, Any], card_id: str) -> Mapping[str, Any] | None: + node = _card_node(sealed, cards, card_id) + return cards.get(node) if node else None + + +def _card_node(sealed: Any, cards: Mapping[str, Any], card_id: str) -> str | None: + """正式运行里这张口径卡的节点：同 id 的（W 自己的运行），或者 caliber_from 固定到它的那张（固定了这一版的工作流）。""" + if card_id in cards: + return card_id + for node in (sealed.graph or {}).get("nodes") or []: + cfg = (node.get("data") or {}).get("config") if isinstance(node, dict) and isinstance(node.get("data"), dict) \ + else None + ref = cfg.get("caliber_from") if isinstance(cfg, dict) else None + if isinstance(ref, dict) and ref.get("node_id") == card_id and node.get("id") in cards: + return str(node["id"]) + return None + + +def _cell(item: Mapping[str, Any], value: Any, text: Any, hide: Any, *, missing: str | None = None, + **where: Any) -> dict[str, Any]: + masked = value is not None and bool(hide(value)) + out = {"k": item.get("k"), "value": None if masked else value, "text": "已遮罩" if masked else text, + "masked": masked, **{k: v for k, v in where.items() if v is not None}} + if missing: + out["missing"] = missing + return out + + +# ========================================================================== +# 「按新版重新出具」「知道了」 +# ========================================================================== + + +async def _reissue_plan(session: Any, task: Any, principal: Any, origin: Mapping[str, Any]) -> Any: + """按新版重新出具的打算（走按口径出具同一套判法，ledger/issue.check_target：已发布未退役、没有人工节点和写入、参数过 + 校验、使用者只限有权使用的应用固定着的那一版、每日上限）。来源记成这条待办列的第一个答案。""" + from app.ledger import issue as ledger_issue + from app.ledger.labels import caliber_key + + workflow_id, version = str(origin.get("workflow_id") or ""), int(origin.get("to_version") or 0) + plan = ledger_issue.Plan(run_id=task.run_id, workflow_id=workflow_id, version=version, + caliber=caliber_key(workflow_id, version), how="recall", + binding=dict(origin.get("binding") or {})) + workflow = await session.get(Workflow, workflow_id) + plan.workflow_name = workflow.name if workflow is not None else "" + answers = _uses_of(origin, _owner_of_key(task.correlation_key or "")) + source = next((u["run_id"] for u in answers if u.get("kind") == "answer"), None) \ + or next((u["run_id"] for u in answers), None) or task.run_id + await ledger_issue.check_target(session, plan, workflow, principal=principal, source_run=str(source)) + return plan, str(source) + + +async def reissue_action(session: Any, task: Any, principal: Any, origin: Mapping[str, Any]) -> dict[str, Any]: + """待办里「按新版重新出具」这一项：能不能、不能的原因、已经出具过的那一次。""" + out: dict[str, Any] = {"decision": "reissue", "available": False} + if task.status != "pending": + issued = (task.outcome or {}).get("issued_run") + return {**out, **({"issued_run": issued} if issued else {})} + plan, _ = await _reissue_plan(session, task, principal, origin) + out["available"] = plan.reason is None + if plan.reason: + out["reason"] = plan.reason + if plan.existing: + out["issued_run"] = plan.existing + return out + + +async def reissue(task_id: str, principal: Any) -> Run: + """按新版重新出具：经 launch_run 发起一次正式运行（来源 ledger_issue，origin 记来源答案、召回运行、这条待办）。同一个来源、 + 同一版、同一组参数出具过的交回那一次。不能出具抛 RecallRefused。""" + from app.engine.launch import LaunchRefused, launch_run + from app.ledger import issue as ledger_issue + from app.process.tasks import Task + + async with SessionLocal() as session: + task = await session.get(Task, task_id) + if task is None: + raise RecallRefused(404, "待办不存在") + run = await session.get(Run, task.run_id) + origin = run.origin if run is not None and isinstance(run.origin, dict) else {} + plan, source = await _reissue_plan(session, task, principal, origin) + if plan.reason is not None: + raise RecallRefused(plan.status, plan.reason, plan.code) + if plan.existing is not None: + return await session.get(Run, plan.existing) + from app.apps.resolve import input_key + + try: + return await launch_run( + session, workflow_id=plan.workflow_id, run_class="formal", version=plan.version, + input_payload=plan.inputs, actor=principal.username, origin_kind=ledger_issue.ORIGIN, + origin={"source_run": source, "recall_run": task.run_id, "task_id": task_id, "caliber": plan.caliber, + "how": "recall", "binding_hash": input_key(plan.binding), "auto": False, + **({"granted_by_app": plan.granted_by_app} if plan.granted_by_app else {})}, + expect_hash=plan.expect_hash) + except LaunchRefused as e: + raise RecallRefused(e.status_code, e.message, e.code or REFUSED) from e + + +# ========================================================================== +# 二、目录事实被驳回或修改 +# ========================================================================== + + +def fact_changes(before: Mapping[str, Any] | None, after: Mapping[str, Any] | None) -> list[dict[str, Any]]: + """这次写入改动了的影响数字的项：写入前看得到（没驳回）、写入后被驳回、删掉或改了值的。{path, label, change}。纯函数。""" + from app.data import catalog + + old, new = catalog._slots(before), catalog._slots(after) + out: list[dict[str, Any]] = [] + for key, item in old.items(): + if item.get("status") == "rejected": + continue + fact = key[0] == "r" or (key[0], key[-1]) in FACT_SLOTS + if not fact: + continue + now = new.get(key) + if now is None: + change = "removed" + elif now.get("status") == "rejected": + change = "rejected" + elif catalog._value_of(key, now) != catalog._value_of(key, item): + change = "changed" + else: + continue + from app.data.catalog_review import slot_path + + out.append({"path": slot_path(key), "label": catalog.describe_slot(key), "change": change}) + return out + + +async def _explore_snapshots(session: Any, source: Any, table: str, *, before: datetime | None = None + ) -> list[tuple[str, str]]: + """查过 source 这张表的探索运行（计入统计的、成功封存的）和那条查询快照：[(运行, 快照)]，新的在前，最多 BASIS_MAX_RUNS + 次运行。表名按 name_key 比，表已经不在表结构里也认（表被删了正要找用过它的）。""" + from app.data.catalog import QUERY_SNAPSHOT_KIND + from app.data.names import name_key + from app.db.models import Artifact + from app.engine.origin import counted_runs + + want = name_key(table) + stmt = (select(Artifact.id, Artifact.meta, Run.id, Run.finished_at).join(Run, Run.id == Artifact.run_id) + .where(Artifact.kind == QUERY_SNAPSHOT_KIND, Run.run_class == "exploratory", Run.status == "succeeded", + Run.manifest_hash.is_not(None), counted_runs()).order_by(Run.finished_at.desc())) + out: list[tuple[str, str]] = [] + runs: set[str] = set() + cut = _aware(before) + for artifact, meta, run_id, finished in (await session.execute(stmt)).all(): + if cut is not None and (_aware(finished) is None or _aware(finished) > cut): + continue + meta = meta if isinstance(meta, dict) else {} + if meta.get("source_id") != source.id and meta.get("source") != source.name: + continue + if not any(name_key(str(t).rsplit(".", 1)[-1]) == want for t in meta.get("tables") or []): + continue + if run_id not in runs and len(runs) >= BASIS_MAX_RUNS: + continue + runs.add(run_id) + out.append((run_id, artifact)) + return out + + +def _frozen_of(snapshot_id: str) -> tuple[dict[str, Any] | None, dict[str, Any] | None]: + """查询快照 → (快照内容, 它当时冻结的表结构快照)。读不出为 None。""" + from app.core.artifact_store import load + + try: + snap = load(snapshot_id) + except ValueError: + return None, None + if not isinstance(snap, dict): + return None, None + try: + schema = load(str(snap.get("schema_artifact") or "")) if snap.get("schema_artifact") else None + except ValueError: + schema = None + return snap, schema if isinstance(schema, dict) else None + + +def _visible(frozen: Mapping[str, Any] | None, table: str, paths: Iterable[str]) -> list[str]: + """冻结的目录里这张表看得到这几项里的哪几项。""" + from app.data import catalog + from app.data.names import name_key + + entries = (frozen or {}).get("catalog") if isinstance((frozen or {}).get("catalog"), dict) else {} + notes = next((e.get("notes") for t, e in entries.items() if name_key(t) == name_key(table) + and isinstance(e, dict)), None) + slots = catalog._slots(notes if isinstance(notes, dict) else {}) + out = [] + for path in paths: + try: + key = catalog.parse_path(path) + except catalog.CatalogPathError: + continue + if key in slots and slots[key].get("status") != "rejected": + out.append(path) + return out + + +def new_errors(sql: str, frozen: Mapping[str, Any], current: Mapping[str, Mapping[str, Any]], + kind: str | None) -> list[dict[str, Any]]: + """这条 SQL 按新目录（current：现在的目录）跑 SQL 检查，比按冻结的目录多出来的错误级问题。纯计算。""" + from app.data.sqlcheck import SqlChecker, frozen_checker + + def key(c: Any) -> tuple[Any, ...]: + return (c.code, (c.table or "").lower(), (c.column or "").lower(), c.relation_id) + + old = frozen_checker(frozen, kind=kind) + before = {key(c) for c in (old.check(sql) if old is not None else []) if c.level == "error"} + cache = {"tables": frozen.get("tables") or {}, **({"schema": frozen["schema"]} if frozen.get("schema") else {})} + after = SqlChecker(kind=kind, schema_cache=cache, notes=dict(current), source_name=frozen.get("source")).check(sql) + return [{"code": c.code, "message": c.message, "table": c.table, **({"column": c.column} if c.column else {})} + for c in after if c.level == "error" and key(c) not in before] + + +async def _basis_marked(session: Any, run_id: str, match: Mapping[str, Any]) -> bool: + rows = (await session.execute(select(RunEvent.data).where( + RunEvent.run_id == run_id, RunEvent.type == EventType.LEDGER_BASIS_CHANGED))).scalars() + return any(isinstance(d, dict) and all(d.get(k) == v for k, v in match.items()) for d in rows) + + +async def _answer_owner(session: Any, run_id: str) -> list[tuple[str, str]]: + """一次探索运行（答案）的主人：对话的 created_by，没有的是发起人。""" + use = ledger_uses.Use(kind="answer", run_id=run_id, version=0, binding={}) + await ledger_uses._chat_owners(session, [use]) + return use.owners + + +async def catalog_changed(source_id: str, table: str, changes: Sequence[Mapping[str, Any]], *, + catalog_version: int | None, kind: str = "catalog", change_id: str | None = None) -> int: + """目录里这张表影响数字的几项被驳回或改了（见模块说明二）：冻结目录里看得到其中某项的探索运行追加 ledger.basis_changed； + 按新目录多出错误级问题的开 recall 待办（reason=catalog）。返回标了几次运行。""" + from app.data import catalog + from app.db.models import DataSource + + paths = [str(c.get("path")) for c in changes if c.get("path")] + if not paths and kind == "catalog": + return 0 + async with SessionLocal() as session: + source = await session.get(DataSource, source_id) + if source is None: + return 0 + found = await _explore_snapshots(session, source, table) + current = {t: e.notes for t, e in (await catalog.read_catalog(session, source_id)).items()} + by_run: dict[str, list[tuple[dict[str, Any], dict[str, Any]]]] = {} + for run_id, snapshot in found: + snap, frozen = await asyncio.to_thread(_frozen_of, snapshot) + if snap is None or frozen is None: + continue + by_run.setdefault(run_id, []).append((snap, frozen)) + marked = 0 + labels = {str(c.get("path")): c for c in changes} + for run_id, items in by_run.items(): + seen = sorted({p for _, frozen in items for p in _visible(frozen, table, paths)}) if paths else [] + if paths and not seen: + continue + errors: list[dict[str, Any]] = [] + for snap, frozen in items: + if isinstance(snap.get("sql"), str): + errors += new_errors(snap["sql"], frozen, current, source.kind) + match = {"kind": kind, "source_id": source_id, "table": table, + **({"catalog_version": catalog_version} if catalog_version is not None else {}), + **({"change": change_id} if change_id else {})} + async with SessionLocal() as session: + if await _basis_marked(session, run_id, match): + continue + data = {**match, "source": source.name, + "items": [{"path": p, "label": labels.get(p, {}).get("label"), "change": labels.get(p, {}).get("change")} + for p in seen], "new_errors": len(errors), "checks": errors[:10]} + from app.engine.run_append import append_events + + written = await append_events(run_id, [(EventType.LEDGER_BASIS_CHANGED, None, data)], require="sealed") + if not written: + continue + marked += 1 + if errors: + await _open_basis_task(run_id, written[0].seq) + return marked + + +async def _open_basis_task(run_id: str, seq: int) -> list[str]: + """答案依据的目录有变、按新目录检查出了新的错误级问题：给答案的主人开一条 recall 待办（reason=catalog，挂在答案的运行 + 上，只有「知道了」）。说的是哪一条依据有变记在 correlation_key 里（那条 ledger.basis_changed 的序号，_basis_of 按它读）。""" + async with SessionLocal() as session: + run = await session.get(Run, run_id) + if run is None: + return [] + owners = await _answer_owner(session, run_id) + return await open_tasks(run, reason="catalog", owners=owners, title=CATALOG_TITLE, extra=str(seq)) + + +def after_review(source_id: str, table: str, before: Mapping[str, Any] | None, after: Mapping[str, Any] | None, + version: int | None) -> int: + """审阅写入之后（data/catalog、catalog_review 调）：有影响数字的项被驳回或改了，在后台标依据有变。返回改了几项。 + 出错只记日志：审阅已经写进去了。""" + try: + changes = fact_changes(before, after) + if changes: + spawn(catalog_changed(source_id, table, changes, catalog_version=version)) + return len(changes) + except Exception: # noqa: BLE001 - 辅助步骤 + logger.exception("审阅写入后排队标依据有变时出错（%s · %s）", source_id, table) + return 0 + + +# ========================================================================== +# 三、表结构变动 +# ========================================================================== + +#: 「依据的表结构已变」认哪几种结构变化（表或列被删、列的类型变了） +STRUCTURE_KINDS = ("table_removed", "column_removed", "column_type") + + +def basis_kind(change: Mapping[str, Any]) -> str | None: + """一条变化记录算哪一种依据有变（triggers_recall 认的里面）：structure / values / catalog_edit；别的结构变化（新加了列、 + 注释变了……）不动答案，为 None。""" + from app.data import source_changes + + if not source_changes.triggers_recall(change): + return None + if (change.get("resolution") or {}).get("catalog_version"): + return "catalog_edit" + if change.get("kind") in STRUCTURE_KINDS: + return "structure" + if change.get("kind") == source_changes.CODES_NEW: + return "values" + return None + + +async def process_change(change: Mapping[str, Any]) -> int: + """处理一条变化记录：用过这张表的探索运行标依据有变；目录有改动的（新码值、处理时改了目录）按新目录查 SQL；表结构变了的 + 让受影响的口径试算一次。返回标了几次运行。""" + kind = basis_kind(change) + if kind is None or not change.get("table"): + return 0 + if kind in ("values", "catalog_edit"): + return await catalog_changed(str(change["source_id"]), str(change["table"]), [], + catalog_version=(change.get("resolution") or {}).get("catalog_version"), + kind=kind, change_id=str(change.get("id"))) + marked = await _mark_structure(change) + await _probe_calibers(change) + return marked + + +async def _mark_structure(change: Mapping[str, Any]) -> int: + from app.db.models import DataSource + from app.engine.run_append import append_events + + async with SessionLocal() as session: + source = await session.get(DataSource, str(change.get("source_id"))) + if source is None: + return 0 + detected = change.get("detected_at") + at = datetime.fromisoformat(detected) if isinstance(detected, str) else None + found = await _explore_snapshots(session, source, str(change["table"]), before=at) + marked = 0 + for run_id in dict.fromkeys(r for r, _ in found): + match = {"kind": "structure", "change": change.get("id")} + async with SessionLocal() as session: + if await _basis_marked(session, run_id, match): + continue + data = {**match, "source_id": source.id, "source": source.name, "table": change.get("table"), + "column": change.get("column"), "change_kind": change.get("kind"), "seq": change.get("seq"), + "items": [{"label": str(change.get("kind")), "column": change.get("column")}]} + if await append_events(run_id, [(EventType.LEDGER_BASIS_CHANGED, None, data)], require="sealed") is not None: + marked += 1 + return marked + + +async def _probe_calibers(change: Mapping[str, Any]) -> list[str]: + """表结构变了：最新的已发布版本里查这张表的口径，按最近一次正式运行的输入（没有就按入口的缺省值）派生数据部分试算一次 + （召回运行，origin.probe）。同一条变化、同一版口径只试算一次。返回发起了的运行。""" + from app.data.names import name_key + from app.engine.launch import LaunchRefused, launch_run + from app.ledger import compose, derive + from app.ledger import match as ledger_match + from app.tools.datasource import READ_ONLY_OPTION + + table = name_key(str(change.get("table") or "").rsplit(".", 1)[-1]) + launched: list[str] = [] + async with SessionLocal() as session: + calibers = await ledger_match.load_calibers(session) + for caliber in calibers: + if not any(f.source == change.get("source_name") and table in f.tables for f in caliber.fetches): + continue + async with SessionLocal() as session: + done = await session.scalar(select(func.count()).select_from(Run).where( + Run.origin_kind == ORIGIN, func.json_extract(Run.origin, "$.probe") == 1, + func.json_extract(Run.origin, "$.workflow_id") == caliber.workflow_id, + func.json_extract(Run.origin, "$.version") == caliber.version, + func.json_extract(Run.origin, "$.change") == change.get("id"))) + if done: + continue + latest = (await session.execute(select(Run.input).where( + Run.workflow_id == caliber.workflow_id, Run.version == caliber.version, Run.run_class == "formal", + Run.status == "succeeded").order_by(Run.finished_at.desc()).limit(1))).scalar_one_or_none() + inputs_from = "formal" if latest else "defaults" + if not latest: + # 这一版还没有正式运行：退回最近一次等价于它的问数回答绑定出的参数，再没有才只用入口的缺省值 + found = await ledger_uses.find_uses(session, caliber.workflow_id, only=caliber.version) + answer = next((u for u in found.uses if u.kind == "answer"), None) + if answer is not None: + latest, inputs_from = dict(answer.binding), "answer" + for card in caliber.cards[:1]: + try: + pinned = await derive.load_pinned(session, caliber.workflow_id, caliber.version, card) + names = {str(f.get("name")) for f in pinned.entry_fields} + inputs = compose.check_inputs(pinned, {k: v for k, v in (latest or {}).items() if k in names}) + except (derive.NotDerivable, compose.ComposeRefused): + continue + probe = derive.caliber_probe(pinned) + origin = {"probe": 1, "workflow_id": caliber.workflow_id, "version": caliber.version, + "change": change.get("id"), "source_id": change.get("source_id"), "table": change.get("table"), + "column": change.get("column"), "change_kind": change.get("kind"), + "inputs_from": inputs_from, "derive": probe.origin} + try: + run = await launch_run(session, graph=probe.graph, input_payload=inputs, run_class="exploratory", + actor=None, origin_kind=ORIGIN, origin=origin, + options={READ_ONLY_OPTION: True}, + workflow_name=f"口径试算：{pinned.workflow_name} v{caliber.version}") + except LaunchRefused as e: + logger.info("表结构变了之后口径试算没能发起（%s v%s）：%s", caliber.workflow_id, caliber.version, + e.message) + continue + launched.append(run.id) + return launched + + +async def _settle_probe(run: Run) -> str: + """试算运行结束：算出来了只记结论；没算出来给那一版口径的发布人开 recall 待办（reason=structure）。""" + from app.apps import versions as app_versions + + origin = run.origin if isinstance(run.origin, dict) else {} + ok = run.status == "succeeded" + tasks: list[str] = [] + if not ok: + async with SessionLocal() as session: + row = await app_versions.version_row(session, str(origin.get("workflow_id")), int(origin.get("version") or 0)) + name = await ledger_uses.workflow_name(session, str(origin.get("workflow_id") or "")) + publisher = getattr(row, "published_by", None) if row is not None else None + tasks = await open_tasks(run, reason="structure", owners=[("user", publisher)] if publisher else [], + title=STRUCTURE_TITLE.format(name=name or "口径", version=origin.get("version"))) + await _mark(run.id, {"role": "recall", "probe": True, "outcome": "unchanged" if ok else "failed", + "changed": None, "workflow_id": origin.get("workflow_id"), "version": origin.get("version"), + "tasks": tasks, **({} if ok else {"error": str(run.error or "")[:300]})}) + return "unchanged" if ok else "failed" + + +async def _cursor(session: Any) -> int: + from app.db.models import Setting + + row = await session.get(Setting, CURSOR_KEY) + value = (row.value or {}).get("seq") if row is not None and isinstance(row.value, dict) else None + return int(value) if isinstance(value, int) else 0 + + +async def _set_cursor(session: Any, seq: int) -> None: + from app.db.models import Setting + + row = await session.get(Setting, CURSOR_KEY) + if row is None: + session.add(Setting(key=CURSOR_KEY, value={"seq": int(seq)})) + else: + row.value = {"seq": int(seq)} + await session.commit() + + +_consuming: set[asyncio.AbstractEventLoop] = set() + + +async def consume_source_changes() -> int: + """按 seq 游标拉变化记录（data/source_changes.list_changes），逐条处理（process_change），游标前移。探查、剖析任务结束时 + 调。同一时刻只拉一份。返回处理了几条。""" + from app.data import source_changes + + if not automatic(): + return 0 + loop = asyncio.get_running_loop() + if loop in _consuming: + return 0 + _consuming.add(loop) + try: + count = 0 + while True: + async with SessionLocal() as session: + since = await _cursor(session) + batch = await source_changes.list_changes(session, since_seq=since, limit=200) + if not batch: + return count + for change in batch: + try: + if basis_kind(change) is not None: + await process_change(change) + count += 1 + except Exception: # noqa: BLE001 - 一条出错不挡后面的 + logger.exception("召回处理变化记录 #%s 出错", change.get("seq")) + async with SessionLocal() as session: + await _set_cursor(session, int(batch[-1]["seq"])) + finally: + _consuming.discard(loop) + + +# ========================================================================== +# 发布钩子、启动时接着做、测试用的 drain +# ========================================================================== + + +def automatic() -> bool: + """自动召回（发布后自动那一批、探查剖析之后拉表结构变化、启动时接着做）开没开：和调度循环同一个开关 + （AGENTLAB_SCHEDULER）。这几样都是在后台替人查业务库（召回运行、口径试算），测试、拷来的数据目录（检查脚本）不做—— + 同 ledger/match 的启动补比、定时任务。手动「接着算」、审阅之后标依据有变（只读应用库）不受它管。""" + from app.core.config import settings + + return bool(settings.scheduler) + + +def after_publish(workflow_id: str, version: int, actor: str | None) -> bool: + """工作流发布了第 version 版（api/workflows.commit_publish 提交之后）：在后台按计划自动召回一批（automatic 关着时不做， + 版本历史里照样列出受影响的答案、能手动接着算）。不挡发布。返回排没排上。""" + if not automatic(): + return False + try: + spawn(launch(workflow_id, int(version), batch="auto", actor=actor)) + return True + except Exception: # noqa: BLE001 - 辅助步骤 + logger.exception("工作流 %s 发布 v%s 后排队召回出错", workflow_id, version) + return False + + +async def resume(*, days: int = RESUME_DAYS) -> dict[str, int]: + """启动时：最近 days 天发布过的版本，自动那一批没发完的接着发；结束了还没收尾的召回运行补上收尾；表结构变化的游标没有 + 时从现在开始（升级之前的变化不追）。""" + from app.db.models import WorkflowVersion + + since = utcnow() - timedelta(days=days) + async with SessionLocal() as session: + rows = (await session.execute(select(WorkflowVersion.workflow_id, WorkflowVersion.version, WorkflowVersion.published_by) + .where(WorkflowVersion.published_at >= since, + WorkflowVersion.level.is_not(None)))).all() + unsettled = [r for (r,) in (await session.execute(select(Run.id).where( + Run.origin_kind == ORIGIN, Run.status.in_(("succeeded", "failed", "cancelled")), + Run.manifest_hash.is_not(None)))).all()] + from app.db.models import Setting + + if await session.get(Setting, CURSOR_KEY) is None: + from app.db.models_sources import SourceChange + + top = int(await session.scalar(select(func.max(SourceChange.seq))) or 0) + await _set_cursor(session, top) + launched = 0 + for workflow_id, version, actor in rows: + launched += len(await launch(workflow_id, int(version), batch="auto", actor=actor)) + settled = 0 + for run_id in unsettled: + settled += await settle(run_id) is not None + return {"launched": launched, "settled": settled} + + +def start_resume() -> bool: + """启动时在后台接着做（engine/runner 的 setup 调）。和调度循环同一个开关（automatic）：拷来的数据目录、测试不替原库补做 + 后台的事。""" + if not automatic(): + return False + spawn(resume()) + return True + + +def _mine() -> list[asyncio.Task[Any]]: + loop = asyncio.get_running_loop() + return [t for t in _background if t.get_loop() is loop] + + +async def drain() -> None: + """等这个事件循环里召回的后台任务都做完（测试用）。""" + while True: + live = [t for t in _mine() if not t.done()] + if not live: + return + await asyncio.gather(*live, return_exceptions=True) + + +async def shutdown() -> None: + tasks = _mine() + for task in tasks: + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + + +__all__ = ["BASIS_KINDS", "BATCHES", "CURSOR_KEY", "OUTCOMES", "ORIGIN", "REASONS", "TASK_KIND", "Group", "Pair", "Plan", + "RecallRefused", "after_publish", "after_review", "automatic", "basis_kind", "batches", "build_graph", "build_plan", + "card_values", "catalog_changed", "compare", "consume_source_changes", "drain", "fact_changes", "launch", + "new_errors", "normalize", "open_tasks", "original_values", "outcome_of", "preview", "process_change", + "recall_card", "recall_runs", "reissue", "reissue_action", "resume", "schedule_settle", "settle", "shutdown", + "source_cards", "spawn", "start_resume", "task_payload", "version_recall_counts"] diff --git a/backend/app/ledger/stats.py b/backend/app/ledger/stats.py index b0f4bb8e..9f5e4832 100644 --- a/backend/app/ledger/stats.py +++ b/backend/app/ledger/stats.py @@ -17,8 +17,9 @@ 段对话的算重复的形状，给出这样的形状几个、有重复形状的运行占几成；问题聚类（app/chat/questions：全站唯一的聚类，只取问题 原文、不含 history、排除系统派生的运行）里 ≥ REPEAT_RUNS 个问题的类占几成。认证队列的转化（第 3 波，ledger/certify）：固化成 口径草稿几份（origin.freeze 的工作流）、其中发布了几份、点了「不是口径」几项。 -- E4 召回：每个口径版本被用了多少次（比对里等价已证明的运行数、这一版的正式运行数）。每次发布召回了几组、其中数变了的比例要等 - 召回（第 4 波），这之前照实写没有数据，只给召回运行的次数。 +- E4 召回：每个口径版本被用了多少次（比对里等价已证明的运行数、这一版的正式运行数）；每次发布（口径升到某一版）召回了几组、 + 收过尾的几组、其中数变了的几组和比例、重算失败的几组（ledger/recall：召回运行和它收尾时记的 ledger.recalled，第 4 波）。 + 按召回运行的发起时间筛窗口；表结构变了之后的口径试算不算召回。 """ from __future__ import annotations @@ -43,7 +44,6 @@ DIFF_GROUPS = ("time", "filter", "aggregate", "join", "other", "parse") STATUSES = ("proven", "composed", "related", "none", "skipped") #: 没有数据时的说法（这一块的功能还没做） NOT_YET_RECONCILE = "对账面板的反馈在第 4 波接上，目前没有数据" -NOT_YET_RECALL = "召回在第 4 波接上，目前没有数据" #: 聚类最多取几个问题（同治理接口的上限） QUESTION_LIMIT = 500 @@ -294,7 +294,50 @@ async def _e4(session: Any, contents: list[dict[str, Any]], **scope: Any) -> dic usage = [{"caliber": key, "workflow_id": key.rsplit("@", 1)[0], "version": int(key.rsplit("@", 1)[1]), "proven_runs": proven.get(key, 0), "formal_runs": formal.get(key, 0)} for key in sorted(set(proven) | set(formal))] - return {"usage": usage, "recall_runs": recalls, "per_publish": {"available": False, "note": NOT_YET_RECALL}} + return {"usage": usage, "recall_runs": recalls, "per_publish": await _per_publish(session, **scope)} + + +async def _per_publish(session: Any, **scope: Any) -> dict[str, Any]: + """每次发布召回了几组、其中数变了的比例（E4 下半）：按（工作流、新版本）分，读召回运行收尾时记的结论。""" + from app.core.events import EventType + from app.db.models import Run, RunEvent + + stmt = select(Run.id, Run.origin).where(Run.origin_kind == "recall") + if scope.get("run_ids") is not None: + stmt = stmt.where(Run.id.in_(list(scope["run_ids"]))) + else: + if scope.get("since") is not None: + stmt = stmt.where(Run.created_at >= scope["since"]) + if scope.get("until") is not None: + stmt = stmt.where(Run.created_at < scope["until"]) + runs = [(rid, o) for rid, o in (await session.execute(stmt)).all() + if isinstance(o, dict) and not o.get("probe") and o.get("workflow_id")] + settled: dict[str, dict[str, Any]] = {} + if runs: + rows = (await session.execute(select(RunEvent.run_id, RunEvent.data).where( + RunEvent.run_id.in_([r for r, _ in runs]), RunEvent.type == EventType.LEDGER_RECALLED) + .order_by(RunEvent.seq))).all() + for rid, data in rows: + if isinstance(data, dict) and data.get("role") == "recall": + settled[rid] = data + by: dict[tuple[str, int], dict[str, int]] = {} + for rid, origin in runs: + key = (str(origin["workflow_id"]), int(origin.get("to_version") or 0)) + item = by.setdefault(key, {"groups": 0, "settled": 0, "changed": 0, "failed": 0}) + item["groups"] += 1 + data = settled.get(rid) + if data is None: + continue + item["settled"] += 1 + if data.get("outcome") == "failed": + item["failed"] += 1 + elif data.get("changed"): + item["changed"] += 1 + items = [{"caliber": ledger_labels.caliber_key(w, v), "workflow_id": w, "version": v, **c, + "changed_rate": _ratio(c["changed"], c["settled"] - c["failed"])} + for (w, v), c in sorted(by.items())] + return {"available": True, "publishes": items, "groups": sum(i["groups"] for i in items), + "changed": sum(i["changed"] for i in items)} async def compute_stats(session: Any, *, since: datetime | None = None, until: datetime | None = None, diff --git a/backend/app/ledger/uses.py b/backend/app/ledger/uses.py new file mode 100644 index 00000000..1bfdd825 --- /dev/null +++ b/backend/app/ledger/uses.py @@ -0,0 +1,382 @@ +"""一份口径的旧版本被谁用过（docs/design/next-round/02-口径账本.md 做法 7 一）：口径发布新版本之后，召回要追的那些答案。 + +**为什么单独一个模块。** 召回（ledger/recall）要按参数去重成组、重算，发布弹窗要预告规模，版本历史要写「这一版被多少问数 +回答、导出、召回用过」——三处要的是同一份「用过」的清单。各数各的，预告的规模和实际召回的就对不上（验收要求两者一致）。 + +反查的对象（工作流 W 的旧版本 V，只看 V < 新版本）： +- **问数回答**（answer）：caliber_match 工件的 meta 里记着 W@V 等价已证明（含助手选用）的探索运行，取这次运行现在这次封存 + 的最新一份比对（ledger/store.latest：之后补比过、按新口径不再等价的不算），每条等价的查询一份参数（绑定）； +- **W@V 的正式运行**：定时任务发起的（schedule，每个定时任务最近 PERIODS 期）、应用发起的（app，每个应用最近 PERIODS 次）、 + 别处发起的（formal：手动、接口、按口径出具……），参数是那次运行的输入； +- **固定了 W@V 的工作流版本发起的正式运行**（pinned：子工作流或口径卡 caliber_from 固定在 W@V，apps/versions.pinned_upstream + 同一个认法），参数是那次运行的输入里 W 的入口有的字段； +- **固定这一版的应用**（apps/versions.pinning_apps）：应用发起过的正式运行在上面；还没有运行的只列出来，没有可重算的参数； +- **导出记录**：上面这些运行上的 report.exported 审计（第 2 波），导出人也是答案的主人； +- **问数轮次**：运行 origin 里封存的 conversation_id / turn_id（origin_ref，第 2 波），老运行退回 conversation_turns.run_id。 + +**答案的主人**（owners，召回待办派给谁）：对话的 created_by（没有的退回运行的发起人）、导出人、定时任务的 created_by、应用的 +管理者（授权里的 manage，没有就是建应用的人）；别处发起的正式运行是它的发起人。只记 (user|group, 名字)，组由开待办时展开。 + +只读应用库：不查业务库、不调模型。系统派生的运行（评测、对账、召回，engine/origin.counted_runs）不算「用过」。数量有上限 +（MAX_ANSWERS、MAX_FORMAL），按新的在前取，超出的照实算进 truncated。 +""" +from __future__ import annotations + +from collections.abc import Iterable, Mapping +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any + +from sqlalchemy import and_, func, or_, select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import Artifact, Run, Workflow, WorkflowVersion + +#: 用法的种类。answer 问数回答；schedule 定时任务发起的正式运行；app 应用发起的；formal 别处发起的；pinned 固定了这一版的 +#: 别的工作流版本发起的 +KINDS = ("answer", "schedule", "app", "formal", "pinned") +#: 每个定时任务、每个应用最近几期（细案「定时任务最近 K 期」） +PERIODS = 4 +#: 最多看多少次问数回答、多少次正式运行（各按新的在前） +MAX_ANSWERS = 2000 +MAX_FORMAL = 500 +#: 种类的叫法（版本用量、召回批次、待办里写「来自问数据」……） +KIND_TEXT = {"answer": "问数回答", "schedule": "定时任务", "app": "应用", "formal": "正式运行", + "pinned": "固定了它的工作流"} +#: 导出审计的动作（api/evidence.py、api/bundle.py 写的） +EXPORTED = "report.exported" + + +@dataclass +class Use: + """一次「用过」：哪次运行、旧版本几、参数（原样，没补缺省值）、什么时候、谁是答案的主人。""" + + kind: str + run_id: str + version: int + binding: dict[str, Any] + at: datetime | None = None + owners: list[tuple[str, str]] = field(default_factory=list) + conversation_id: str | None = None + turn_id: str | None = None + schedule_id: str | None = None + app_id: str | None = None + #: 问数回答：等价的是哪条查询（查询快照）、口径的哪个取数节点 + query: str | None = None + node_id: str | None = None + #: 固定了这一版的是哪个工作流的哪一版（pinned） + via_workflow: str | None = None + via_version: int | None = None + exporters: list[str] = field(default_factory=list) + + def add_owner(self, kind: str, name: str | None) -> None: + if name and (kind, name) not in self.owners: + self.owners.append((kind, name)) + + def describe(self) -> dict[str, Any]: + """记进召回运行 origin 的那一份（召回结束时据此开待办、在原答案上记账）：只放编号、参数的来处和主人。""" + out: dict[str, Any] = {"kind": self.kind, "run_id": self.run_id, "version": self.version, + "owners": [list(o) for o in self.owners]} + for key in ("conversation_id", "turn_id", "schedule_id", "app_id", "query", "node_id", "via_workflow", + "via_version"): + if (value := getattr(self, key)) is not None: + out[key] = value + if self.exporters: + out["exporters"] = list(self.exporters) + if self.at is not None: + out["at"] = self.at.isoformat() + return out + + +@dataclass +class Found: + """反查的结果：用法、固定着旧版本的应用（{版本: [应用]}）、有没有因为上限少看了。""" + + uses: list[Use] + apps: dict[int, list[dict[str, Any]]] = field(default_factory=dict) + truncated: bool = False + + def versions(self) -> list[int]: + return sorted({u.version for u in self.uses} | set(self.apps)) + + +# ========================================================================== +# 反查 +# ========================================================================== + + +async def _answers(session: AsyncSession, workflow_id: str, below: int | None, only: int | None + ) -> tuple[list[Use], bool]: + """问数回答：比对工件 meta 里等价已证明的那几份口径里有 W@V 的探索运行（见模块说明）。""" + from app.engine.origin import counted_runs + from app.ledger import store + from app.ledger.labels import caliber_key + + prefix = f"{workflow_id}@" + rows = (await session.execute( + select(Artifact.run_id, Artifact.meta, Run.finished_at).join(Run, Run.id == Artifact.run_id) + .where(Artifact.kind == store.KIND, Run.status == "succeeded", counted_runs()) + .order_by(Run.finished_at.desc()))).all() + wanted: dict[str, datetime | None] = {} + for run_id, meta, at in rows: + keys = [k for k in ((meta or {}).get("calibers") or []) if isinstance(k, str) and k.startswith(prefix)] + if not keys or run_id in wanted: + continue + versions = {int(k.rsplit("@", 1)[1]) for k in keys if k.rsplit("@", 1)[1].isdigit()} + if any((only is None or v == only) and (below is None or v < below) for v in versions): + wanted[run_id] = at + truncated = len(wanted) > MAX_ANSWERS + out: list[Use] = [] + for run_id, at in list(wanted.items())[:MAX_ANSWERS]: + recorded = await store.latest(run_id) + content = recorded.content if recorded is not None else None + if not isinstance(content, dict): + continue + seen: set[tuple[int, str]] = set() + for query in content.get("queries") or []: + for cand in query.get("candidates") or []: + if cand.get("status") != "proven" or cand.get("workflow_id") != workflow_id: + continue + version = cand.get("version") + if not isinstance(version, int) or (only is not None and version != only) \ + or (below is not None and version >= below): + continue + if cand.get("caliber") != caliber_key(workflow_id, version): + continue + binding = dict(cand.get("binding") or {}) + key = (version, _key(binding)) + if key in seen: + continue + seen.add(key) + out.append(Use(kind="answer", run_id=run_id, version=version, binding=binding, at=at, + query=query.get("artifact"), node_id=cand.get("node_id"))) + await _chat_owners(session, [u for u in out]) + return out, truncated + + +def _key(binding: Mapping[str, Any]) -> str: + from app.apps.resolve import input_key + + return input_key(dict(binding)) + + +async def _chat_owners(session: AsyncSession, found: list[Use]) -> None: + """问数回答的轮次和主人：运行 origin 里封存的对话、轮次（origin_ref），老运行退回 conversation_turns.run_id；主人是 + 对话的 created_by，没有的是运行的发起人。""" + from app.db.models import Conversation, ConversationTurn + + ids = sorted({u.run_id for u in found}) + if not ids: + return + runs = {rid: (origin, started) for rid, origin, started in (await session.execute( + select(Run.id, Run.origin, Run.started_by).where(Run.id.in_(ids)))).all()} + turns: dict[str, tuple[str, str]] = {} + for rid, (origin, _) in runs.items(): + if isinstance(origin, dict) and origin.get("conversation_id") and origin.get("turn_id"): + turns[rid] = (str(origin["conversation_id"]), str(origin["turn_id"])) + missing = [rid for rid in ids if rid not in turns] + if missing: + for turn_id, conv, rid in (await session.execute( + select(ConversationTurn.id, ConversationTurn.conversation_id, ConversationTurn.run_id) + .where(ConversationTurn.run_id.in_(missing)).order_by(ConversationTurn.created_at))).all(): + turns.setdefault(rid, (conv, turn_id)) + convs = sorted({c for c, _ in turns.values()}) + owners = dict((await session.execute(select(Conversation.id, Conversation.created_by) + .where(Conversation.id.in_(convs)))).all()) if convs else {} + for use in found: + if use.run_id in turns: + use.conversation_id, use.turn_id = turns[use.run_id] + owner = owners.get(use.conversation_id) if use.conversation_id else None + use.add_owner("user", owner or runs.get(use.run_id, (None, None))[1]) + + +async def _app_managers(session: AsyncSession, app_ids: Iterable[str]) -> dict[str, list[tuple[str, str]]]: + """应用的管理者：授权里的 manage（人或组）；没有授权管理者的是建应用的人。""" + from app.db.models_apps import App, AppGrant + + ids = sorted({a for a in app_ids if a}) + if not ids: + return {} + out: dict[str, list[tuple[str, str]]] = {a: [] for a in ids} + for app_id, kind, name in (await session.execute( + select(AppGrant.app_id, AppGrant.principal_kind, AppGrant.principal) + .where(AppGrant.app_id.in_(ids), AppGrant.role == "manage"))).all(): + if kind in ("user", "group") and name and (kind, name) not in out[app_id]: + out[app_id].append((kind, name)) + for app_id, created_by in (await session.execute(select(App.id, App.created_by).where(App.id.in_(ids)))).all(): + if not out[app_id] and created_by: + out[app_id].append(("user", created_by)) + return out + + +async def _formal(session: AsyncSession, clause: Any, *, kind_of: Any) -> tuple[list[Use], bool]: + """满足 clause 的已封存、成功的正式运行（新的在前，最多 MAX_FORMAL 次）→ 用法。定时任务、应用各只留最近 PERIODS 期。 + kind_of(run) 给出这次运行算哪一种（schedule / app / formal / pinned）和旧版本。""" + from app.db.models import Schedule + from app.engine.origin import counted_runs + + rows = list((await session.execute( + select(Run).where(clause, Run.run_class == "formal", Run.status == "succeeded", + Run.manifest_hash.is_not(None), counted_runs()) + .order_by(Run.finished_at.desc()).limit(MAX_FORMAL + 1))).scalars()) + truncated = len(rows) > MAX_FORMAL + per: dict[tuple[str, str], int] = {} + out: list[Use] = [] + for run in rows[:MAX_FORMAL]: + kind, version, via = kind_of(run) + group = ("schedule", run.schedule_id) if run.schedule_id else ("app", run.app_id) if run.app_id else None + if group is not None: + per[group] = per.get(group, 0) + 1 + if per[group] > PERIODS: + continue + use = Use(kind=kind, run_id=run.id, version=version, binding=dict(run.input or {}), at=run.finished_at, + schedule_id=run.schedule_id, app_id=run.app_id) + if via is not None: + use.via_workflow, use.via_version = via + out.append(use) + schedules = sorted({u.schedule_id for u in out if u.schedule_id}) + makers = dict((await session.execute(select(Schedule.id, Schedule.created_by) + .where(Schedule.id.in_(schedules)))).all()) if schedules else {} + managers = await _app_managers(session, [u.app_id for u in out if u.app_id]) + starters = dict((await session.execute(select(Run.id, Run.started_by).where( + Run.id.in_([u.run_id for u in out])))).all()) if out else {} + for use in out: + if use.schedule_id: + use.add_owner("user", makers.get(use.schedule_id) or starters.get(use.run_id)) + elif use.app_id: + for owner in managers.get(use.app_id) or []: + use.add_owner(*owner) + else: + use.add_owner("user", starters.get(use.run_id)) + return out, truncated + + +def _pins(graph: Any, workflow_id: str) -> set[int]: + """一张图里固定了 workflow_id 的哪几版（子工作流、口径卡的 caliber_from；同 apps/versions.pinned_upstream 的认法， + 直接读 JSON，不为了这个把整张图解析成 GraphSpec）。""" + out: set[int] = set() + for node in (graph or {}).get("nodes") or [] if isinstance(graph, dict) else []: + if not isinstance(node, dict): + continue + cfg = (node.get("data") or {}).get("config") if isinstance(node.get("data"), dict) else None + if not isinstance(cfg, dict): + continue + if node.get("type") == "subgraph": + wid, version = cfg.get("workflow_id"), cfg.get("workflow_version") + elif node.get("type") == "metrics" and isinstance(cfg.get("caliber_from"), dict): + wid, version = cfg["caliber_from"].get("workflow_id"), cfg["caliber_from"].get("workflow_version") + else: + continue + if wid == workflow_id and str(version or "").isdigit(): + out.add(int(version)) + return out + + +async def pinned_versions(session: AsyncSession, workflow_id: str) -> dict[tuple[str, int], set[int]]: + """别的工作流的已发布版本里固定了 workflow_id 的：{(工作流, 版本): 固定的 W 的版本}。""" + rows = (await session.execute( + select(WorkflowVersion.workflow_id, WorkflowVersion.version, WorkflowVersion.graph) + .where(WorkflowVersion.workflow_id != workflow_id, WorkflowVersion.level.is_not(None)))).all() + return {(wid, int(v)): pins for wid, v, graph in rows if (pins := _pins(graph, workflow_id))} + + +async def find_uses(session: AsyncSession, workflow_id: str, *, below: int | None = None, + only: int | None = None) -> Found: + """W 的旧版本被谁用过（见模块说明）。below：只看版本号小于它的（召回：新版本）；only：只看这一版（版本用量）。""" + from app.apps.versions import pinning_apps + + def version_ok(v: Any) -> bool: + return isinstance(v, int) and (only is None or v == only) and (below is None or v < below) + + answers, cut_a = await _answers(session, workflow_id, below, only) + own = Run.workflow_id == workflow_id + if only is not None: + own = and_(own, Run.version == only) + elif below is not None: + own = and_(own, Run.version < below) + formal, cut_f = await _formal(session, own, kind_of=lambda r: ( + "schedule" if r.schedule_id else "app" if r.app_id else "formal", int(r.version), None)) + pins = await pinned_versions(session, workflow_id) + pinned: list[Use] = [] + cut_p = False + targets = {key: sorted(v for v in versions if version_ok(v)) for key, versions in pins.items()} + targets = {key: vs for key, vs in targets.items() if vs} + if targets: + clause = or_(*(and_(Run.workflow_id == wid, Run.version == v) for (wid, v) in targets)) + pinned, cut_p = await _formal(session, clause, kind_of=lambda r: ( + "pinned", targets[(r.workflow_id, int(r.version))][-1], (r.workflow_id, int(r.version)))) + uses = [*answers, *formal, *pinned] + await _exporters(session, uses) + apps: dict[int, list[dict[str, Any]]] = {} + for item in await pinning_apps(session, workflow_id): + if version_ok(item.get("version")): + apps.setdefault(int(item["version"]), []).append(item) + return Found(uses=uses, apps=apps, truncated=cut_a or cut_f or cut_p) + + +async def _exporters(session: AsyncSession, uses: list[Use]) -> None: + """这些运行上的报告导出审计：导出人进答案的主人。""" + from app.db.models import AuditEvent + + ids = sorted({u.run_id for u in uses}) + if not ids: + return + by: dict[str, list[str]] = {} + for rid, actor in (await session.execute(select(AuditEvent.subject_id, AuditEvent.actor).where( + AuditEvent.action == EXPORTED, AuditEvent.subject_kind == "run", + AuditEvent.subject_id.in_(ids)).order_by(AuditEvent.at))).all(): + if actor and actor not in by.setdefault(rid, []): + by[rid].append(actor) + for use in uses: + use.exporters = list(by.get(use.run_id) or []) + for actor in use.exporters: + use.add_owner("user", actor) + + +# ========================================================================== +# 版本用量（版本历史、发布弹窗） +# ========================================================================== + + +async def exports_count(session: AsyncSession, run_ids: Iterable[str]) -> int: + from app.db.models import AuditEvent + + ids = sorted(set(run_ids)) + if not ids: + return 0 + return int(await session.scalar(select(func.count()).select_from(AuditEvent).where( + AuditEvent.action == EXPORTED, AuditEvent.subject_kind == "run", AuditEvent.subject_id.in_(ids))) or 0) + + +def summarize(found: Found) -> dict[str, Any]: + """用法的计数（不含参数和主人）：问数回答几次（运行数）、导出几次、定时任务几个、应用几个、各种正式运行几次。""" + answers = {u.run_id for u in found.uses if u.kind == "answer"} + schedules = {u.schedule_id for u in found.uses if u.schedule_id} + apps = {u.app_id for u in found.uses if u.app_id} | {a["app_id"] for items in found.apps.values() for a in items} + return {"answers": len(answers), "schedules": len(schedules), "apps": len(apps), + "formal": len({u.run_id for u in found.uses if u.kind in ("formal", "schedule", "app")}), + "pinned": len({u.run_id for u in found.uses if u.kind == "pinned"}), + "exports": sum(len(u.exporters) for u in {u.run_id: u for u in found.uses}.values()), + "truncated": found.truncated} + + +async def version_usage(session: AsyncSession, workflow_id: str, version: int) -> dict[str, Any]: + """一个版本的用法计数（账本接口 GET /api/ledger/workflow-versions/{wf}/{v} 加的那几项）：问数回答、导出、召回。 + + 召回分两头：这一版作为新版本召回过几组（recall_runs：以它为新版本发起过的召回运行；changed：其中数变了的）、作为旧版本 + 被召回过几组（recalled_from）。""" + from app.ledger import recall + + found = await find_uses(session, workflow_id, only=version) + out = summarize(found) + out["exports"] = await exports_count(session, [u.run_id for u in found.uses]) + out["recall"] = await recall.version_recall_counts(session, workflow_id, version) + return out + + +async def workflow_name(session: AsyncSession, workflow_id: str) -> str: + return str(await session.scalar(select(Workflow.name).where(Workflow.id == workflow_id)) or "") + + +__all__ = ["EXPORTED", "Found", "KINDS", "KIND_TEXT", "MAX_ANSWERS", "MAX_FORMAL", "PERIODS", "Use", "exports_count", + "find_uses", "pinned_versions", "summarize", "version_usage", "workflow_name"] diff --git a/backend/app/notify/service.py b/backend/app/notify/service.py index 8d0fea86..b8bfeac4 100644 --- a/backend/app/notify/service.py +++ b/backend/app/notify/service.py @@ -35,7 +35,9 @@ from typing import Any from sqlalchemy import select from app.db.base import SessionLocal, utcnow -from app.db.models import App, AppVersion, NotifyChannel, NotifyDelivery, Run, Setting, Task, TaskAssignment, User +from app.db.models import ( + App, AppVersion, NotifyChannel, NotifyDelivery, Run, Setting, Task, TaskAssignment, User, Workflow, +) from app.notify.core import Message, Outcome, host_of logger = logging.getLogger(__name__) @@ -69,8 +71,9 @@ EVENTS: dict[str, Event] = { # ---- 本轮登记的（总方案 X6）：scopes 为空的还没有推送；实现推送的那一块只改自己这一行的 scopes，消息照模块说明的规矩 ---- # 数据源有要处理的结构 / 取值变化（03 B3）：只推 severity 为 action 的，按源汇总；源名、各级别条数、链接 "source.changed": Event("数据源有变化要处理", ()), - # 口径升版后召回重算，数真变了（02 第 7 项）：答案的主人有 recall 待办；口径名、版本、链接，不放新旧值和差值 - "ledger.recall": Event("口径升版，答案的数变了", ()), + # 口径升版后召回重算，数真变了（02 第 7 项，notify_recall）：答案的主人有 recall 待办；推全局渠道和答案主人自己的个人 + # 渠道（召回运行不归哪个应用）。口径名、版本、几个答案、链接，不放新旧值和差值 + "ledger.recall": Event("口径升版，答案的数变了", ("global", "user")), # 定时评测有题从通过变成不通过 / 评测没跑完（01 第 15 项）：题集名、通过数、退步题数、链接；不放期望值和答案里的数。 # 只推全局渠道：评测是搭建者们共用的一把尺，不归哪个人（notify_eval）。只推定时发起的（手动发起的那次人自己在看）， # 叫法就照实带「定时」；没评完包括失败、到了上限、止损停下，所以不叫「失败」 @@ -103,7 +106,7 @@ _TASK_HEAD = {"task.opened": "新的待办", "task.reassigned": "待办已改派 "task.reminded": "待办催办"} _KIND_TEXT = {"approve": "审批", "input": "补充输入", "edit": "编辑稿件", "tool": "工具调用审批", "code": "代码执行审批", "dual_check": "双盲复算裁决", "action_confirm": "确认写入外部系统", "reconcile": "核实写入结果", - "signal": "等外部系统回调", "review": "审核个人应用"} + "signal": "等外部系统回调", "review": "审核个人应用", "recall": "口径召回"} _ENV_TEXT = {"prod": "生产环境", "test": "测试环境", "simulated": "演练"} _background: set[asyncio.Task[Any]] = set() @@ -302,8 +305,9 @@ async def ensure_assignee_mail(users: list[str]) -> int: return made -async def notify_task(task_id: str, event: str, *, users: list[str]) -> int: +async def notify_task(task_id: str, event: str, *, users: list[str], exclude: frozenset[str] = frozenset()) -> int: """待办的通知（task.opened / reassigned / escalated / reminded）。users 是这次要通知到的人（派给的用户名）。返回推了几个渠道。 + exclude 是这一次已经推过别的事件的渠道（口径召回先推了 ledger.recall 的，notify_recall），同一个渠道只推一次。 管理员开了「默认发邮件给待办处理人」时，先给还没有个人邮件渠道的处理人建一个（ensure_assignee_mail），这一条就推得到。""" try: @@ -315,7 +319,7 @@ async def notify_task(task_id: str, event: str, *, users: list[str]) -> int: if task is None: return 0 run = await session.get(Run, task.run_id) - channels = await _channels(session, event, run, users) + channels = [c for c in await _channels(session, event, run, users) if c.id not in exclude] if not channels: return 0 base = await _base_url(session) @@ -338,6 +342,47 @@ async def notify_task(task_id: str, event: str, *, users: list[str]) -> int: task_id=task_id) +async def notify_recall(task_id: str, users: list[str]) -> int: + """口径召回的待办开出来了（app/ledger/recall）：口径升版、数真变了的那一种推 ledger.recall——订阅了它的全局渠道、答案主人 + 自己的个人渠道；然后照别的待办推 task.opened（已经推过 ledger.recall 的渠道不再推，同一个渠道只推一次）。目录、表结构变了 + 的那两种只推 task.opened。 + + 消息照模块说明的规矩（X6）：口径名（搭建时起的工作流名）、新旧版本号、这条待办列了几个答案、召回编号、去处理的链接；不放 + 旧口径的值、新口径的值、差值和原答案的值。返回推了几个渠道。""" + from app.ledger import recall + + try: + await ensure_assignee_mail(users) + except Exception: # noqa: BLE001 - 自动建渠道出错不挡这条待办的通知 + logger.exception("给待办 %s 的处理人自动建邮件渠道失败", task_id) + event = "ledger.recall" + async with SessionLocal() as session: + task = await session.get(Task, task_id) + if task is None: + return 0 + run = await session.get(Run, task.run_id) + reason = (task.correlation_key or "").split(":", 1)[0] + channels = await _channels(session, event, run, users) if reason == "caliber" else [] + origin = run.origin if run is not None and isinstance(run.origin, dict) else {} + name = str(await session.scalar(select(Workflow.name).where(Workflow.id == origin.get("workflow_id"))) + or "") if origin.get("workflow_id") else "" + base = await _base_url(session) if channels else None + pushed = 0 + if channels: + answers = len(recall._uses_of(origin, recall._owner_of_key(task.correlation_key or ""))) + lines = [("状态", "待处理"), ("口径", f"「{name[:80]}」"), + ("版本", f"v{origin.get('from_version')} → v{origin.get('to_version')}"), ("答案", f"{answers} 个"), + ("召回", f"#{task.run_id[:6]}")] + message = {"event": event, "status": "pending", "ok": False, + "title": recall.CHANGED_TITLE.format(name=name[:80] or "口径", version=origin.get("to_version")), + "lines": lines, "links": [("去处理", f"{base}/apps/inbox/{task_id}")] if base else [], + "subject": {"task_id": task_id, "run_id": task.run_id, "workflow_id": origin.get("workflow_id")}} + pushed = await _push(channels, message, subject_kind="task", subject_id=task_id, event=event, + run_id=task.run_id, task_id=task_id) + return pushed + await notify_task(task_id, "task.opened", users=users, + exclude=frozenset(c.id for c in channels)) + + async def notify_run(run_id: str, event: str) -> int: """应用实例结束、失败（run.finished / run.failed）。只推应用发起的运行。 diff --git a/backend/app/process/tasks.py b/backend/app/process/tasks.py index e2ee4842..0080edb5 100644 --- a/backend/app/process/tasks.py +++ b/backend/app/process/tasks.py @@ -15,6 +15,9 @@ 时限，但调度不对它们做提醒、升级、超时驳回——试运行不该把待办推到真实的处理人、上级那里。 - 个人应用提交审核（kind=review，N7）：没有 Approval、不恢复运行，挂在个人应用最近一次成功的刷新上；派给选定的搭建者 或组。在待办里只能退回，同意要在共享副本上走发布单，发布完自动记为已发布（blueprints/personal.py）。 +- 口径召回（kind=recall，docs/design/next-round/02-口径账本.md 做法 7）：没有 Approval、不恢复运行。口径升版后数真变了的 + 挂在召回运行上，派给答案的主人（开登录时每人一条，关登录时任何人）；动作「按新版重新出具」（走按口径出具同一套判法） + 和「知道了」。答案依据的目录、表结构变了的另两种只有「知道了」。开、收尾、内容都在 app/ledger/recall.py。 """ from __future__ import annotations @@ -163,6 +166,29 @@ async def open_review(session: AsyncSession, *, run: Run, app_id: str, blueprint return task +#: 口径召回的待办（app/ledger/recall.py）。没有 Approval，按 correlation_key（来由:运行[:附加]:派给的种类:名字）认同一条 +RECALL_KIND = "recall" +#: recall 待办的两种决定：按新版重新出具、知道了（目录、表结构那两种只有「知道了」） +RECALL_DECISIONS = ("reissue", "ack") + + +async def open_recall(session: AsyncSession, *, run: Run, title: str, owner: tuple[str, str], + correlation_key: str) -> Task: + """开一条 recall 待办（挂在 run 上），派给 owner：人、组（在这一刻展开成成员，同改派）、任何人；来源记 recall。调用方 + 提交，提交后 announce。""" + task = Task(run_id=run.id, node_id=None, app_id=None, app_version=None, kind=RECALL_KIND, title=title[:300], + policy={"kind": "any"}, correlation_key=correlation_key[:200]) + session.add(task) + await session.flush() + kind, name = owner + targets = await _expand(session, task, kind, name, RECALL_KIND) if kind in ("user", "group") and name \ + else [("anyone", "", RECALL_KIND)] + for k, n, source in dict.fromkeys(targets): + session.add(TaskAssignment(task_id=task.id, principal_kind=k, principal=n, source=source)) + await session.flush() + return task + + async def review_handler(session: AsyncSession, task: Task, principal: Any) -> bool: """这个人是不是这条审核待办的处理人（派给他或他的组、没撤回，同 handles）：只有他们能取个人蓝图的共享副本。""" return handles((await _assignments(session, [task.id])).get(task.id, []), principal) @@ -327,8 +353,15 @@ async def announce(task_id: str, event: EventType = EventType.TASK_OPENED) -> No return from app.notify.service import spawn + users = [a.principal for a in rows.get(task_id, []) if a.principal_kind == "user"] + if task.kind == RECALL_KIND: + # 口径召回：口径升版、数变了的推 ledger.recall，订阅了 task.opened 的渠道另推一条（同一个渠道只推一次） + from app.notify.service import notify_recall + + spawn(notify_recall(task_id, users)) + return # 推送放到后台：接收端挂住时超时加重试要几十秒，不能让运行停在「执行中」等它 - spawn(_notify(task_id, "task.opened", [a.principal for a in rows.get(task_id, []) if a.principal_kind == "user"])) + spawn(_notify(task_id, "task.opened", users)) # -------------------------------------------------------------------------- @@ -542,6 +575,8 @@ async def respond(task_id: str, principal: Any, *, decision: Any, values: dict[s return await _respond_reconcile(session, task, principal, decision, values, note) if task.kind == REVIEW_KIND: return await _respond_review(session, task, principal, decision, note) + if task.kind == RECALL_KIND: + return await _respond_recall(session, task, principal, decision, note) if task.status != "pending": raise TaskRefused(_closed(task.status)) if inquiry_busy(task.run_id): @@ -669,6 +704,54 @@ async def _respond_review(session: AsyncSession, task: Task, principal: Any, dec return {"task": await detail(task_id, principal), "resolved": True, "run": await _run(run_id)} +async def _respond_recall(session: AsyncSession, task: Task, principal: Any, decision: Any, + note: str | None) -> dict[str, Any]: + """recall 待办：「按新版重新出具」（app/ledger/recall.reissue，走按口径出具同一套判法；不能出具照实拒绝，待办留着）或 + 「知道了」。处理完记成已处理（条件 UPDATE，同 close_review），结论和出具的那次正式运行写进 outcome，另记一条审计：运行 + 已经封存，事件记不进去。""" + if task.status != "pending": + raise TaskRefused(_closed(task.status)) + reason = (task.correlation_key or "").split(":", 1)[0] + allowed = RECALL_DECISIONS if reason == "caliber" else ("ack",) + if decision not in allowed: + raise TaskRefused("这条待办只能「按新版重新出具」或「知道了」" if reason == "caliber" + else "这条待办只能「知道了」", status_code=422, code="task_decision_invalid") + rows = (await _assignments(session, [task.id])).get(task.id, []) + mine_id = _pick(task, rows, principal).id + task_id, run_id = task.id, task.run_id + await session.rollback() + outcome: dict[str, Any] = {"decision": decision, "note": (note or "").strip()[:2000]} + issued: Any = None + if decision == "reissue": + from app.ledger import recall + + try: + issued = await recall.reissue(task_id, principal) + except recall.RecallRefused as e: + raise TaskRefused(e.message, status_code=e.status_code, code=e.code) from e + outcome["issued_run"] = issued.id + async with SessionLocal() as session: + now = utcnow() + claimed = await session.execute( + update(Task).where(Task.id == task_id, Task.status == "pending") + .values(status="resolved", resolved_by=principal.username, resolved_at=now, outcome=outcome)) + if claimed.rowcount != 1: + await session.rollback() + raise TaskRefused(_closed("resolved")) + await session.execute(update(TaskAssignment).where(TaskAssignment.id == mine_id).values( + status="responded", responded_by=principal.username, responded_at=now, verified=bool(principal.verified), + response={"decision": decision, "note": outcome["note"]})) + await session.execute(update(TaskAssignment).where(TaskAssignment.task_id == task_id, + TaskAssignment.status == "pending") + .values(status="withdrawn")) + session.add(AuditEvent(actor=principal.username, verified=bool(principal.verified), + action=f"ledger.recall.{decision}", subject_kind="task", subject_id=task_id, + data={"run_id": run_id, **({"issued_run": issued.id} if issued is not None else {}), + **key_trace(principal)})) + await session.commit() + return {"task": await detail(task_id, principal), "resolved": True, "run": await _run(run_id)} + + async def close_reconcile(session: AsyncSession, action_id: str, principal: Any, outcome: str, note: str | None) -> str | None: """核实之后把这个动作的核实待办收掉（条件 UPDATE，和 tick、别人的回复抢同一行）。返回待办 id，没有返回 None。""" @@ -1102,6 +1185,10 @@ async def _describe(session: AsyncSession, tasks: list[Task], principal: Any) -> from app.blueprints.personal import review_payload payload = await review_payload(session, task) + elif task.kind == RECALL_KIND: + from app.ledger.recall import task_payload + + payload = await task_payload(session, task, principal) run = runs.get(task.run_id) out.append({ "id": task.id, "approval_id": task.approval_id, "run_id": task.run_id, "node_id": task.node_id, diff --git a/backend/tests/test_app_versions.py b/backend/tests/test_app_versions.py index a36765d9..a5039022 100644 --- a/backend/tests/test_app_versions.py +++ b/backend/tests/test_app_versions.py @@ -304,7 +304,9 @@ async def test_version_detail_has_form_and_usage(client): assert detail["usage"]["effects"] == 1 and detail["usage"]["formal_runs"] == 1 assert [a["name"] for a in detail["usage"]["apps"]] == ["周报看板"] ledger = (await client.get(f"/api/ledger/workflow-versions/{wf}/1")).json() - assert ledger == detail["usage"] + # 账本接口在同一份之上另加口径账本的用量（问数回答、导出、定时任务、召回，next-round 02 做法 8） + assert {k: ledger[k] for k in detail["usage"]} == detail["usage"] + assert ledger["answers"] == 0 and ledger["recall"] == {"groups": 0, "changed": 0, "recalled_from": 0} effects = (await client.get("/api/ledger/effects", params={"workflow_id": wf})).json()["items"] assert [(e["capability"], e["state"], e["workflow_version"]) for e in effects] == [("erp_pay", "confirmed", 1)] diff --git a/backend/tests/test_ledger_api.py b/backend/tests/test_ledger_api.py index 24cc969a..5a5a2001 100644 --- a/backend/tests/test_ledger_api.py +++ b/backend/tests/test_ledger_api.py @@ -237,7 +237,8 @@ async def test_stats_count_what_the_fixtures_did(client, monkeypatch): assert body["e3"]["chat_runs"] == 0 and body["e3"]["threshold"] == {"runs": 3, "conversations": 2} usage = {u["caliber"]: u for u in body["e4"]["usage"]} assert usage[f"{wf}@1"]["proven_runs"] == 1 - assert body["e4"]["per_publish"]["available"] is False + # 召回（第 4 波）：这一段时间里没有发布新版本，没有召回 + assert body["e4"]["per_publish"]["available"] is True and body["e4"]["per_publish"]["groups"] == 0 async def test_stats_take_the_run_ids_the_caller_names(monkeypatch): diff --git a/backend/tests/test_ledger_recall.py b/backend/tests/test_ledger_recall.py new file mode 100644 index 00000000..86c74aea --- /dev/null +++ b/backend/tests/test_ledger_recall.py @@ -0,0 +1,563 @@ +"""口径召回（next-round 02 做法 7，拍板 L6、L7；app/ledger/uses.py、recall.py）。 + +真的跑：虚构店铺库、mock 模型写报告。受管口径「周销售」v1 只算已支付的，问数据问过几周、正式运行跑过；把它发布成 v2 +（改一个筛选条件）之后： +- 计划：按（旧版本、新版本、参数）分组去重——问数回答绑定出的参数和正式运行的输入补齐缺省值之后是同一组；上限 recall_limit + 之内自动发起，超出的写「另有 M 组未重算」，手动接着算；再排一次、服务重启之后重算计划，算过的不再发起（幂等）； +- 召回运行：派生图里新旧两版的取数和口径卡各一份（前缀不撞），召回卡里算两版之差；同一份数据上两版各算一次； +- 收尾：数变了才开 recall 待办（挂在召回运行上，旧值、新值、差值、原答案当时的值都给），推 ledger.recall，消息里没有数； + 数没变的在原答案上记 ledger.recalled（已按 vN 核对，不影响），不开待办； +- 不能重算的照实记：新版入口要的参数这一组里没有、图里有人工节点（有副作用）； +- 待办「按新版重新出具」走按口径出具同一套判法、「知道了」； +- 目录里影响数字的一项被改了：冻结目录里看得到它的答案标「依据有变」，按新目录多出错误级问题的才开待办； +- 表结构变了：用过这张表的答案标「依据的表结构已变」，口径试算算不出来才开待办； +- E4：每次发布召回了几组、其中数变了的比例。 +数据都是编的。 +""" +from __future__ import annotations + +import pytest +from httpx import ASGITransport, AsyncClient +from sqlalchemy import select + +from app.core.events import EventType +from app.db.base import SessionLocal +from app.db.models import Run, RunEvent, Setting, Task, TaskAssignment +from app.engine.runner import run_manager +from app.ledger import match as ledger_match +from app.ledger import recall +from app.main import app +from tests import ledger_samples as S + +#: v2：已支付和已退款的都算（召回要算出差值的那种改法） +WITH_REFUNDED = ("SELECT SUM(amount) AS gmv FROM orders WHERE status IN ('paid', 'refunded') " + "AND paid_at >= '{{ input.start }}' AND paid_at < '{{ input.end }}'") +WEEK1 = {"start": "2026-09-01", "end": "2026-09-08"} + + +def week_sql(start: str, end: str) -> str: + return (f"select sum(o.amount) as 销售额 from orders o where o.paid_at >= '{start}' and o.status = 'paid' " + f"and o.paid_at < '{end}'") + + +@pytest.fixture(autouse=True) +async def engine_up(tmp_path): + await run_manager.setup() + path = tmp_path / "shop.db" + await S.fresh_source(path) + await ledger_settings(auto_issue=False, issue_daily_limit=100_000) + yield path + await ledger_match.drain() + await recall.drain() + await run_manager.shutdown() + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +async def ledger_settings(**values) -> None: + async with SessionLocal() as session: + row = await session.get(Setting, "ledger") + if row is not None: + await session.delete(row) + session.add(Setting(key="ledger", value=values)) + await session.commit() + + +async def answer(monkeypatch, start: str = "2026-09-01", end: str = "2026-09-08") -> Run: + """问数据问一周的销售额（独立写出、和口径等价的 SQL）。""" + return await S.explore(monkeypatch, week_sql(start, end), text="本周销售额 [[v:Q1.r0.销售额]]。", origin="chat") + + +async def publish_v2(wf: str, sql: str = WITH_REFUNDED, *, graph=None) -> None: + await S.publish(graph=graph or S.caliber_graph(sql), version=2, workflow_id=wf) + + +async def run_recall(wf: str, version: int = 2, *, batch: str = "auto") -> list[Run]: + """按计划发起一批、等召回运行跑完、等收尾做完。""" + ids = await recall.launch(wf, version, batch=batch) + done = [await S.wait(rid) for rid in ids] + await recall.drain() + return done + + +async def events(run_id: str, kind: EventType) -> list[dict]: + async with SessionLocal() as session: + return [d for d in (await session.execute(select(RunEvent.data).where( + RunEvent.run_id == run_id, RunEvent.type == kind).order_by(RunEvent.seq))).scalars()] + + +async def recall_tasks(run_id: str) -> list[Task]: + async with SessionLocal() as session: + return list((await session.execute(select(Task).where(Task.run_id == run_id, Task.kind == "recall"))).scalars()) + + +# -------------------------------------------------------------------------- +# 计划：分组、去重、上限、幂等 +# -------------------------------------------------------------------------- + + +async def test_plan_groups_answers_and_formal_runs_by_their_binding(monkeypatch): + wf = await S.publish() + a1 = await answer(monkeypatch) + a2 = await answer(monkeypatch) # 同一周再问一次：同一组 + a3 = await answer(monkeypatch, "2026-09-08", "2026-09-15") + await publish_v2(wf) + async with SessionLocal() as session: + plan = await recall.build_plan(session, wf, 2) + assert plan.problem is None and [p.old.node_id for p in plan.pairs[1]] == ["card"] + groups = {tuple(sorted(g.binding.items())): g for g in plan.groups} + week1 = groups[(("end", "2026-09-08"), ("start", "2026-09-01"), ("store", "all"))] + assert sorted(u.run_id for u in week1.uses) == sorted([a1.id, a2.id]), "同一组参数去重成一组（缺省值补齐后比）" + assert week1.old_version == 1 and week1.new_version == 2 and week1.problem is None + assert week1.inputs == {"start": "2026-09-01", "end": "2026-09-08", "store": "all"} + assert len(plan.groups) == 2 and len(plan.todo()) == 2 + assert {u.run_id for g in plan.groups for u in g.uses} == {a1.id, a2.id, a3.id} + assert all(u.kind == "answer" and u.turn_id is None for g in plan.groups for u in g.uses) + + +async def test_limit_continue_and_idempotent_replanning(monkeypatch): + await ledger_settings(auto_issue=False, issue_daily_limit=100_000, recall_limit=1) + wf = await S.publish() + await answer(monkeypatch) + await answer(monkeypatch, "2026-09-08", "2026-09-15") + await publish_v2(wf) + async with SessionLocal() as session: + before = await recall.preview(session, wf, 2) + assert before["groups"] == 2 and before["will_recompute"] == 1 and before["over_limit"] == 1 + assert before["versions"][0]["answers"] == 2 and before["caliber"] is True + first = await run_recall(wf) + assert len(first) == 1 and first[0].origin["batch"] == "auto" + # 再排一次（重复的发布钩子、服务重启后重算计划）：自动那一批已经发满，不再发起 + assert await recall.launch(wf, 2, batch="auto") == [] + async with SessionLocal() as session: + view = await recall.batches(session, wf, 2) + assert view["counts"]["pending"] == 1 and view["remaining"] == 1 and view["can_continue"] is True + assert sum(1 for g in view["groups"] if g.get("run_id")) == 1 + # 手动接着算：发起剩下那一组；再接着算就没有了 + second = await run_recall(wf, batch="manual") + assert len(second) == 1 and second[0].origin["batch"] == "manual" + async with SessionLocal() as session: + view = await recall.batches(session, wf, 2) + assert view["remaining"] == 0 and view["can_continue"] is False + assert await recall.launch(wf, 2, batch="manual") == [] + + +async def test_zero_limit_lists_but_does_not_recompute(monkeypatch): + await ledger_settings(auto_issue=False, issue_daily_limit=100_000, recall_limit=0) + wf = await S.publish() + await answer(monkeypatch) + await publish_v2(wf) + assert await recall.launch(wf, 2, batch="auto") == [] + async with SessionLocal() as session: + view = await recall.batches(session, wf, 2) + assert view["counts"]["pending"] == 1 and view["batch_size"] == 50, "上限是 0 时手动接着算按缺省的组数" + + +# -------------------------------------------------------------------------- +# 召回运行与收尾 +# -------------------------------------------------------------------------- + + +async def test_changed_numbers_open_a_recall_task_with_all_three_values(client, monkeypatch): + wf = await S.publish() + a1 = await answer(monkeypatch) + await publish_v2(wf) + [done] = await run_recall(wf) + assert done.status == "succeeded", done.error + assert done.origin_kind == "recall" and done.run_class == "exploratory" and done.workflow_id is None + ids = [n["id"] for n in done.graph["nodes"]] + assert ids == ["derived_in", "old_fetch", "old_card", "new_fetch", "new_card", "recall", "derived_out"] + nodes = {n["id"]: n for n in done.graph["nodes"]} + assert nodes["old_card"]["data"]["config"]["caliber_from"] == { + "workflow_id": wf, "workflow_version": 1, "node_id": "card", "nodes": {"fetch": "old_fetch"}} + assert nodes["new_card"]["data"]["config"]["caliber_from"]["workflow_version"] == 2 + assert nodes["derived_out"]["data"]["config"]["contract"] == {"metrics_from": "recall"} + assert not [n for n in done.graph["nodes"] if n["type"] in ("report", "llm", "agent")] + + values = await recall.card_values(done.id) + # v1 只算已支付：100 + 50.5 + 20；v2 加上这一周退款的 30 + 8 + assert values["old_0"]["value"] == 170.5 and values["new_0"]["value"] == 208.5 + assert values["diff_0"]["value"] == 38.0 and values["diff_0"]["rendered"] == "38.00元" + + [settled] = [d for d in await events(done.id, EventType.LEDGER_RECALLED) if d.get("role") == "recall"] + assert settled["changed"] is True and settled["answers"] == 1 and len(settled["tasks"]) == 1 + assert settled["post_seal"] is True + assert await events(a1.id, EventType.LEDGER_RECALLED) == [], "数变了的不在原答案上记「不影响」" + [task] = await recall_tasks(done.id) + assert task.title == "口径「周销售」升到 v2，答案的数变了" + res = await client.get(f"/api/tasks/{task.id}") + assert res.status_code == 200, res.text + payload = res.json()["payload"] + assert payload["kind"] == "recall" and payload["reason"] == "caliber" and payload["to_version"] == 2 + [metric] = payload["metrics"] + assert (metric["old"]["value"], metric["new"]["value"], metric["diff"]["value"]) == (170.5, 208.5, 38.0) + assert metric["changed"] is True and metric["old"]["metric"] == "old_0" and metric["diff"]["metric"] == "diff_0" + [ans] = payload["answers"] + assert ans["run_id"] == a1.id and ans["kind"] == "answer" + assert ans["original"][0]["value"] == 170.5 and ans["original"][0]["column"] == "销售额" + assert [a["decision"] for a in payload["actions"]] == ["reissue", "ack"] + assert payload["actions"][0]["available"] is True + # 三个数各自点得开：召回卡上的差值的出处链追到两版的口径卡 + chain = await client.get(f"/api/runs/{done.id}/evidence/metrics/recall/diff_0") + assert chain.status_code == 200, chain.text + assert {s["node_id"] for s in chain.json()["chain"] if s["step"] == "input"} == {"old_card", "new_card"} + + +async def test_unchanged_numbers_are_only_booked_on_the_answer(monkeypatch): + wf = await S.publish() + a1 = await answer(monkeypatch) + # v2 只换了报告怎么写，口径的数据部分没变 + graph = S.caliber_graph() + graph["nodes"][3]["data"]["config"]["instructions"] = "写周销售（新的写法）" + await publish_v2(wf, graph=graph) + [done] = await run_recall(wf) + assert done.status == "succeeded", done.error + assert await recall_tasks(done.id) == [] + [booked] = await events(a1.id, EventType.LEDGER_RECALLED) + assert booked == {"role": "answer", "changed": False, "recall_run": done.id, "workflow_id": wf, + "from_version": 1, "to_version": 2, "post_seal": True} + # 再收一次尾不会重复记 + assert await recall.settle(done.id) is None + assert len(await events(a1.id, EventType.LEDGER_RECALLED)) == 1 + + +async def test_incompatible_entry_and_side_effects_are_not_recomputed(monkeypatch): + wf = await S.publish() + await answer(monkeypatch) + graph = S.caliber_graph(WITH_REFUNDED) + graph["nodes"][0]["data"]["config"]["fields"].append({"name": "region", "required": True}) + await publish_v2(wf, graph=graph) + async with SessionLocal() as session: + [group] = (await recall.build_plan(session, wf, 2)).groups + assert group.problem is not None and group.problem.startswith("新版 v2 的入口要的参数这一组里没有或不合规") + assert await recall.launch(wf, 2) == [] + async with SessionLocal() as session: + view = await recall.batches(session, wf, 2) + assert view["counts"]["not_recomputable"] == 1 and view["groups"][0]["reason"] == group.problem + assert view["can_continue"] is False + + wf2 = await S.publish(name="周销售（要审批）") + await answer(monkeypatch) + human = S.caliber_graph(WITH_REFUNDED) + human["nodes"].insert(3, S.node("ok", "human", mode="approve", title="确认")) + await S.publish(graph=human, version=2, workflow_id=wf2) + async with SessionLocal() as session: + plan = await recall.build_plan(session, wf2, 2) + assert plan.groups and all("人工节点" in (g.problem or "") for g in plan.groups) + + +async def test_plan_from_formal_runs_and_exporters_as_owners(monkeypatch): + """正式运行也是用过旧版的答案；导出过报告的人也是答案的主人（关登录时待办派给任何人，主人照记在召回运行里）。""" + from app.apps.audit import record + from app.engine.launch import launch_run + + wf = await S.publish() + S.script(monkeypatch, "本周销售额 [[m:gmv]]。") + async with SessionLocal() as session: + formal = await launch_run(session, workflow_id=wf, run_class="formal", version=1, input_payload=WEEK1, + actor="analyst") + formal = await S.wait(formal.id) + assert formal.status == "succeeded", formal.error + async with SessionLocal() as session: + await record(session, action="report.exported", subject_kind="run", subject_id=formal.id, actor="reader", + data={}) + await session.commit() + await publish_v2(wf) + async with SessionLocal() as session: + [group] = (await recall.build_plan(session, wf, 2)).groups + usage = await recall.preview(session, wf, 2) + [use] = group.uses + assert use.kind == "formal" and use.exporters == ["reader"] + assert ("user", "analyst") in use.owners and ("user", "reader") in use.owners + assert usage["versions"][0]["exports"] == 1 and usage["versions"][0]["formal"] == 1 + [done] = await run_recall(wf) + [task] = await recall_tasks(done.id) + async with SessionLocal() as session: + [assignment] = (await session.execute(select(TaskAssignment).where(TaskAssignment.task_id == task.id))).scalars() + assert assignment.principal_kind == "anyone", "关登录时派给任何人" + from app.api.deps import Principal + from app.ledger.recall import task_payload + + async with SessionLocal() as session: + payload = await task_payload(session, await session.get(Task, task.id), + Principal(username=None, role="admin", mode="off", via="off")) + [ans] = payload["answers"] + assert ans["kind"] == "formal" and ans["original"][0]["value"] == 170.5 + + +# -------------------------------------------------------------------------- +# 待办的两个动作、通知 +# -------------------------------------------------------------------------- + + +async def test_reissue_and_acknowledge(client, monkeypatch): + wf = await S.publish() + await answer(monkeypatch) + await answer(monkeypatch, "2026-09-01", "2026-09-05") # 这几天也有退款：两组的数都变了 + await publish_v2(wf) + runs = await run_recall(wf) + tasks = [t for r in runs for t in await recall_tasks(r.id)] + assert len(tasks) == 2 + S.script(monkeypatch, "本周销售额 [[m:gmv]]。") + res = await client.post(f"/api/tasks/{tasks[0].id}/respond", json={"decision": "reissue"}) + assert res.status_code == 200, res.text + body = res.json()["task"] + assert body["status"] == "resolved" and body["outcome"]["decision"] == "reissue" + async with SessionLocal() as session: + issued = await session.get(Run, body["outcome"]["issued_run"]) + assert issued.origin_kind == "ledger_issue" and issued.run_class == "formal" and issued.version == 2 + assert issued.origin["recall_run"] == tasks[0].run_id and issued.origin["how"] == "recall" + assert issued.input["start"] == "2026-09-01" and issued.input["end"] in ("2026-09-08", "2026-09-05") + await S.wait(issued.id) + bad = await client.post(f"/api/tasks/{tasks[1].id}/respond", json={"decision": "approve"}) + assert bad.status_code == 422 + ok = await client.post(f"/api/tasks/{tasks[1].id}/respond", json={"decision": "ack", "note": "知道了"}) + assert ok.status_code == 200 and ok.json()["task"]["outcome"]["decision"] == "ack" + again = await client.post(f"/api/tasks/{tasks[1].id}/respond", json={"decision": "ack"}) + assert again.status_code == 409 + + +async def test_recall_notification_has_no_numbers(monkeypatch): + from app.core.crypto import encrypt + from app.db.models import NotifyChannel + from app.notify import service + + sent: list[dict] = [] + + async def capture(channel, message): + sent.append({"channel": channel.name, **message}) + from app.notify.core import Outcome + + return Outcome(True, 200, 1, None) + + monkeypatch.setattr("app.notify.send", capture) + async with SessionLocal() as session: + row = NotifyChannel(name="召回-全局", scope="global", url=encrypt("https://hooks.example.com/recall-7d"), + events=["ledger.recall", "task.opened"]) + session.add(row) + await session.commit() + channel_id = row.id + try: + wf = await S.publish() + await answer(monkeypatch) + await publish_v2(wf) + await run_recall(wf) + await service.drain() + finally: + async with SessionLocal() as session: + await session.delete(await session.get(NotifyChannel, channel_id)) + await session.commit() + mine = [m for m in sent if m["channel"] == "召回-全局"] + assert [m["event"] for m in mine] == ["ledger.recall"], "订阅了两样的渠道只推一次" + # 看给人读的部分（标题、各行）：编号是十六进制的 id，不拿来比 + text = mine[0]["title"] + "".join(f"{k}{v}" for k, v in mine[0]["lines"]) + for number in ("170.5", "208.5", "38.0", "38.00"): + assert number not in text, f"消息里不放数：{number}" + assert dict(mine[0]["lines"]) == {"状态": "待处理", "口径": "「周销售」", "版本": "v1 → v2", "答案": "1 个", + "召回": dict(mine[0]["lines"])["召回"]} + assert "周销售" in mine[0]["title"] and "v2" in mine[0]["title"] + + +# -------------------------------------------------------------------------- +# 目录事实被改、表结构变了 +# -------------------------------------------------------------------------- + + +async def _catalog(source_id: str, notes: dict) -> None: + """探查一次表结构（查询快照才冻结得进目录）、写这张表的目录。""" + from app.data import catalog + from app.data.engine import engines + from app.data.introspect import introspect + from app.db.models import DataSource + + async with SessionLocal() as session: + row = await session.get(DataSource, source_id) + row.schema_cache = await introspect(row) + await session.commit() + await engines.invalidate(source_id) + async with SessionLocal() as session: + entry = await catalog.read_entry(session, source_id, "orders") + await catalog.write_entry(session, source_id, "orders", notes, if_version=entry.version if entry else 0, + actor="tester") + + +def _item(value, status="confirmed"): + return {"value": value, "source": "human", "status": status} + + +async def test_catalog_fact_change_marks_answers_and_notifies_only_on_new_errors(engine_up, monkeypatch): + from app.data import catalog + + source_id = await S.set_source(engine_up) + await _catalog(source_id, {"business_date": _item({"column": "paid_at"}), + "columns": {"amount": {"measure": _item("flow")}}}) + row = await answer(monkeypatch) + unrelated = await S.explore(monkeypatch, S.ELSEWHERE, text="门店 [[v:Q1.r0.n]] 家。", origin="chat") + async with SessionLocal() as session: + entry = await catalog.read_entry(session, source_id, "orders") + # 改成存量：按新目录，对存量列求和、分组里没有业务日期是错误级问题（跨期加总存量） + await catalog.save_human_edit(session, source_id, "orders", + {**entry.notes, "columns": {"amount": {"measure": _item("stock")}}}, + if_version=entry.version, actor="tester") + await recall.drain() + [marked] = await events(row.id, EventType.LEDGER_BASIS_CHANGED) + assert marked["kind"] == "catalog" and marked["table"] == "orders" + assert marked["items"] == [{"path": "columns.amount.measure", "label": "列 amount 的度量类型", "change": "changed"}] + assert marked["new_errors"] == 1 and marked["checks"][0]["code"] == "stock_summed" + assert await events(unrelated.id, EventType.LEDGER_BASIS_CHANGED) == [], "没查过这张表的不标" + [task] = await recall_tasks(row.id) + assert task.correlation_key.startswith("catalog:") + async with SessionLocal() as session: + from app.api.deps import Principal + + payload = await recall.task_payload(session, task, Principal(username=None, role="admin", mode="off", via="off")) + assert payload["reason"] == "catalog" and payload["basis"]["new_errors"] == 1 + assert [a["decision"] for a in payload["actions"]] == ["ack"] + + # 只改名称说明（不影响数字的项）：不标 + async with SessionLocal() as session: + entry = await catalog.read_entry(session, source_id, "orders") + await catalog.save_human_edit(session, source_id, "orders", {**entry.notes, "label": _item("订单")}, + if_version=entry.version, actor="tester") + await recall.drain() + assert len(await events(row.id, EventType.LEDGER_BASIS_CHANGED)) == 1 + + +def test_fact_changes_only_counts_items_that_affect_numbers(): + before = {"valid_filter": _item("status <> 'open'"), "label": _item("订单"), + "columns": {"amount": {"measure": _item("flow"), "label": _item("金额")}}} + after = {"valid_filter": {**_item("status <> 'open'"), "status": "rejected"}, "label": _item("订单表"), + "columns": {"amount": {"label": _item("金额")}}} + changes = recall.fact_changes(before, after) + assert [(c["path"], c["change"]) for c in changes] == [("valid_filter", "rejected"), + ("columns.amount.measure", "removed")] + assert recall.fact_changes(after, after) == [] + + +async def test_structure_change_marks_answers_and_probe_failure_notifies(engine_up, monkeypatch): + import sqlite3 + + from app.data import source_changes as sc + from app.db.models import DataSource, WorkflowVersion + + wf = await S.publish() + async with SessionLocal() as session: + row = (await session.execute(select(WorkflowVersion).where(WorkflowVersion.workflow_id == wf))).scalar_one() + row.published_by = "publisher" + await session.commit() + a1 = await answer(monkeypatch) + # 列真的删掉了：口径的取数按这一版再算就算不出来 + db = sqlite3.connect(engine_up) + db.executescript("ALTER TABLE orders DROP COLUMN amount;") + db.close() + async with SessionLocal() as session: + source = (await session.execute(select(DataSource).where(DataSource.name == S.CURRENT["source"]))).scalar_one() + change, _ = await sc.record(session, source, sc.NewChange(kind="column_removed", table="orders", column="amount", + severity="action", key="v2"), + sc.Detected(job_kind="introspect")) + await session.commit() + out = sc.change_out(change) + assert recall.basis_kind(out) == "structure" + assert await recall.process_change(out) == 1 + [marked] = await events(a1.id, EventType.LEDGER_BASIS_CHANGED) + assert marked["kind"] == "structure" and marked["change"] == out["id"] and marked["column"] == "amount" + async with SessionLocal() as session: + [probe] = [r for r in (await session.execute(select(Run).where(Run.origin_kind == "recall"))).scalars() + if (r.origin or {}).get("change") == out["id"]] + assert probe.origin["probe"] == 1 and probe.origin["workflow_id"] == wf and probe.origin["inputs_from"] == "answer" + probe = await S.wait(probe.id) + assert probe.status == "failed" + await recall.drain() + [task] = await recall_tasks(probe.id) + assert task.title == "口径「周销售」v1 依据的表结构变了，试算没有算出来" + # 同一条变化再处理一次：不再标、不再试算 + assert await recall.process_change(out) == 0 + async with SessionLocal() as session: + probes = (await session.execute(select(Run).where(Run.origin_kind == "recall"))).scalars().all() + assert len([p for p in probes if (p.origin or {}).get("change") == out["id"]]) == 1 + + +# -------------------------------------------------------------------------- +# 版本用量、预告接口、E4 +# -------------------------------------------------------------------------- + + +async def test_usage_preview_batches_and_stats_endpoints(client, monkeypatch): + wf = await S.publish() + await answer(monkeypatch) + usage = (await client.get(f"/api/ledger/workflow-versions/{wf}/1")).json() + assert usage["answers"] == 1 and usage["exports"] == 0 and usage["recall"] == {"groups": 0, "changed": 0, + "recalled_from": 0} + await publish_v2(wf) + preview = (await client.get(f"/api/workflows/{wf}/recall-preview", params={"version": 2})).json() + assert preview["groups"] == 1 and preview["will_recompute"] == 1 and preview["over_limit"] == 0 + await run_recall(wf) + view = (await client.get(f"/api/ledger/recall/{wf}/2")).json() + assert view["counts"]["changed"] == 1 and view["groups"][0]["state"] == "changed" + assert view["groups"][0]["changed_metrics"] == ["销售额"] + again = await client.post(f"/api/ledger/recall/{wf}/2/continue") + assert again.status_code == 409 + usage = (await client.get(f"/api/ledger/workflow-versions/{wf}/2")).json() + assert usage["recall"]["groups"] == 1 and usage["recall"]["changed"] == 1 + old = (await client.get(f"/api/ledger/workflow-versions/{wf}/1")).json() + assert old["recall"]["recalled_from"] == 1 + stats = (await client.get("/api/ledger/stats")).json()["e4"]["per_publish"] + mine = next(p for p in stats["publishes"] if p["workflow_id"] == wf) + assert mine == {"caliber": f"{wf}@2", "workflow_id": wf, "version": 2, "groups": 1, "settled": 1, "changed": 1, + "failed": 0, "changed_rate": 1.0} + + +async def test_publish_hook_queues_recall_when_automatic(client, monkeypatch): + from app.core.config import settings + + monkeypatch.setattr(settings, "scheduler", True) + wf = await S.publish() + await answer(monkeypatch) + await S.publish(graph=S.caliber_graph(WITH_REFUNDED), version=2, workflow_id=wf, level="governed") + async with SessionLocal() as session: + from app.db.models import Workflow + + (await session.get(Workflow, wf)).published_version = 1 + await session.commit() + res = await client.post(f"/api/workflows/{wf}/publish", json={"level": "published", "version": 2}) + assert res.status_code == 200 and res.json()["ok"], res.text + await recall.drain() + async with SessionLocal() as session: + runs = await recall.recall_runs(session, wf, 2) + assert len(runs) == 1 and runs[0].origin["batch"] == "auto" + await S.wait(runs[0].id) + await recall.drain() + + +async def test_recall_graph_is_deterministic_and_events_have_offline_labels(monkeypatch): + """同样的组永远得到同一张召回图；两个新事件在可验证报告的离线核对页、封存后事件摘要里有说法（不带数）。""" + import re + from pathlib import Path + + from app.bundle.build import _summary + + wf = await S.publish() + await answer(monkeypatch) + await publish_v2(wf) + async with SessionLocal() as session: + plan = await recall.build_plan(session, wf, 2) + [group] = plan.groups + a, meta = recall.build_graph(group, plan.pairs[1]) + b, _ = recall.build_graph(group, plan.pairs[1]) + assert a.graph == b.graph and a.kind == "recall" + assert meta == [{"k": 0, "card": "card", "metric": "gmv", "name": "销售额", "unit": "元", "decimals": 2, + "format": "thousands", "expression": "cell(nodes.fetch, 0, 'gmv')"}] + card = next(n for n in a.graph["nodes"] if n["id"] == "recall")["data"]["config"] + assert [m["id"] for m in card["metrics"]] == ["old_0", "new_0", "diff_0"] and card["on_missing"] == "null" + + script = (Path(__file__).resolve().parents[1] / "app" / "bundle" / "verifier.js").read_text(encoding="utf-8") + block = re.search(r"const EVENT_LABEL = \{(.*?)\n \}", script, re.S) + labels = dict(re.findall(r"'([a-z_.]+)': '([^']+)'", block.group(1))) + assert labels["ledger.recalled"] == "口径召回核对" and labels["ledger.basis_changed"] == "答案的依据有变" + assert labels["ledger.matched"] == "比对已发布口径" + assert _summary("ledger.recalled", {"role": "answer", "changed": False, "to_version": 2}) == "已按 v2 核对，不影响" + assert _summary("ledger.basis_changed", {"kind": "structure", "items": [{"label": "x"}]}) == "依据的表结构有变（1 项）" diff --git a/backend/tests/test_notify_events.py b/backend/tests/test_notify_events.py index 4957bfb5..cc8ec89e 100644 --- a/backend/tests/test_notify_events.py +++ b/backend/tests/test_notify_events.py @@ -41,7 +41,8 @@ PUSHED_THIS_ROUND = { "key.expiring": {"global", "user"}, # 第 4 波：W4-d 打开 source.changed 时在这里加它推给的几条路 # （两行之间隔一行不动的，两个包各改各的、合并不撞） - # 第 4 波：W4-a 打开 ledger.recall 时在这里加它推给的几条路 + # 第 4 波 W4-a：口径召回数真变了的待办推全局渠道和答案主人自己的个人渠道（notify_recall） + "ledger.recall": {"global", "user"}, } @@ -120,19 +121,24 @@ async def test_events_endpoint_says_who_receives_each_event(client): assert [r["event"] for r in rows] == list(service.EVENTS) by = {r["event"]: r for r in rows} assert by["eval.regressed"] == {"event": "eval.regressed", "label": "定时评测有退步", "scopes": ["global"]} - assert by["ledger.recall"]["scopes"] == [] + assert by["ledger.recall"]["scopes"] == ["global", "user"] assert by["review.done"]["scopes"] == ["global", "user"] -async def test_events_not_pushed_yet_cannot_be_subscribed(client): +#: 登记了、还没有推送的演示事件（第 4 波之后本轮的六个都推了，用它守「不收订阅、不列」这条规矩） +UNPUSHED = "demo.unpushed" + + +async def test_events_not_pushed_yet_cannot_be_subscribed(client, monkeypatch): + monkeypatch.setitem(service.EVENTS, UNPUSHED, service.Event("还没推送的演示事件", ())) resp = await client.post("/api/notify/channels", json={ - "name": "事件登记-评测", "url": URL, "events": ["task.opened", "ledger.recall"]}) + "name": "事件登记-评测", "url": URL, "events": ["task.opened", UNPUSHED]}) assert resp.status_code == 422, resp.text detail = resp.json()["detail"] - assert "口径升版，答案的数变了" in detail and "还没有推送" in detail + assert "还没推送的演示事件" in detail and "还没有推送" in detail unknown = await client.post("/api/notify/channels", json={"name": "事件登记-未知", "url": URL, "events": ["nope"]}) assert unknown.status_code == 422 - assert "口径升版，答案的数变了" not in unknown.json()["detail"], "可选的里面不列还没有推送的" + assert "还没推送的演示事件" not in unknown.json()["detail"], "可选的里面不列还没有推送的" assert "待办打开" in unknown.json()["detail"] @@ -163,16 +169,17 @@ async def test_scope_mismatch_is_refused_only_for_newly_ticked_events(client): async def test_channels_only_walk_the_paths_the_event_is_registered_for(monkeypatch): monkeypatch.setitem(service.EVENTS, "demo.personal", service.Event("只推个人的演示事件", ("user",))) + monkeypatch.setitem(service.EVENTS, UNPUSHED, service.Event("还没推送的演示事件", ())) async with SessionLocal() as session: row = NotifyChannel(name="事件登记-全局", scope="global", url=encrypt(URL), - events=["demo.personal", "task.opened", "ledger.recall"]) + events=["demo.personal", "task.opened", UNPUSHED]) session.add(row) await session.commit() try: ids = lambda found: [c.id for c in found] # noqa: E731 assert row.id not in ids(await service._channels(session, "demo.personal", None, [])), "全局渠道这条路没登记" assert row.id in ids(await service._channels(session, "task.opened", None, [])) - assert await service._channels(session, "ledger.recall", None, []) == [], "还没有推送的：谁都不推" + assert await service._channels(session, UNPUSHED, None, []) == [], "还没有推送的：谁都不推" with pytest.raises(ValueError, match="没有登记"): await service._channels(session, "demo.unregistered", None, []) finally: