From e15e94f93e6583eedb7cd1ccd142848214506f3b Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Thu, 12 Nov 2026 12:58:12 +0800 Subject: [PATCH] =?UTF-8?q?=E5=8F=A3=E5=BE=84=E5=B7=A5=E4=BD=9C=E5=8F=B0?= =?UTF-8?q?=EF=BC=9A=E7=BB=93=E6=9E=84=E5=8C=96=E8=AF=AD=E4=B9=89=E5=AF=BC?= =?UTF-8?q?=E5=85=A5=E3=80=81=E8=B5=B7=E8=8D=89=E7=BB=AD=E8=B7=91=E6=81=A2?= =?UTF-8?q?=E5=A4=8D=E4=B8=8E=E6=9C=AC=E4=BD=93=E5=85=B3=E7=B3=BB=E7=BC=96?= =?UTF-8?q?=E6=8E=92?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增口径结构化语义定义、分批解析与 SQL 生成管道(semantic / semantic_import / semantic_sql) - 支持起草断点续跑恢复、指标自动匹配对齐与存量口径语义迁移面板(MigrationPanel / SemanticPanel) - 口径草稿与变更流程接入候选本体边提议、审阅确认与一键修改保护,同步更新可移植 SQLite 基线校验 --- backend/app/api/ledger.py | 439 ++++++++++++++- backend/app/api/workflows.py | 2 +- backend/app/ledger/authoring.py | 483 +++++++++++++++-- backend/app/ledger/changes.py | 33 ++ backend/app/ledger/continuation.py | 76 +++ backend/app/ledger/docdraft.py | 8 + backend/app/ledger/draftplan.py | 81 ++- backend/app/ledger/jobs.py | 11 +- backend/app/ledger/metric_matching.py | 18 + backend/app/ledger/migration.py | 70 +++ backend/app/ledger/nlchange.py | 158 +++++- backend/app/ledger/semantic.py | 153 ++++++ backend/app/ledger/semantic_import.py | 155 ++++++ backend/app/ledger/semantic_sql.py | 73 +++ .../tests/fixtures/oee_metric_definitions.md | 107 ++++ .../tests/fixtures/recipes/p2_baseline.json | 1 + backend/tests/sqlite_baseline.py | 17 + backend/tests/test_authoring_recovery.py | 256 +++++++++ backend/tests/test_definition_batches.py | 249 +++++++++ backend/tests/test_ledger_nlchange.py | 13 +- backend/tests/test_recipe_contract.py | 4 +- backend/tests/test_recipe_engine_p3.py | 6 +- backend/tests/test_semantic_pipeline.py | 360 +++++++++++++ deploy/agentlab.env | 10 +- frontend/src/api/caliberLedger.ts | 17 + frontend/src/pages/ledger/CaliberDrafts.tsx | 172 ++++-- frontend/src/pages/ledger/MigrationPanel.tsx | 43 ++ frontend/src/pages/ledger/SemanticPanel.tsx | 498 ++++++++++++++++++ 28 files changed, 3376 insertions(+), 137 deletions(-) create mode 100644 backend/app/ledger/continuation.py create mode 100644 backend/app/ledger/metric_matching.py create mode 100644 backend/app/ledger/migration.py create mode 100644 backend/app/ledger/semantic.py create mode 100644 backend/app/ledger/semantic_import.py create mode 100644 backend/app/ledger/semantic_sql.py create mode 100644 backend/tests/fixtures/oee_metric_definitions.md create mode 100644 backend/tests/sqlite_baseline.py create mode 100644 backend/tests/test_authoring_recovery.py create mode 100644 backend/tests/test_definition_batches.py create mode 100644 backend/tests/test_semantic_pipeline.py create mode 100644 frontend/src/pages/ledger/MigrationPanel.tsx create mode 100644 frontend/src/pages/ledger/SemanticPanel.tsx diff --git a/backend/app/api/ledger.py b/backend/app/api/ledger.py index f17d687c..4b0f9b88 100644 --- a/backend/app/api/ledger.py +++ b/backend/app/api/ledger.py @@ -75,7 +75,7 @@ from app.db.models_process import ActionRecord logger = logging.getLogger(__name__) from app.ledger.authoring import ( - _can_repair, _columns_digest, _draft_payload, _entry_notes, _focus_tables, _fresh_metrics, _merge_questions, _merged_fields, _problems_text, _published_metrics, _question_count, _question_key, _schema_material, _source_tables, _status_for + require_authoring_source, _can_repair, _columns_digest, _draft_payload, _entry_notes, _focus_tables, _fresh_metrics, _merge_questions, _merged_fields, _problems_text, _published_metrics, _question_count, _question_key, _schema_material, _source_tables, _status_for ) router = APIRouter(prefix="/api/ledger", tags=["ledger"]) @@ -1263,9 +1263,12 @@ def _draft_summary(draft: CaliberDraft, *, graph: dict[str, Any] | None = None) card = node_map(graph).get(card_id or (cards[0]["id"] if cards else "")) or {} payload = {**payload, "metrics": (card.get("config") or {}).get("metrics") or []} metrics = payload.get("metrics") + definitions = payload.get("semantic_inventory") return {"id": draft.id, "title": draft.title, "status": draft.status, "stage": docdraft.stage_of(draft.status), "metric_count": len(metrics) if isinstance(metrics, list) else 0, + **({"definition_count": sum(item.get("kind") == "metric" for item in definitions.get("items", []))} + if isinstance(definitions, dict) else {}), "question_count": _question_count(payload), "workflow_id": draft.workflow_id, "created_at": draft.created_at.isoformat() if draft.created_at else None, "error": draft.error} @@ -1340,7 +1343,7 @@ async def _draft_view(session: AsyncSession, draft: CaliberDraft, *, ensure_reco job = await session.get(CaliberDraftJob, draft.id) if job: view["job"] = {"status": job.status, "attempt": job.attempt, "error": job.error, - "retryable": draft.status == "failed" and not draft.workflow_id + "retryable": draft.status == "failed" and (not draft.workflow_id or payload.get("finalization_pending", False)) and job.status not in ("queued", "running")} return view @@ -1769,6 +1772,151 @@ class DraftExtractIn(BaseModel): fixed_filters: str = Field(default="", max_length=1000) +@router.post("/drafts/definitions", status_code=201) +async def import_definitions(payload: DraftExtractIn, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """Import source definitions first. No model calls, SQL execution, or publication.""" + draft = await _definition_draft(payload, session) + session.add(draft) + await session.commit() + return {"draft_id": draft.id, "status": draft.status} + + +async def _definition_draft(payload: DraftExtractIn, session: AsyncSession) -> CaliberDraft: + from app.api.deps import current_principal, ensure_role + from app.ledger.semantic_import import inventory + from app.core.artifact_store import put_json + + ensure_role("builder") + docs = [] + for doc_id in dict.fromkeys(payload.documents): + doc = await session.get(Document, doc_id) + if doc is None: + raise HTTPException(422, "文档不存在,可能已被删除") + docs.append(doc) + documents = [{"document_id": d.id, "title": d.title or d.source, "content": d.content or ""} for d in docs] + catalog = inventory(documents) + if not catalog["items"]: + raise HTTPException(422, "没有识别到指标定义表。请使用以「指标/量/符号」为首列的 Markdown 表格,或使用原有 AI 起草入口") + if payload.datasource_id and await session.get(DataSource, payload.datasource_id) is None: + raise HTTPException(422, "数据源不存在") + artifact = await put_json({"documents": documents, "inventory": catalog}, kind="semantic_definitions") + draft = CaliberDraft(title=_doc_title(docs), collection=payload.collection or docs[0].collection or "default", + datasource_id=payload.datasource_id, status="path_pending", + payload={"semantic_inventory": catalog, "definition_artifact": artifact, + "documents": [{"document_id": d.id, "title": d.title, "chunk_ids": []} for d in docs], + "metrics": [], "open_questions": [], "selected_definition_ids": []}, + origin={"documents": list(dict.fromkeys(payload.documents)), "guidance": payload.guidance, + "fixed_filters": payload.fixed_filters}, created_by=current_principal().username) + return draft + + +@router.get("/drafts/{draft_id}/migration") +async def migration_audit(draft_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + from app.api.deps import ensure_role + from app.ledger.migration import audit + + ensure_role("builder") + draft = await _draft_or_404(session, draft_id) + workflow = await session.get(Workflow, draft.workflow_id) if draft.workflow_id else None + if workflow is None: + raise HTTPException(409, "这份草稿尚未绑定取数图,请先从定义清单实现指标") + available = {s.name for s in (await session.execute(select(DataSource))).scalars() if s.enabled and _source_tables(s)} + result = audit(workflow.graph or {}, workflow_id=workflow.id, version=workflow.version, sources=available) + result["can_rebuild"] = bool((draft.origin or {}).get("documents") or (draft.payload or {}).get("documents")) + return result + + +class MigrationIn(BaseModel): + expected_revision: int = Field(ge=1) + + +@router.post("/drafts/{draft_id}/migration", status_code=201) +async def prepare_migration(draft_id: str, payload: MigrationIn, + session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """Create an independent source-definition draft; no old graph/SQL/publication is changed.""" + from app.api.deps import ensure_role + + ensure_role("builder") + old = await _draft_or_404(session, draft_id) + workflow = await session.get(Workflow, old.workflow_id) if old.workflow_id else None + if workflow is None or workflow.version != payload.expected_revision: + raise HTTPException(409, "旧口径已变化,请重新检查后准备迁移") + for existing in (await session.execute(select(CaliberDraft).where(CaliberDraft.status != "rejected"))).scalars(): + migration = (existing.origin or {}).get("migration_from", {}) + if migration.get("draft_id") == old.id and migration.get("revision") == workflow.version: + return {"draft_id": existing.id, "status": existing.status} + origin = old.origin or {} + docs = origin.get("documents") or [d["document_id"] for d in (old.payload or {}).get("documents", [])] + if not docs: + raise HTTPException(409, "旧口径没有来源文档,请先导入定义文档,或在原草稿中逐项补齐问数设置") + source = await session.get(DataSource, old.datasource_id) if old.datasource_id else None + draft = await _definition_draft(DraftExtractIn( + documents=docs, datasource_id=source.id if source else None, + guidance=str(origin.get("guidance") or "")[:2000], fixed_filters=str(origin.get("fixed_filters") or "")[:1000]), session) + draft.title = f"{old.title} · 迁移定义" + draft.origin = {**draft.origin, "migration_from": {"draft_id": old.id, "workflow_id": workflow.id, + "revision": workflow.version}} + # Preserve the exact old graph as review material, never as executable fallback or formula authority. + from app.core.artifact_store import put_json + draft.payload = {**draft.payload, "migration_baseline": await put_json( + {"graph": workflow.graph, "workflow_id": workflow.id, "revision": workflow.version}, kind="migration_baseline")} + session.add(draft) + await session.commit() + return {"draft_id": draft.id, "status": draft.status} + + +class ImplementDefinitionsIn(BaseModel): + selected: list[str] | None = Field(default=None, min_length=1, max_length=2000) + datasource_id: str = Field(min_length=1, max_length=32) + + +@router.post("/drafts/{draft_id}/implement", status_code=202) +async def implement_definitions(draft_id: str, payload: ImplementDefinitionsIn, background: BackgroundTasks, + session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + from app.api.deps import ensure_role + from app.db.models import CaliberDraftJob + from sqlalchemy import update + + ensure_role("builder") + draft = await _draft_or_404(session, draft_id) + stored = dict(draft.payload or {}) + catalog = stored.get("semantic_inventory") or {} + from app.ledger.semantic_import import source_matches + + docs = [await session.get(Document, doc_id) for doc_id in catalog.get("document_hashes", {})] + if any(doc is None or doc.status != "ready" for doc in docs) or not source_matches( + catalog, [{"document_id": doc.id, "content": doc.content} for doc in docs if doc is not None] + ): + raise HTTPException(409, "原文已更新或尚未处理完成,请重新导入定义后再实现") + from app.ledger.semantic_import import implementation_scope + + try: + selected, issues = implementation_scope(catalog, payload.selected) + except ValueError as e: + raise HTTPException(422, str(e)) from None + if not selected: + raise HTTPException(422, "没有可自动实现的指标,请先补充数据来源或确认同名口径") + source = await session.get(DataSource, payload.datasource_id) + try: + require_authoring_source(source) + except ValueError as exc: + raise HTTPException(422, str(exc)) from None + stored["selected_definition_ids"] = selected + stored["implementation_mode"] = "all" if payload.selected is None else "selected" + stored["definition_issues"] = issues + stored["implementation_progress"] = {"total": len(selected), "completed": 0, "pending": len(selected), + "blocked": 0, "active_names": []} + result = await session.execute(update(CaliberDraft).where( + CaliberDraft.id == draft_id, CaliberDraft.status == "path_pending", CaliberDraft.workflow_id.is_(None) + ).values(status="drafting", datasource_id=source.id, payload=stored)) + if result.rowcount != 1: + raise HTTPException(409, "这份定义已开始生成,请刷新查看进度") + session.add(CaliberDraftJob(id=draft_id)) + await session.commit() + background.add_task(draft_jobs.start, draft_id) + return {"draft_id": draft_id, "status": "drafting"} + + @router.post("/drafts/extract", status_code=201) async def draft_extract(payload: DraftExtractIn, background: BackgroundTasks, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: @@ -1791,10 +1939,10 @@ async def draft_extract(payload: DraftExtractIn, background: BackgroundTasks, if missing: raise HTTPException(422, f"文档不存在,可能已被删除:{'、'.join(missing)}") source = await session.get(DataSource, payload.datasource_id) - if source is None: - raise HTTPException(422, "数据源不存在,可能已被删除") - if not _source_tables(source): - raise HTTPException(422, "这个数据源还没有表结构:先探查表结构,再起草口径") + try: + require_authoring_source(source) + except ValueError as exc: + raise HTTPException(422, str(exc)) from None docs = [found[doc_id] for doc_id in doc_ids] draft = CaliberDraft(title=_doc_title(docs), collection=payload.collection or (docs[0].collection or "default"), @@ -2323,6 +2471,285 @@ def _with_change(view: dict[str, Any], change: Any, *, revision: int | None) -> return {**view, "change": changes.summary(change, applied=bool(change.applied), revision=revision)} +class ContinueDefinitionsIn(BaseModel): + expected_revision: int = Field(ge=0) + guidance: str = Field(default="", max_length=2000) + selected: list[str] | None = Field(default=None, min_length=1, max_length=2000) + datasource_id: str | None = Field(default=None, min_length=1, max_length=32) + + +@router.post("/drafts/{draft_id}/continue", status_code=202) +async def continue_definitions(draft_id: str, payload: ContinueDefinitionsIn, background: BackgroundTasks, + session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """Generate missing definitions as an independent candidate; never overwrite reviewed SQL.""" + from app.api.deps import ensure_role, current_principal + from app.core.artifact_store import canonical_json, content_hash + from app.db.models import CaliberDraftJob + from sqlalchemy import update + + ensure_role("builder") + draft = await _draft_or_404(session, draft_id) + if draft.status not in ("questions_open", "ready", "path_pending", "failed"): + raise HTTPException(409, "请等待当前任务完成后再继续生成") + workflow = await session.get(Workflow, draft.workflow_id) if draft.workflow_id else None + revision = workflow.version if workflow else 0 + if revision != payload.expected_revision: + raise HTTPException(409, "口径已发生变化,请刷新后继续") + stored = dict(draft.payload or {}) + inventory = stored.get("semantic_inventory") + if not inventory: + raise HTTPException(409, "这份草稿没有导入定义,请先导入定义文档") + requested_source_id = payload.datasource_id or draft.datasource_id + if payload.datasource_id and payload.datasource_id != draft.datasource_id and workflow is not None: + raise HTTPException(409, "已有取数图的草稿不能在续作时切换数据源,请另建迁移定义草稿") + source = await session.get(DataSource, requested_source_id) if requested_source_id else None + try: + require_authoring_source(source) + except ValueError as exc: + raise HTTPException(422, str(exc)) from None + from app.ledger.semantic_import import source_matches + docs = [await session.get(Document, key) for key in inventory.get("document_hashes", {})] + if any(d is None or d.status != "ready" for d in docs) or not source_matches( + inventory, [{"document_id": d.id, "content": d.content} for d in docs if d is not None] + ): + raise HTTPException(409, "来源文档已经变化,请重新导入定义") + view = await _draft_view(session, draft, ensure_records=False) + done = {m.get("source_definition_id") for m in view.get("metrics", [])} + missing = [key for key in stored.get("selected_definition_ids", []) if key not in done] + if not missing: + raise HTTPException(409, "所选指标已经全部生成,无需继续") + if payload.selected is not None: + wanted = list(dict.fromkeys(str(k) for k in payload.selected if str(k).strip())) + if not wanted or not set(wanted) <= set(missing): + raise HTTPException(422, "仅可选择当前尚未完成的指标继续生成") + target_missing = wanted + else: + target_missing = missing + key = content_hash(canonical_json({"revision": revision, "missing": target_missing, + "guidance": payload.guidance.strip(), "source": source.id, + "origin": draft.origin, "inventory": inventory})) + candidates = dict(stored.get("continuation_candidates") or {}) + if key in candidates: + candidate = await session.get(CaliberDraft, candidates[key]) + if candidate is not None: + return {"draft_id": candidate.id, "status": candidate.status, + "original_draft_id": draft_id, "merge_required": True} + candidate = CaliberDraft(title=f"{draft.title[:185]} · 补充指标", collection=draft.collection, + datasource_id=source.id, status="drafting", created_by=current_principal().username, + origin={**(draft.origin or {}), "guidance": "\n".join(filter(None, [ + str((draft.origin or {}).get("guidance") or ""), payload.guidance.strip()])), + "continuation_from": {"draft_id": draft.id, "workflow_id": draft.workflow_id, + "revision": revision, "merge_required": True}}, + payload={"semantic_inventory": inventory, "selected_definition_ids": target_missing, + "implementation_mode": "selected", "definition_issues": [], + "definition_artifact": stored.get("definition_artifact")}) + session.add(candidate) + await session.flush() + candidates[key] = candidate.id + update_values: dict[str, Any] = {"payload": {**stored, "continuation_candidates": candidates}} + if workflow is None and source.id != draft.datasource_id: + update_values["datasource_id"] = source.id + claimed = await session.execute(update(CaliberDraft).where( + CaliberDraft.id == draft.id, CaliberDraft.updated_at == draft.updated_at, + CaliberDraft.status == draft.status, + *([select(Workflow.id).where(Workflow.id == workflow.id, Workflow.version == revision).exists()] + if workflow else [CaliberDraft.workflow_id.is_(None)]) + ).values(**update_values).execution_options(synchronize_session=False)) + if claimed.rowcount != 1: + await session.rollback() + raise HTTPException(409, "草稿已变化或已有续作请求,请刷新后继续") + session.add(CaliberDraftJob(id=candidate.id)) + await session.commit() + background.add_task(draft_jobs.start, candidate.id) + return {"draft_id": candidate.id, "status": "drafting", "original_draft_id": draft_id, + "merge_required": True} + + +class MergeContinuationIn(BaseModel): + expected_revision: int = Field(ge=0) + candidate_revision: int = Field(ge=1) + + +@router.post("/drafts/{draft_id}/merge") +async def merge_continuation(draft_id: str, payload: MergeContinuationIn, + session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + from app.api.deps import current_principal, ensure_role + from app.apps.versions import append_version + from app.ledger import fieldbaseline + from app.ledger.continuation import append_candidate + from sqlalchemy import update + + ensure_role("builder") + # Serialize metadata updates with draft edits and competing merge requests. + await session.execute(update(CaliberDraft).where(CaliberDraft.id == draft_id) + .values(updated_at=CaliberDraft.updated_at).execution_options(synchronize_session=False)) + candidate = await _draft_or_404(session, draft_id) + origin = dict(candidate.origin or {}) + continuation = dict(origin.get("continuation_from") or {}) + if not continuation.get("draft_id"): + raise HTTPException(409, "这份草稿不是补充指标候选") + await session.execute(update(CaliberDraft).where(CaliberDraft.id == continuation["draft_id"]) + .values(updated_at=CaliberDraft.updated_at).execution_options(synchronize_session=False)) + original = await _draft_or_404(session, continuation["draft_id"]) + if continuation.get("merged_revision") is not None: + return {**await _draft_view(session, original, ensure_records=False), "merged_candidate_id": candidate.id, + "replayed": True} + if candidate.status not in ("ready", "questions_open") or original.status in ("drafting", "rejected", "attached"): + raise HTTPException(409, "请等待生成完成,并确认原稿仍可编辑") + incoming = await session.get(Workflow, candidate.workflow_id) if candidate.workflow_id else None + workflow = await session.get(Workflow, original.workflow_id) if original.workflow_id else None + if incoming is None or incoming.version != payload.candidate_revision: + raise HTTPException(409, "候选已变化,请刷新后重新核对") + if (workflow.version if workflow else 0) != payload.expected_revision: + raise HTTPException(409, "原稿已变化,请刷新后重新核对,不会覆盖现有编辑") + source = await session.get(DataSource, original.datasource_id) if original.datasource_id else None + try: + require_authoring_source(source) + except ValueError as exc: + raise HTTPException(422, str(exc)) from None + if candidate.datasource_id != original.datasource_id: + raise HTTPException(409, "候选与原稿数据源不一致,请重新核对") + # Lock the accepted candidate revision too; it must not change during the merge. + claimed = await session.execute(update(Workflow).where( + Workflow.id == incoming.id, Workflow.version == payload.candidate_revision + ).values(version=Workflow.version).execution_options(synchronize_session=False)) + if claimed.rowcount != 1: + raise HTTPException(409, "候选已变化,请刷新后重新核对") + original_payload = dict(original.payload or {}) + _, incoming_metrics = fieldbaseline.accepted_card(incoming.graph) + definition_ids = [m.get("source_definition_id") for m in incoming_metrics] + incoming_names = [m.get("name") for m in incoming_metrics] + if (not definition_ids or None in definition_ids + or not all(incoming_names) or len(set(incoming_names)) != len(incoming_names) + or not set(definition_ids) <= set(original_payload.get("selected_definition_ids") or [])): + raise HTTPException(409, "候选包含未请求的或重复的指标定义,请核对;候选已保留") + if ((candidate.payload or {}).get("semantic_inventory", {}).get("document_hashes") + != original_payload.get("semantic_inventory", {}).get("document_hashes")): + raise HTTPException(409, "候选与原稿的来源定义版本不同,请重新核对") + had_workflow = workflow is not None + prefix = f"supp_{candidate.id[:12]}" + try: + if had_workflow: + graph = append_candidate(workflow.graph, incoming.graph, prefix=prefix, + allowed_definitions=set(original_payload.get("selected_definition_ids") or [])) + await append_version(session, workflow, graph, expected_revision=payload.expected_revision, + note=f"合并补充指标候选 {candidate.id}") + else: + # Keep the independent candidate independent even when the original had no graph. + from app.db.models import WorkflowVersion + from app.core.artifact_store import graph_hash + workflow = Workflow(name=original.title, graph=copy.deepcopy(incoming.graph), status="draft", + tags=["doc_draft"], origin={"kind": "doc_draft", "draft_id": original.id, + "datasource_id": original.datasource_id}) + session.add(workflow) + await session.flush() + session.add(WorkflowVersion(workflow_id=workflow.id, version=1, graph=workflow.graph, + graph_hash=graph_hash(workflow.graph), note="接受补充指标候选")) + original.workflow_id = workflow.id + live = await _draft_graph_payload(session, workflow, workflow.version, draft=original, payload=original_payload) + except ValueError as exc: + await session.rollback() + raise HTTPException(409, str(exc)) from None + if live: + original_payload.update(live) + original_payload["workflow_id"] = workflow.id + if not had_workflow: + from app.core.artifact_store import graph_hash + if not original_payload.get("source_snapshot") and isinstance((candidate.payload or {}).get("source_snapshot"), dict): + original_payload["source_snapshot"] = copy.deepcopy(candidate.payload["source_snapshot"]) + original_payload["accepted_graph_hash"] = graph_hash(workflow.graph) + workflow_id = workflow.id + workflow_version = workflow.version + workflow_graph = copy.deepcopy(workflow.graph) + original_id = original.id + original_created_by = original.created_by + candidate_id = candidate.id + candidate_snapshot = ( + copy.deepcopy(candidate.payload["source_snapshot"]) + if isinstance((candidate.payload or {}).get("source_snapshot"), dict) else None + ) + completed = {m.get("source_definition_id") for m in original_payload.get("metrics", [])} + selected = set(original_payload.get("selected_definition_ids") or []) + completed_def_names = { + i["name"] for i in original_payload.get("semantic_inventory", {}).get("items", []) + if i["id"] in completed and i.get("name") + } + names = {m.get("name") for m in original_payload.get("metrics", []) if m.get("name")} | completed_def_names + remaining_count = len(selected - completed) + cleaned_questions: list[str] = [] + for q in original_payload.get("open_questions", []): + if any(q.startswith(f"{name}:") for name in names): + continue + if q.startswith("另有 ") and "项指标尚未生成可用实现" in q: + if remaining_count > 0: + cleaned_questions.append(f"另有 {remaining_count} 项指标尚未生成可用实现,请按上方定义清单补充字段映射或确认口径") + continue + cleaned_questions.append(q) + original_payload["open_questions"] = cleaned_questions + if isinstance(original_payload.get("definition_diagnostics"), dict): + original_payload["definition_diagnostics"] = { + k: v for k, v in original_payload["definition_diagnostics"].items() + if k in selected - completed + } + original_payload["uncovered"] = [i["name"] for i in original_payload.get("semantic_inventory", {}).get("items", []) + if i["id"] in selected - completed] + original_payload["implementation_progress"] = {"total": len(selected), "completed": len(selected & completed), + "pending": 0, "blocked": len(selected - completed), "active_names": []} + original_payload["trial"] = {"status": "stale", "run_id": "", "error": "已合并补充指标,请重新试运行验证完整口径"} + original.error = None + original.payload, original.status = original_payload, _status_for(original_payload) + candidate.origin = {**origin, "continuation_from": {**continuation, "merge_required": False, + "merged_revision": workflow_version}} + await session.commit() + + # Commit the write transaction before put_json opens its own session (avoids SQLite write-lock deadlock). + ref: dict[str, Any] | None = None + card_id, card_metrics = fieldbaseline.accepted_card(workflow_graph) + if had_workflow: + new_metric_ids = {f"{prefix}_{m['id']}" for m in incoming_metrics if m.get("id")} + updates: dict[str, dict[str, Any]] = {} + for metric in card_metrics: + mid = str(metric.get("id") or "") + if mid in new_metric_ids: + fields = fieldbaseline.fields_of(metric) + updates[mid] = { + "name": str(metric.get("name") or ""), + "fields": {path: cell["value"] for path, cell in fields.items()}, + "expected_fields": sorted(fields), + } + if updates: + ref = await fieldbaseline.advance( + ref=fieldbaseline.ref_of(original_payload), + updates=updates, + card_id=card_id, + workflow_id=workflow_id, + accepted_revision=workflow_version, + draft_id=original_id, + source_snapshot=original_payload.get("source_snapshot") or candidate_snapshot, + accepted_by=current_principal().username, + card_metric_ids=[str(m.get("id") or "") for m in card_metrics if m.get("id")], + ) + elif card_metrics: + ref = await fieldbaseline.put_metrics( + metrics=card_metrics, + card_id=card_id, + workflow_id=workflow_id, + accepted_revision=workflow_version, + source_snapshot=original_payload.get("source_snapshot") or candidate_snapshot, + draft_id=original_id, + accepted_by=original_created_by, + ) + original = await _draft_or_404(session, original_id) + if ref is not None: + updated_payload = dict(original.payload or {}) + updated_payload[fieldbaseline.REF_KEY] = ref + updated_payload[fieldbaseline.HISTORY_KEY] = [*fieldbaseline.history(updated_payload), ref] + original.payload = updated_payload + await session.commit() + original = await _draft_or_404(session, original_id) + return {**await _draft_view(session, original, ensure_records=False), "merged_candidate_id": candidate_id} + + @router.post("/drafts/{draft_id}/retry") async def retry_draft(draft_id: str, background: BackgroundTasks, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: diff --git a/backend/app/api/workflows.py b/backend/app/api/workflows.py index 20dc59c6..d6cccde5 100644 --- a/backend/app/api/workflows.py +++ b/backend/app/api/workflows.py @@ -602,7 +602,7 @@ async def publish_workflow( if not snapshot: raise HTTPException(404, f"不存在 v{version} 版本:画布上的修改还没有保存,请先保存再发布") - issues = await gate_issues(session, snapshot.graph, payload.level) + issues = await gate_issues(session, snapshot.graph, payload.level, workflow_id=workflow_id) if issues is None: return {"ok": False, "issues": [{"level": "error", "message": _unparsed(snapshot.graph)}]} if any(i["level"] == "error" for i in issues): diff --git a/backend/app/ledger/authoring.py b/backend/app/ledger/authoring.py index e7d16524..baa6f7f6 100644 --- a/backend/app/ledger/authoring.py +++ b/backend/app/ledger/authoring.py @@ -14,6 +14,8 @@ from app.db.models import CaliberDraft, Chunk, DataSource, Document, Workflow logger = logging.getLogger(__name__) MAX_ROUNDS = 4 MAX_METRICS = 60 +DEFINITIONS_PER_BATCH = 6 +DEFINITION_ATTEMPTS = 2 _QUESTION_KEY = 30 _CLAUSE = ",,。;;::!!??" _PUNCT = set(" \t\r\n,。;:、!?,.!?;:'\"()()[]【】《》<>-_/\\|`~") @@ -50,14 +52,75 @@ def _source_tables(source: DataSource) -> dict[str, Any]: return tables if isinstance(tables, dict) else {} -def _focus_tables(tables: list[str], text: str, limit: int = 8) -> list[str]: +def require_authoring_source(source: DataSource | None) -> None: + """Shared eligibility gate for entrypoints, retry and the actual worker.""" + if source is None: + raise ValueError("数据源不存在,可能已被删除") + if not source.enabled: + raise ValueError("数据源已停用,请启用后再生成口径") + if not _source_tables(source): + raise ValueError("这个数据源还没有表结构:先探查表结构,再起草口径") + + +async def _resume_finalization(draft_id: str, *, run_trial: Any, save_trial: Any, trial_binding: Any) -> bool: + """Resume durable graph acceptance without another model call or graph creation.""" + from app.core.artifact_store import graph_hash + async with SessionLocal() as session: + draft = await session.get(CaliberDraft, draft_id) + if draft is None or draft.status != "drafting" or not draft.workflow_id: + return False + payload = dict(draft.payload or {}) + source = await session.get(DataSource, draft.datasource_id) if draft.datasource_id else None + try: + require_authoring_source(source) + workflow = await session.get(Workflow, draft.workflow_id) + if (workflow is None or not payload.get("finalization_pending") + or graph_hash(workflow.graph) != payload.get("accepted_graph_hash")): + raise ValueError("已生成的工作流发生变化,已保留当前编辑,请人工核对后试运行") + except ValueError as exc: + draft.status, draft.error = "failed", str(exc) + await session.commit() + return True + graph = workflow.graph + await session.commit() + baseline = await _capture_field_baseline(draft=draft, graph=graph, workflow_id=workflow.id, + source_snapshot=payload.get("source_snapshot")) + draft.payload = {**payload, **baseline, "finalization_pending": False} + draft.status, draft.error = _status_for(draft.payload), None + await session.commit() + fields = payload.get("entry_fields") or [] + values = {f["name"]: f.get("default") for f in fields if f.get("default") not in (None, "")} + try: + trial = await run_trial(draft_id=draft_id, workflow_id=workflow.id, graph=graph, values=values, + started_by=draft.created_by, entry_fields=fields) + except Exception as exc: + trial = {**trial_binding(1, graph, values), "trial_id": uuid4().hex, "run_id": "", + "status": "failed", "error": f"试运行出错:{exc}"} + await save_trial(draft_id, trial) + return True + + +_GENERIC_COLUMN_TOKENS = frozenset({ + "select", "from", "where", "group", "order", "count", "value", "values", "total", "status", + "state", "start", "end", "date", "time", "code", "name", "type", "line", "plant", "daily", + "weekly", "monthly", "period", "report", "table", "column", "remark", "notes", "load_time", + "create_time", "update_time", "insert_time", +}) + + +def _focus_tables(tables: list[str], text: str, limit: int = 8, *, + source: DataSource | None = None) -> list[str]: """文档里点到的表优先展开:按它在原文里第一次出现的位置排,没点到的按目录顺序补几张。 目录动辄十万字符,全塞给模型会把文档和别的表挤掉。先给全量列名(见 _columns_digest),再把文档点到的 那几张表展开注释和目录——写 SQL 需要的列名一个不少,细节只给用得上的。 + 数仓视图常带层级前缀(如 vi_ike_mes_bhgsj 对应文档写的 MES_BHGSJ),以及文档直接点名特征字段(如 attribute36), + 依次在精确表名匹配之后补充,且正式表优先于 his 历史归档表。 """ lowered = (text or "").lower() - hit: list[tuple[int, str]] = [] + exact_hit: list[tuple[int, str]] = [] + suffix_hit: list[tuple[int, int, str]] = [] + matched: set[str] = set() for table in tables: name = str(table).lower() short = name.split(".")[-1] @@ -65,9 +128,49 @@ def _focus_tables(tables: list[str], text: str, limit: int = 8) -> list[str]: if at < 0: at = lowered.find(short) if at >= 0: - hit.append((at, table)) - hit.sort(key=lambda pair: pair[0]) - picked = [table for _, table in hit][:limit] + exact_hit.append((at, table)) + matched.add(table) + continue + parts = [p for p in short.split("_") if p] + for idx in range(1, len(parts)): + suffix = "_".join(parts[idx:]) + if len(suffix) < 5 or ("_" not in suffix and len(suffix) < 7): + continue + m = re.search(rf"(?= limit: + break + if table not in picked: + picked.append(table) + if source is not None and len(picked) < limit and lowered: + from app.data import introspect + + tokens = {tok for tok in re.findall(r"[a-z][a-z0-9_]{3,}", lowered) if tok not in _GENERIC_COLUMN_TOKENS} + if tokens: + col_hits: list[tuple[int, int, str]] = [] + for table in tables: + if table in picked: + continue + short = str(table).lower().split(".")[-1] + meta = introspect.find_table(source, table) + cols = {str(c.get("name") or "").lower() for c in (meta or {}).get("columns") or [] if c.get("name")} + overlap = len((cols - _GENERIC_COLUMN_TOKENS) & tokens) + if overlap > 0: + is_his = 1 if "_his_" in f"_{short}_" else 0 + col_hits.append((-overlap, is_his, table)) + col_hits.sort() + for _, _, table in col_hits: + if len(picked) >= limit: + break + picked.append(table) for table in tables: if len(picked) >= min(limit, 5): break @@ -76,15 +179,23 @@ def _focus_tables(tables: list[str], text: str, limit: int = 8) -> list[str]: return picked -def _columns_digest(source: DataSource, tables: list[str]) -> str: +def _columns_digest(source: DataSource, tables: list[str], *, focused: list[str] | None = None) -> str: """每张表的列名清单(只给名字):要写 SQL,先得知道有哪些列。注释和目录留给下面展开的那几张。""" from app.data import introspect + focus_set = set(focused or ()) + ordered = [*(t for t in (focused or ()) if t in set(tables)), *(t for t in tables if t not in focus_set)] + compact_unfocused = len(tables) > 20 lines = ["各表的列(只列名字;下面展开的表另有类型、注释和数据目录):"] - for table in tables: + for table in ordered: meta = introspect.find_table(source, table) columns = [str(c.get("name")) for c in (meta or {}).get("columns") or [] if c.get("name")] - if columns: + if not columns: + continue + if compact_unfocused and table not in focus_set and len(columns) > 45: + shown = ", ".join(columns[:35]) + lines.append(f"{table}: {shown} …(共 {len(columns)} 列)") + else: lines.append(f"{table}: {', '.join(columns)}") return "\n".join(lines) @@ -93,7 +204,7 @@ async def _schema_material(session: AsyncSession, source: DataSource, focus_text ) -> tuple[str, set[str]]: """这个数据源给模型看的表结构 + 数据目录文本,以及 SQL 允许出现的表名集合。 - 文本按「概览 → 全量表索引 → 全量列名 → 文档点到的表展开」的顺序排:预算不够时先丢的是最后那一段, + 文本按「概览 → 全量表索引 → 全量列名(重点表优先) → 文档点到的表展开」的顺序排:预算不够时先丢的是最后那一段, 列名不会丢。表名一律照目录原文,模型照着抄。 """ from app.data import catalog, schema_view @@ -104,11 +215,12 @@ async def _schema_material(session: AsyncSession, source: DataSource, focus_text return "", set() entries = await catalog.read_catalog(session, source.id) notes = {table: entry.notes for table, entry in entries.items()} + focused = _focus_tables(tables, focus_text, source=source) lines = [schema_view.overview_line(source, catalog_tables=sum( 1 for entry in entries.values() if catalog.visible_notes(entry.notes)))] lines.append(schema_view.table_index(source, notes)) - lines.append(_columns_digest(source, tables)) - for table in _focus_tables(tables, focus_text): + lines.append(_columns_digest(source, tables, focused=focused)) + for table in focused: lines.append(schema_view.table_detail(source, table, catalog_notes=notes)) schema = cache.get("schema") allowed: set[str] = set() @@ -330,6 +442,91 @@ async def _capture_field_baseline(*, draft: Any, graph: dict[str, Any], workflow return {fieldbaseline.REF_KEY: ref, fieldbaseline.HISTORY_KEY: [*history, ref]} +def _dedupe_problems(problems: list[dict[str, Any]]) -> list[dict[str, Any]]: + has_specific = any(isinstance(p, dict) and p.get("name") != "metrics" for p in problems) + seen: set[tuple[str, str]] = set() + out: list[dict[str, Any]] = [] + for item in problems: + if not isinstance(item, dict): + continue + name = str(item.get("name") or "") + reason = str(item.get("reason") or "") + if has_specific and name == "metrics" and reason == "没有一条指标通过校验": + continue + key = (name, reason) + if key not in seen: + seen.add(key) + out.append({"name": name, "reason": reason}) + return out + + +def _scoped_definition_chunks(chunks: dict[str, dict[str, Any]], semantic_inventory: dict[str, Any], + active_definitions: list[str]) -> tuple[dict[str, dict[str, Any]], dict[str, str]]: + from app.ledger import docdraft + + ordered_ids = docdraft._order(chunks) + active_set = set(active_definitions) + active_items = [item for item in semantic_inventory.get("items") or [] if item.get("id") in active_set] + hints: dict[str, str] = {} + matched_chunks: set[str] = set() + for item in active_items: + quote = str((item.get("source") or {}).get("quote") or "") + doc_id = str((item.get("source") or {}).get("document_id") or "") + if len(docdraft._collapse(quote)) < docdraft.MIN_QUOTE_CHARS: + continue + for cid in ordered_ids: + chunk = chunks[cid] + if doc_id and str(chunk.get("document_id") or "") != doc_id: + continue + if docdraft.verify_quote(str(chunk.get("content") or ""), quote) is not None: + hints[str(item["id"])] = cid + matched_chunks.add(cid) + break + if not active_items or len(hints) < len(active_items) or len(chunks) <= 2: + return chunks, hints + return {cid: chunks[cid] for cid in ordered_ids if cid in matched_chunks}, hints + + +def _attach_verified_definition_citations(raw_group: Any, *, visible: dict[str, dict[str, Any]], + semantic_inventory: dict[str, Any], active_definitions: list[str], + chunk_hints: dict[str, str]) -> Any: + from app.ledger import docdraft + + if not isinstance(raw_group, dict): + return raw_group + raw_metrics = raw_group.get("metrics") + if not isinstance(raw_metrics, list): + return raw_group + active_set = set(active_definitions) + by_id = {str(i["id"]): i for i in semantic_inventory.get("items") or [] if i.get("id") in active_set} + by_name = {str(i["name"]): i for i in by_id.values() if i.get("name")} + patched: list[Any] = [] + for item in raw_metrics: + if not isinstance(item, dict): + patched.append(item) + continue + existing = [c for c in docdraft._citation_list(item) if docdraft._check_citation(c, visible)] + if existing: + patched.append(item) + continue + def_id = str(item.get("source_definition_id") or "") + def_item = by_id.get(def_id) or by_name.get(str(item.get("name") or "").strip()) + cid = chunk_hints.get(str(def_item["id"])) if def_item else None + quote = str((def_item.get("source") or {}).get("quote") or "") if def_item else "" + fallback = {"chunk": cid, "quote": quote} if cid and cid in visible and quote else None + if fallback and docdraft._check_citation(fallback, visible): + copied = dict(item) + definition = dict(copied.get("definition")) if isinstance(copied.get("definition"), dict) else {} + definition["citations"] = [fallback] + copied["definition"] = definition + copied.pop("citations", None) + copied.setdefault("source_definition_id", str(def_item["id"])) + patched.append(copied) + else: + patched.append(item) + return {**raw_group, "metrics": patched} + + async def extract(draft_id: str, *, run_trial: Any, save_trial: Any, trial_binding: Any) -> None: """后台起草:读文档的段落和数据目录,分组反复调模型直到覆盖完,收成一张多取数的草稿图、建工作流、再跑一次试跑。 @@ -353,6 +550,9 @@ async def extract(draft_id: str, *, run_trial: Any, save_trial: Any, trial_bindi from app.db.models import WorkflowVersion from app.ledger import docdraft, draftplan + if await _resume_finalization(draft_id, run_trial=run_trial, save_trial=save_trial, trial_binding=trial_binding): + return + async with SessionLocal() as session: draft = await session.get(CaliberDraft, draft_id) if draft is None or draft.status != "drafting": @@ -366,6 +566,8 @@ async def extract(draft_id: str, *, run_trial: Any, save_trial: Any, trial_bindi docs.append(doc) chunks: dict[str, dict[str, Any]] = {} entries: list[dict[str, Any]] = [] + definition_state = dict(draft.payload or {}) + has_inventory = isinstance(definition_state.get("semantic_inventory"), dict) for doc in docs: rows = (await session.execute(select(Chunk).where(Chunk.document_id == doc.id) .order_by(Chunk.ordinal, Chunk.id))).scalars() @@ -374,16 +576,27 @@ async def extract(draft_id: str, *, run_trial: Any, save_trial: Any, trial_bindi chunks[chunk.id] = {"ordinal": chunk.ordinal, "content": chunk.content or "", "document_id": doc.id} chunk_ids.append(chunk.id) + if not chunk_ids and has_inventory and (doc.content or "").strip(): + fallback_cid = f"doc-{doc.id}" + chunks[fallback_cid] = {"ordinal": 1, "content": doc.content or "", "document_id": doc.id} + chunk_ids.append(fallback_cid) entries.append({"document_id": doc.id, "title": doc.title or doc.source or doc.id, "chunk_ids": chunk_ids}) source = await session.get(DataSource, draft.datasource_id) if draft.datasource_id else None - if source is None: - draft.error = "没有数据源:生成取数 SQL 必须对照数据目录里的表和列" + try: + require_authoring_source(source) + except ValueError as exc: + draft.error = str(exc) draft.status = "failed" await session.commit() return - focus_text = "\n".join(str(c.get("content") or "") for c in chunks.values()) + guidance = str(origin.get("guidance") or "") + fixed_filters = str(origin.get("fixed_filters") or "") + doc_body = "\n".join(str(c.get("content") or "") for c in chunks.values()) + if not doc_body: + doc_body = "\n".join(str(d.content or "") for d in docs) + focus_text = "\n".join(part for part in (doc_body, guidance, fixed_filters) if part) tables_text, allowed_tables = await _schema_material(session, source, focus_text) if not allowed_tables: draft.error = "这个数据源还没有表结构:先探查表结构,再起草口径" @@ -392,8 +605,32 @@ async def extract(draft_id: str, *, run_trial: Any, save_trial: Any, trial_bindi return datasource = {"name": source.name, "type": source.kind, "description": source.description} - guidance = str(origin.get("guidance") or "") - fixed_filters = str(origin.get("fixed_filters") or "") + semantic_inventory = definition_state.get("semantic_inventory") + selected_definitions = definition_state.get("selected_definition_ids") or [] + selected_names = None + if semantic_inventory and selected_definitions: + from app.ledger.semantic_import import implementation_guidance, source_matches + + if not source_matches(semantic_inventory, [{"document_id": d.id, "content": d.content} for d in docs]): + draft.error = "原文与导入定义的版本不一致,请重新导入后再实现" + draft.status = "failed" + await session.commit() + return + + selected_names = {i["name"] for i in semantic_inventory["items"] if i["id"] in selected_definitions} + + active_definitions = list(selected_definitions) + + def selected_metrics(metrics): + if selected_names is None: + return metrics + definitions = {i["name"]: i["id"] for i in semantic_inventory["items"] if i["id"] in active_definitions} + kept = [] + for metric in metrics: + definition_id = metric.get("source_definition_id") or definitions.get(metric.get("name")) + if definition_id in active_definitions: + kept.append({**metric, "source_definition_id": definition_id}) + return kept # 提取这一刻就把来源存下来:原文按内容寻址进工件库,草稿只记引用。重导入要拿来当 B 的是这份快照, # 不是草稿现在的样子;存原文是为了还原,光有哈希还原不了。先提交一次会话,落工件时不再握着读事务。 @@ -424,13 +661,37 @@ async def extract(draft_id: str, *, run_trial: Any, save_trial: Any, trial_bindi from app.core.artifact_store import canonical_json, content_hash material_hash = content_hash(canonical_json({"documents": entries, "chunks": chunks, - "datasource": datasource, "catalog": tables_text, "guidance": guidance, "filters": fixed_filters})) + "datasource": datasource, "catalog": tables_text, "guidance": guidance, "filters": fixed_filters, + **({"inventory": semantic_inventory, "selected": selected_definitions, "batch_policy": 1} + if selected_names is not None else {}), + "plan_policy": draftplan.PROMPT_VERSION})) previous = draft.payload if isinstance(draft.payload, dict) else {} + attempts: dict[str, int] = {} + definition_diagnostics: dict[str, dict[str, Any]] = {} + round_offset = 0 if previous.get("material_hash") == material_hash: blocks = copy.deepcopy((previous.get("checkpoint") or {}).get("groups") or []) covered = [str(m["name"]) for block in blocks for m in block.get("metrics") or [] if m.get("name")] questions = list(previous.get("open_questions") or []) - problems = list(previous.get("dropped") or []) + problems = _dedupe_problems(list(previous.get("dropped") or [])) + attempts = dict((previous.get("checkpoint") or {}).get("definition_attempts") or {}) + saved_diag = ((previous.get("checkpoint") or {}).get("definition_diagnostics") + or previous.get("definition_diagnostics") or {}) + if isinstance(saved_diag, dict): + definition_diagnostics = copy.deepcopy(saved_diag) + round_offset = int((previous.get("progress") or {}).get("round") or 0) + progress = dict(previous.get("progress") or {}) + prompts = list(previous.get("prompts") or []) + + def implemented_ids() -> set[str]: + return {m.get("source_definition_id") for b in blocks for m in b["metrics"]} + + def implementation_progress() -> dict: + done = implemented_ids() + blocked = sum(key not in done and attempts.get(key, 0) >= DEFINITION_ATTEMPTS for key in selected_definitions) + return {"total": len(selected_definitions), "completed": len(set(selected_definitions) & done), + "pending": len(set(selected_definitions) - done) - blocked, "blocked": blocked, + "active_names": [i["name"] for i in semantic_inventory["items"] if i["id"] in active_definitions]} def state() -> dict[str, Any]: payload = _draft_payload(entries=entries, blocks=blocks, problems=problems, questions=questions, @@ -440,19 +701,54 @@ async def extract(draft_id: str, *, run_trial: Any, save_trial: Any, trial_bindi # 来源基线状态现算现写:有可还原的快照就是 known,没有就是 baseline_unknown。它只说明「当时的原文 # 还原得出来」,不等于字段基线,也不等于这条口径验证通过。 payload["source_baseline"] = docdraft.source_baseline(payload) - return {**payload, "material_hash": material_hash, - "checkpoint": {"groups": copy.deepcopy(blocks)}} + done_ids = implemented_ids() if semantic_inventory else set() + unfinished_diagnostics = { + str(k): v for k, v in definition_diagnostics.items() + if str(k) in set(selected_definitions) and str(k) not in done_ids + } if semantic_inventory else {} + return {**payload, **({"semantic_inventory": semantic_inventory, + "selected_definition_ids": selected_definitions, + "implementation_mode": definition_state.get("implementation_mode", "selected"), + "definition_issues": definition_state.get("definition_issues", []), + **({"migration_baseline": definition_state["migration_baseline"]} + if definition_state.get("migration_baseline") else {}), + "implementation_progress": implementation_progress(), + "definition_diagnostics": unfinished_diagnostics, + "definition_artifact": definition_state.get("definition_artifact")} + if semantic_inventory else {}), "material_hash": material_hash, + "checkpoint": {"groups": copy.deepcopy(blocks), + **({"definition_attempts": dict(attempts), + "definition_diagnostics": copy.deepcopy(definition_diagnostics)} + if semantic_inventory else {})}} try: - for rnd in range(1, MAX_ROUNDS + 1): + rounds = len(selected_definitions) * DEFINITION_ATTEMPTS if selected_names is not None else MAX_ROUNDS + for step in range(1, rounds + 1): + rnd = round_offset + step + round_guidance = guidance + round_chunks = chunks + chunk_hints: dict[str, str] = {} + if selected_names is not None: + done = implemented_ids() + pending = [key for key in selected_definitions if key not in done + and attempts.get(key, 0) < DEFINITION_ATTEMPTS] + active_definitions = sorted(pending, key=lambda key: attempts.get(key, 0))[:DEFINITIONS_PER_BATCH] + if not active_definitions: + break + round_chunks, chunk_hints = _scoped_definition_chunks(chunks, semantic_inventory, active_definitions) + round_guidance += "\n" + implementation_guidance( + semantic_inventory, active_definitions, chunk_hints=chunk_hints) + # Commit the previous completed work before waiting on another model request. + draft.payload = state() await session.commit() # 先算一次材料:这一份既进提示,也拿来核对引用、记提示记录——两处用同一份,引用才核得上实际可见范围 - fitted = draftplan.fit_material(entries, chunks, guidance, datasource, tables_text, + fitted = draftplan.fit_material(entries, round_chunks, round_guidance, datasource, tables_text, covered, fixed_filters) visible = fitted[0] async with SessionLocal() as planning: - result = await draftplan.plan_groups(planning, docs=entries, chunks=chunks, guidance=guidance, - datasource=datasource, tables_text=tables_text, covered=covered, + result = await draftplan.plan_groups(planning, docs=entries, chunks=round_chunks, + guidance=round_guidance, datasource=datasource, + tables_text=tables_text, covered=covered, fixed=fixed_filters, fitted=fitted) doc_truncated = doc_truncated or fitted[2] catalog_truncated = catalog_truncated or fitted[3] @@ -461,22 +757,36 @@ async def extract(draft_id: str, *, run_trial: Any, save_trial: Any, trial_bindi prompts.append(_prompt_trace(round=rnd, purpose="plan", fitted=fitted, model_id=model_id, salvaged=bool(result.get("salvaged")), groups=len(result.get("groups") or []))) + if selected_names is not None: + prompts[-1]["definition_ids"] = list(active_definitions) new_count = 0 failed: list[tuple[dict[str, Any], list[dict[str, Any]]]] = [] + round_recorded_problems: list[dict[str, Any]] = [] + round_open_questions: list[str] = [] for raw_group in result.get("groups") or []: + prepared_group = ( + _attach_verified_definition_citations( + raw_group, visible=visible, semantic_inventory=semantic_inventory, + active_definitions=active_definitions, chunk_hints=chunk_hints) + if selected_names is not None else raw_group + ) # 引用核对只认实际进提示的段落:visible 里没有的段落,模型引用它也核不上 - plan, round_problems = draftplan.normalize_plan(raw_group, visible, - allowed_tables=allowed_tables) + plan, round_problems = draftplan.normalize_plan(prepared_group, visible, + allowed_tables=allowed_tables, strict_scalar=True) fresh = _fresh_metrics(plan["metrics"], covered) + fresh = selected_metrics(fresh) if plan["sql"] and fresh: blocks.append({"sql": plan["sql"], "entry_fields": plan["entry_fields"], "metrics": fresh}) covered += [m["name"] for m in fresh if m.get("name")] new_count += len(fresh) problems += round_problems - elif _can_repair(raw_group): - failed.append((raw_group, round_problems)) + round_recorded_problems += round_problems + elif _can_repair(prepared_group): + failed.append((prepared_group, round_problems)) else: problems += round_problems + round_recorded_problems += round_problems + round_open_questions = _merge_questions(round_open_questions, plan["open_questions"]) questions = _merge_questions(questions, plan["open_questions"]) if failed: @@ -484,8 +794,8 @@ async def extract(draft_id: str, *, run_trial: Any, save_trial: Any, trial_bindi semaphore = asyncio.Semaphore(3) # 重写和起草的材料一样,但 fit 的入参不同(covered 是空的、fixed 照旧,见 repair_group): # 先算一次,各次重写与引用核对共用同一份可见范围 - repair_fitted = draftplan.fit_material(entries, chunks, guidance, datasource, tables_text, - (), fixed_filters) + repair_fitted = draftplan.fit_material(entries, round_chunks, round_guidance, datasource, + tables_text, (), fixed_filters) prompts.append(_prompt_trace(round=rnd, purpose="repair", fitted=repair_fitted, model_id=model_id, groups=len(failed))) @@ -494,8 +804,8 @@ async def extract(draft_id: str, *, run_trial: Any, save_trial: Any, trial_bindi try: async with SessionLocal() as repairing: fixed = await draftplan.repair_group( - repairing, group=raw, problems=issues, docs=entries, chunks=chunks, - guidance=guidance, datasource=datasource, tables_text=tables_text, + repairing, group=raw, problems=issues, docs=entries, chunks=round_chunks, + guidance=round_guidance, datasource=datasource, tables_text=tables_text, fixed=fixed_filters, fitted=repair_fitted) except docdraft.DraftUnavailable as e: return None, [{"name": "重写这一组", "reason": str(e) or "AI 起草不可用"}] @@ -503,33 +813,115 @@ async def extract(draft_id: str, *, run_trial: Any, save_trial: Any, trial_bindi return None, [{"name": "重写这一组", "reason": f"重写出错:{e}"}] if not fixed: return None, [] - return draftplan.normalize_plan(fixed, repair_fitted[0], allowed_tables=allowed_tables) + prepared_fixed = ( + _attach_verified_definition_citations( + fixed, visible=repair_fitted[0], semantic_inventory=semantic_inventory, + active_definitions=active_definitions, chunk_hints=chunk_hints) + if selected_names is not None else fixed + ) + return draftplan.normalize_plan(prepared_fixed, repair_fitted[0], allowed_tables=allowed_tables, + strict_scalar=True) repaired = await asyncio.gather(*(repair(raw, issues) for raw, issues in failed)) for (_raw, issues), item in zip(failed, repaired): plan, repair_problems = item fresh = _fresh_metrics(plan["metrics"], covered) if plan is not None else [] + fresh = selected_metrics(fresh) if plan is not None and plan["sql"] and fresh: blocks.append({"sql": plan["sql"], "entry_fields": plan["entry_fields"], "metrics": fresh}) covered += [m["name"] for m in fresh if m.get("name")] new_count += len(fresh) problems += repair_problems + round_recorded_problems += repair_problems else: # 重写没成:原来这一组的问题照实留着,再附上重写这轮的说明 problems += issues + repair_problems + round_recorded_problems += issues + repair_problems if plan is not None: + round_open_questions = _merge_questions(round_open_questions, plan["open_questions"]) questions = _merge_questions(questions, plan["open_questions"]) - questions = _merge_questions(questions, list(result.get("open_questions") or [])) + top_questions = list(result.get("open_questions") or []) + round_open_questions = _merge_questions(round_open_questions, top_questions) + questions = _merge_questions(questions, top_questions) + problems = _dedupe_problems(problems) uncovered = list(result.get("uncovered") or []) + if selected_names is not None: + for key in active_definitions: + attempts[key] = attempts.get(key, 0) + 1 + implemented = implemented_ids() + for done_id in list(definition_diagnostics): + if done_id in implemented: + definition_diagnostics.pop(done_id, None) + deduped_round = _dedupe_problems(round_recorded_problems) + active_by_id = {str(i["id"]): i for i in semantic_inventory["items"] if i["id"] in active_definitions} + for def_id in active_definitions: + if def_id in implemented: + continue + def_item = active_by_id.get(str(def_id)) or {} + item_name = str(def_item.get("name") or "") + matched_reasons = [ + str(p["reason"]) for p in deduped_round + if p.get("reason") and ( + p.get("name") == item_name or (item_name and item_name in str(p.get("name") or "")) + ) + ] + if not matched_reasons: + matched_reasons = [ + str(p["reason"]) for p in deduped_round + if p.get("reason") and p.get("name") in ("sql", "metrics", "重写这一组") + ] + other_names = { + str(v.get("name") or "") for k, v in active_by_id.items() + if k != str(def_id) and v.get("name") + } + matched_questions = [q for q in round_open_questions if item_name and item_name in q] + if not matched_questions: + matched_questions = [ + q for q in round_open_questions + if not any(other in q for other in other_names) + ] + prev_diag = definition_diagnostics.get(str(def_id)) or {} + reasons = list(dict.fromkeys([*(prev_diag.get("reasons") or []), *matched_reasons]))[:5] + item_questions = list(dict.fromkeys([*(prev_diag.get("questions") or []), *matched_questions]))[:5] + if reasons or item_questions or attempts.get(def_id, 0) >= DEFINITION_ATTEMPTS: + kind = "validation_failed" if reasons and not item_questions else "needs_info" + definition_diagnostics[str(def_id)] = { + "kind": kind, "reasons": reasons, "questions": item_questions, + } + uncovered = [i["name"] for i in semantic_inventory["items"] + if i["id"] in selected_definitions and i["id"] not in implemented] diagnostics = result.get("diagnostics") or diagnostics progress = {"round": rnd, "covered": len(covered), "uncovered": len(uncovered)} # 每轮先落一次库:这一轮要等一次模型调用,页面这时就能显示「第几轮、起草了几条、还剩几条」 draft.payload = state() await session.commit() - if not new_count or not uncovered or len(covered) >= MAX_METRICS: + if selected_names is None and (not new_count or not uncovered or len(covered) >= MAX_METRICS): break + if selected_names is not None: + active_definitions = [] + covered_names = set(covered) + problems = [p for p in _dedupe_problems(problems) if p.get("name") not in covered_names] + unresolved = [i for i in semantic_inventory["items"] if i["id"] in selected_definitions + and i["id"] not in implemented_ids()] + issue_questions = [f"{i['name']}:{i['reason']}" for i in definition_state.get("definition_issues", [])] + if blocks or questions or issue_questions or not problems: + if len(unresolved) <= 10: + per_item: list[str] = [] + for i in unresolved: + if any(i["name"] in q for q in questions): + continue + diag = definition_diagnostics.get(str(i["id"])) or {} + specific = str((diag.get("questions") or diag.get("reasons") or [""])[0] or "").strip() + note = specific or "尚未生成可用实现,请补充字段映射或确认口径" + per_item.append(note if note.startswith(f"{i['name']}:") else f"{i['name']}:{note}") + questions = _merge_questions(questions, per_item) + elif not questions and not issue_questions: + questions = _merge_questions( + questions, [f"另有 {len(unresolved)} 项指标尚未生成可用实现,请按上方定义清单补充字段映射或确认口径"]) + questions = _merge_questions(questions, issue_questions) + # 多组从各自 m1 开始:先把 id 在全图里改成唯一,图、payload、groups 和试跑值才指同一件事 draftplan.unique_metric_ids(blocks) @@ -574,9 +966,11 @@ async def extract(draft_id: str, *, run_trial: Any, save_trial: Any, trial_bindi metrics = [m for block in blocks for m in block["metrics"]] alignment = docdraft.alignment(metrics, existing) + covered_set = set(covered) asked = list(dict.fromkeys(questions + [ f"{p['name']}:{p['reason']}" for p in problems - if p.get("name") not in ("sql", "metrics")])) + if p.get("name") not in ("sql", "metrics") + and not (selected_names is not None and p.get("name") in covered_set)])) workflow = Workflow( name=(draft.title.strip() or "文档口径草稿")[:200], description=(f"从业务文档起草的口径:{len(metrics)} 个指标、{len(blocks)} 组取数,取数 SQL 由模型对照" @@ -588,18 +982,17 @@ async def extract(draft_id: str, *, run_trial: Any, save_trial: Any, trial_bindi await session.flush() session.add(WorkflowVersion(workflow_id=workflow.id, version=1, graph=graph, graph_hash=graph_hash(graph), note="从业务文档分组生成")) - # 落字段基线走的是工件库的另一个会话:先把工作流和 v1 提交掉,别让两个会话同时握着写锁(和 _capture_source - # 同一个道理)。提交之后 draft / workflow 仍然可用(SessionLocal 是 expire_on_commit=False)。 - await session.flush() + # Commit ownership and phase with the graph: cancellation cannot orphan it. + draft.workflow_id = workflow.id + draft.model_id = model_id or None + draft.payload = {**state(), "workflow_id": workflow.id, "open_questions": asked, + "alignment": alignment, "finalization_pending": True, + "accepted_graph_hash": graph_hash(graph)} await session.commit() - # 首次接受导入:接受的那一刻就把被接受的字段值存成 B(K05,按 metric_id + field_path)。存的是**图里那 - # 一份**(接受的就是它),带上这次抽取的来源快照引用和接受版本;存不下时不写这一项(照实说没基线)。 + payload = dict(draft.payload) baseline = await _capture_field_baseline(draft=draft, graph=graph, workflow_id=workflow.id, source_snapshot=source_snapshot) - payload = { - **state(), "workflow_id": workflow.id, "open_questions": asked, - "alignment": alignment, **baseline, - } + payload = {**payload, **baseline, "finalization_pending": False} except docdraft.DraftUnavailable as e: draft.error = str(e) or "AI 起草不可用" draft.status = "failed" diff --git a/backend/app/ledger/changes.py b/backend/app/ledger/changes.py index a7928206..41eccaa7 100644 --- a/backend/app/ledger/changes.py +++ b/backend/app/ledger/changes.py @@ -82,6 +82,8 @@ class Field: #: 能按字段改的路径。定义类的判定和 data/caliber_identity.SEMANTIC_FIELDS 对齐:语义字段算定义,其余(同义词)算展示。 FIELDS: dict[str, Field] = { + "/semantic": Field("definition", "semantic", "问数语义"), + "/links": Field("definition", "links", "本体关系"), "/name": Field("definition", "text", "指标名称", nonempty=True), "/synonyms": Field("display", "texts", "同义词"), "/unit": Field("definition", "text", "单位"), @@ -194,6 +196,21 @@ def _checked(index: int, path: str, field: Field, value: Any) -> Any: if value is None: refuse(f"{path} 不能设成 null:「清空」用 remove、「声明成空」用 ''(数组用 []),两种都不是就写具体值") + if field.value_type == "semantic": + from pydantic import ValidationError + from app.ledger.semantic import QuerySemantics + + try: + return QuerySemantics.model_validate(value).model_dump(mode="json") + except ValidationError as error: + refuse(f"问数语义不完整:{error}") + if field.value_type == "links": + from app.ledger.ontology import OntologyInvalid, normalize_links + + try: + return normalize_links(value) + except OntologyInvalid as error: + refuse(error.message) if field.value_type == "choice": if not isinstance(value, str): refuse(f"{field.label}({path})要是字符串") @@ -294,6 +311,22 @@ def apply_operations(graph: Mapping[str, Any] | None, *, card_id: str, definition_hash=(before_hash, before_hash), binding=bindings.analyze(graph or {}, repo.card_id)) new_graph = repo.replace_metrics(changed) + if any(op.get("path") == "/links" for op in report): + from app.ledger.ontology import OntologyCycleError, check_factor_dag, workflow_ontology_index + + idx = workflow_ontology_index(workflow_id="", version=0, graph=new_graph) + factor_edges = [ + ( + f"{e['from']['workflow_id'] or 'self'}:{e['from']['card_id']}:{e['from']['metric_id']}", + f"{e['to']['workflow_id'] or 'self'}:{e['to']['card_id']}:{e['to']['metric_id']}", + ) + for e in idx["edges"] + if e["kind"] == "factor" and e["status"] != "rejected" + ] + try: + check_factor_dag(factor_edges) + except OntologyCycleError as error: + raise ChangeRefused(error.message, code=OPERATION_VALUE, path="/links") from error final, binding_report, synced = _sync_bindings(new_graph, repo.card_id) after = GraphBackedCaliberRepository(final, card_id=repo.card_id) return Change(graph=final, card=repo.card_id, applied=True, diff --git a/backend/app/ledger/continuation.py b/backend/app/ledger/continuation.py new file mode 100644 index 00000000..17682609 --- /dev/null +++ b/backend/app/ledger/continuation.py @@ -0,0 +1,76 @@ +"""Conservative graph append for independently reviewed missing-definition candidates.""" +from __future__ import annotations + +import copy +import re +from typing import Any + + +def append_candidate(original: dict[str, Any], candidate: dict[str, Any], *, prefix: str, + allowed_definitions: set[str]) -> dict[str, Any]: + from app.engine.schema import GraphSpec + + graph, extra = copy.deepcopy(original), copy.deepcopy(candidate) + def one(nodes, kind): + found = [n for n in nodes if n.get('type') == kind] + if len(found) != 1: + raise ValueError('图中入口或口径卡不唯一,请在画布中核对合并;候选已保留') + return found[0] + nodes, added = graph['nodes'], extra['nodes'] + start, card = one(nodes, 'input'), one(nodes, 'metrics') + incoming_start, incoming_card = one(added, 'input'), one(added, 'metrics') + def config(node): + return node['data']['config'] + old_metrics, new_metrics = config(card)['metrics'], config(incoming_card)['metrics'] + ids = [m.get('source_definition_id') for m in new_metrics] + new_names = [m.get('name') for m in new_metrics] + if not ids or None in ids or not set(ids) <= allowed_definitions: + raise ValueError('候选包含未请求的指标定义,请核对;候选已保留') + if not all(new_names) or len(new_names) != len(set(new_names)): + raise ValueError('候选包含同名重复指标,请核对;候选已保留') + if set(ids) & {m.get('source_definition_id') for m in old_metrics}: + raise ValueError('原稿已有这些指标定义,不会覆盖现有实现;候选已保留') + if set(new_names) & {m.get('name') for m in old_metrics}: + raise ValueError('存在同名指标,请确认口径后在画布中合并;候选已保留') + fields = config(start).setdefault('fields', []) + by_name = {f['name']: f for f in fields} + for field in config(incoming_start).get('fields', []): + if field['name'] in by_name and field != by_name[field['name']]: + raise ValueError(f"入口参数「{field['name']}」定义冲突,不会覆盖原参数;候选已保留") + if field['name'] not in by_name: + fields.append(field) + tools = [n for n in added if n.get('type') == 'tool'] + if not tools or any(n.get('type') not in ('input', 'tool', 'metrics', 'report', 'output') for n in added): + raise ValueError('候选取数结构已变化,请在画布中核对合并;候选已保留') + rename = {n['id']: f"{prefix}_{n['id']}" for n in tools} + rename[incoming_start['id']], rename[incoming_card['id']] = start['id'], card['id'] + if set(rename[n['id']] for n in tools) & {n['id'] for n in nodes}: + raise ValueError('候选节点已存在,请刷新后核对') + # Generated SQL uses input parameters, not upstream node references. Refuse edited dependencies. + if any('nodes.' in str(config(n)) for n in tools): + raise ValueError('候选查询依赖其他节点,请在画布中核对合并;候选已保留') + edges = [e for e in extra.get('edges', []) if e['source'] in rename and e['target'] in rename] + expected_edges = {(a['id'], b['id']) for a, b in zip([incoming_start, *tools], [*tools, incoming_card])} + if {(e['source'], e['target']) for e in edges} != expected_edges: + raise ValueError('候选取数链路已变化,请在画布中核对合并;候选已保留') + for metric in new_metrics: + expression = metric.get('expression', '') + refs = set(re.findall(r'\bnodes\.([A-Za-z0-9_]+)', expression)) + if not refs or not refs <= {n['id'] for n in tools}: + raise ValueError('候选指标依赖未合并的节点,请核对;候选已保留') + metric['expression'] = re.sub(r'\bnodes\.([A-Za-z0-9_]+)', lambda m: 'nodes.' + rename[m[1]], expression) + metric['id'] = f"{prefix}_{metric['id']}" + if {m['id'] for m in new_metrics} & {m['id'] for m in old_metrics}: + raise ValueError('指标编号冲突,请核对;候选已保留') + old_metrics.extend(new_metrics) + for node in tools: + node['id'] = rename[node['id']] + nodes.append(node) + graph['edges'].extend({**e, 'source': rename[e['source']], 'target': rename[e['target']], + 'id': f'{prefix}_edge_{index}'} for index, e in enumerate(edges)) + for node in nodes: + contract = config(node).get('contract') + if node.get('type') == 'output' and isinstance(contract, dict) and contract.get('metrics_from') == card['id']: + contract['required'] = [*contract.get('required', []), *[m['id'] for m in new_metrics]] + GraphSpec.model_validate(graph) + return graph diff --git a/backend/app/ledger/docdraft.py b/backend/app/ledger/docdraft.py index 57f7af71..80f84b49 100644 --- a/backend/app/ledger/docdraft.py +++ b/backend/app/ledger/docdraft.py @@ -320,6 +320,14 @@ def normalize_metrics(raw: Any, chunks: dict[str, dict]) -> tuple[list[dict], li questions = cleaned.get("open_questions") cleaned["open_questions"] = [q for q in questions if isinstance(q, str) and q.strip()] \ if isinstance(questions, list) else [] + if "links" in cleaned: + from app.ledger.ontology import attach_docdraft_links + + proposed_links = attach_docdraft_links(cleaned, cleaned.get("links"), chunks) + if proposed_links: + cleaned["links"] = proposed_links + else: + cleaned.pop("links", None) good.append(cleaned) return good, dropped diff --git a/backend/app/ledger/draftplan.py b/backend/app/ledger/draftplan.py index cb17e199..bb34cdd9 100644 --- a/backend/app/ledger/draftplan.py +++ b/backend/app/ledger/draftplan.py @@ -39,10 +39,12 @@ from app.ledger import docdraft #: 一次发给模型的材料上限(按 chunk 边界截断;表目录先截到这个数以内) MAX_INPUT_CHARS = docdraft.MAX_INPUT_CHARS MAX_TABLES_CHARS = 24000 +MIN_DOC_CHARS = 8000 +MIN_TABLES_CHARS = 8000 #: 取数 SQL 的长度上限(字符)。再长的查询不是草稿该承载的 MAX_SQL_CHARS = 20000 #: 这一版提示词的版本(改了 _SYSTEM 就 +1)。写进草稿的提示记录,历史结果才知道是按哪一版起草的。 -PROMPT_VERSION = 1 +PROMPT_VERSION = 2 #: 入口字段能写的类型(apps/spec.FORM_TYPES 的子集;日期用 string + description 说清) ENTRY_TYPES = ("string", "number", "integer", "boolean") @@ -68,7 +70,8 @@ _SYSTEM = """你是业务口径与取数模板的起草助手。读给定的业 硬性约束: 1. 只输出一个 JSON 对象,不要写解释,不要写 Markdown 代码块之外的话。 2. 每组的 sql 是取数模板,只能是**一条只读 SELECT**:不要多语句、不要 INSERT/UPDATE/DELETE/DDL,也不要在注释后面再藏一条。 - 表和列只能来自「数据目录」,表名、列名照抄,不许猜、不许编。 + 业务表和列只能来自「数据目录」,表名、列名照抄,不许猜、不许编。文档明确的常量或日期计算不需要扫业务表; + 这种单行计算可以不用 FROM,Oracle 使用 DUAL。常量仍须引用原文依据。 3. sql 里需要按入口筛选时写 '{{ input.字段名 }}'(字符串)或 {{ input.字段名 }}(数),并在该组的 entry_fields 里声明同名 字段;不要把日期、门店之类的字面量写死。 4. 每条指标都要有 expression,固定写成 cell(nodes.fetch, 0, '列名'),列名是这组 sql 最外层 SELECT 的输出列名。 @@ -78,18 +81,23 @@ _SYSTEM = """你是业务口径与取数模板的起草助手。读给定的业 7. **一次尽量多交几组。** 每组是能共用同一条 SQL 的指标,最多 6 条;不同粒度、不同表、不同时间字段的指标不要硬凑 进同一组。文档里其余的指标不要硬凑,把名称写进 uncovered(字符串数组),下一轮再起草。human 里会列出「已经起草 过的指标」:这些不要重复起草,剩下的未起草指标都写进 uncovered。 -8. **sql 要短。** 能用 GROUP BY 和聚合表达的,不要为每个属性枚举一长串 CASE WHEN 列;输出额度有限,SQL 写长了, - 后面的指标就写不出来了。 +8. **单值的粒度必须对齐。** 每组最终查询必须恰好返回一行。可在子查询按班次聚合,但最外层按文档定义汇总; + 不得直接返回明细或分组行后取第 0 行,也不得用 LIMIT/ROWNUM 隐藏多行。比率不能直接相加,加权规则不得自行更换。 9. 写不下时宁可少写几组、把剩下的写进 uncovered,**也不要让 JSON 半截断掉**。 -10. **同一件事只做一个入口。** 工厂、产线这类维度全图只给一个字段,不要为不同的表各造一个 - (factory_code / gc_code / scx_code 说的是同一件事);跨表的编码差异在 sql 里用 JOIN 或子查询映射, +10. **同一实体只做一个入口。** 例如 factory_code / gc_code 都指工厂时共用参数;产线 scx_code 是另一实体,不能混同。 + 同一实体跨表的编码差异在 sql 里用 JOIN 或子查询映射, 不要让填参数的人重复填等价编码。 -11. **编码和主键不当必填入口。** object_id、line_code 这类字段只有业务上确实每次按它跑时才做入口, - 而且必须是可选(留空不过滤);人填不出来才能跑的入口,等于这份口径跑不了。 -12. **可选维度要写成「空值不过滤」**:`AND ('{{ input.line_code }}' = '' OR t.line_code = '{{ input.line_code }}')`。 - 不这么写,留空会把结果筛成空。 +11. **按对象定义的指标必须提供对象参数。** 产线/工厂不是同一实体,不得把工厂编号直接当产线主键。 + 不可跨对象汇总的指标,其对象参数必须必填,不能用空值扩大到全库。 +12. 只有文档明确允许跨对象汇总的维度才可选;缺少规则时列入未决问题,不自动取消过滤。 13. **入口最多 4 个**,时间范围占两个。再多就把不常变的条件写死在 sql 里(见第 14 条)。 14. **不常变的条件写进 sql,不做入口**:固定筛选段里给出的取值(工厂、产线、属性组、状态)直接写成常量。 +15. 每条指标补 semantic 草稿:{version:1, enabled:false, aliases:[], grain:"输出粒度", + aggregation:"原文的汇总与缺失规则", time:null, entities:[]}。只有材料给全时才补 time + {start:"入口开始字段",end:"入口结束字段",timezone:"业务时区",basis:"calendar_day 或 production_day", + periods:["day"],description:"归属规则"};日指标 periods=["day"],月指标=["month"],两者算法一致且文档允许才同时填。 + 和 entities:[{field:"入口字段",label:"工厂或产线",members:[{value:"已验证编号",label:"名称",aliases:[]}]}]。 + 没有实体字典就 members 不填、entities 留空,列出待补项;不能编造编号,不得把 enabled 写为 true。 输出格式: { @@ -217,10 +225,37 @@ def fit_material(docs: list[dict], chunks: dict[str, dict], guidance: str, def size() -> int: return _chars(prompt(docs, kept, guidance, datasource, tables, covered, fixed)) + def doc_chars() -> int: + return sum(len(str((c or {}).get("content") or "")) for c in kept.values()) + order = _order(kept) + while size() > MAX_INPUT_CHARS and len(order) > 1 and (len(tables) <= MIN_TABLES_CHARS or doc_chars() > MIN_DOC_CHARS): + kept.pop(order.pop()) + doc_truncated = True + if size() > MAX_INPUT_CHARS and len(tables) > MIN_TABLES_CHARS: + excess = size() - MAX_INPUT_CHARS + target = max(MIN_TABLES_CHARS, len(tables) - excess) + if target < len(tables): + tables = tables[:target] + catalog_truncated = True while size() > MAX_INPUT_CHARS and len(order) > 1: kept.pop(order.pop()) doc_truncated = True + if size() > MAX_INPUT_CHARS and order: + chunk_id = order[0] + chunk = dict(kept[chunk_id]) if isinstance(kept[chunk_id], Mapping) else {} + content = str(chunk.get("content") or "") + if len(content) > 512: + excess = size() - MAX_INPUT_CHARS + trimmed = max(512, len(content) - excess) + if trimmed < len(content): + chunk["content"] = content[:trimmed] + kept[chunk_id] = chunk + doc_truncated = True + if size() > MAX_INPUT_CHARS and tables: + excess = size() - MAX_INPUT_CHARS + tables = tables[:max(0, len(tables) - excess)] + catalog_truncated = True if size() > MAX_INPUT_CHARS and order: chunk_id = order[0] chunk = dict(kept[chunk_id]) if isinstance(kept[chunk_id], Mapping) else {} @@ -229,10 +264,6 @@ def fit_material(docs: list[dict], chunks: dict[str, dict], guidance: str, chunk["content"] = content[:max(0, len(content) - excess)] kept[chunk_id] = chunk doc_truncated = True - if size() > MAX_INPUT_CHARS: - excess = size() - MAX_INPUT_CHARS - tables = tables[:max(0, len(tables) - excess)] - catalog_truncated = True return kept, tables, doc_truncated, catalog_truncated @@ -529,6 +560,8 @@ def _sql_problem(sql: str, allowed_tables: set[str]) -> str | None: if not name or name.casefold() in cte_names or short.casefold() in cte_names: continue db = str(table.db or "") + if name.casefold() == "dual" and not db and not table.catalog: + continue # Oracle 的单行计算源,不是需要探查目录的业务表。 keys = {name.casefold(), short.casefold()} if db: keys.add(f"{db}.{name}".casefold()) @@ -536,9 +569,6 @@ def _sql_problem(sql: str, allowed_tables: set[str]) -> str | None: used.append(name) if used: return f"SQL 用了数据目录里没有的表:{'、'.join(dict.fromkeys(used))}" - if not any(True for table in tree.find_all(exp.Table) - if str(table.name or "").casefold() not in cte_names): - return "SQL 没有引用任何表:请对照数据目录写取数" return None @@ -576,6 +606,14 @@ def _metric(item: Mapping, position: int, problems: list[dict]) -> dict | None: grain, time_field = _text(item.get("grain")), _text(item.get("time_field")) numerator, denominator = _text(item.get("numerator")), _text(item.get("denominator")) dimensions, filters = _texts(item.get("dimensions")), _texts(item.get("filters")) + semantic = None + if isinstance(item.get("semantic"), Mapping): + from app.ledger.semantic import QuerySemantics + from pydantic import ValidationError + try: + semantic = QuerySemantics.model_validate({**item["semantic"], "enabled": False}).model_dump(mode="json") + except ValidationError: + problems.append({"name": name, "reason": "问数语义不完整,需确认对象、时间与汇总规则"}) return { "id": str(item.get("id") or "").strip(), "name": name, @@ -585,6 +623,9 @@ def _metric(item: Mapping, position: int, problems: list[dict]) -> dict | None: "decimals": decimals, "expression": expression.strip(), "contract": {"shape": "scalar"}, + **({"source_definition_id": _text(item.get("source_definition_id"))} + if _text(item.get("source_definition_id")) else {}), + **({"semantic": semantic} if semantic else {}), **({"grain": grain} if grain else {}), **({"time_field": time_field} if time_field else {}), **({"numerator": numerator} if numerator else {}), @@ -606,7 +647,8 @@ def _metric(item: Mapping, position: int, problems: list[dict]) -> dict | None: } -def normalize_plan(raw: Any, chunks: Mapping[str, dict], *, allowed_tables: set[str]) -> tuple[dict, list[dict]]: +def normalize_plan(raw: Any, chunks: Mapping[str, dict], *, allowed_tables: set[str], + strict_scalar: bool = False) -> tuple[dict, list[dict]]: """校验模型交回的草案。返回 (可落库的 plan, problems)。 problems 的形状是 {"name": …, "reason": …},name 能认出是哪一条就用名字,认不出按序号说。硬规则:SQL 单条只读、表都在 @@ -629,6 +671,9 @@ def normalize_plan(raw: Any, chunks: Mapping[str, dict], *, allowed_tables: set[ problems.append({"name": "sql", "reason": "没有给出取数 SQL"}) else: reason = _sql_problem(sql_text, allowed_tables) + if reason is None and strict_scalar: + from app.ledger.semantic_sql import scalar_problem + reason = scalar_problem(sql_text) if reason: problems.append({"name": "sql", "reason": reason}) sql_text = "" diff --git a/backend/app/ledger/jobs.py b/backend/app/ledger/jobs.py index ce345a90..2cbace5e 100644 --- a/backend/app/ledger/jobs.py +++ b/backend/app/ledger/jobs.py @@ -7,7 +7,7 @@ from sqlalchemy import select, update from app.core import job_runtime as runtime from app.db.base import SessionLocal, current_owner, utcnow -from app.db.models import CaliberDraft, CaliberDraftJob +from app.db.models import CaliberDraft, CaliberDraftJob, DataSource executor: Callable[[str], Awaitable[None]] | None = None @@ -78,10 +78,15 @@ async def startup(*, live: Any = None) -> int: async def retry(session: Any, draft: CaliberDraft) -> None: - if draft.status != "failed" or draft.workflow_id: + from app.ledger.authoring import require_authoring_source + + require_authoring_source(await session.get(DataSource, draft.datasource_id) if draft.datasource_id else None) + recovering = bool((draft.payload or {}).get("finalization_pending")) + if draft.status != "failed" or (draft.workflow_id and not recovering): raise ValueError("只有尚未生成工作流的失败草稿可以重试") claimed = await session.execute(update(CaliberDraft).where( - CaliberDraft.id == draft.id, CaliberDraft.status == "failed", CaliberDraft.workflow_id.is_(None) + CaliberDraft.id == draft.id, CaliberDraft.status == "failed", CaliberDraft.workflow_id == draft.workflow_id, + CaliberDraft.updated_at == draft.updated_at ).values(status="drafting", error=None).execution_options(synchronize_session=False)) if claimed.rowcount != 1: raise ValueError("草稿状态已经变化,请刷新后再重试") diff --git a/backend/app/ledger/metric_matching.py b/backend/app/ledger/metric_matching.py new file mode 100644 index 00000000..14faa07d --- /dev/null +++ b/backend/app/ledger/metric_matching.py @@ -0,0 +1,18 @@ +"""Deterministic candidate retrieval. Matching a name is not permission to execute a metric.""" +from __future__ import annotations + +import re +from collections.abc import Iterable + +from app.ledger.semantic import mention + + +def score(question: str, name: str, aliases: Iterable[str] = ()) -> int: + terms = [name, *aliases] + exact = [len(term) for term in terms if isinstance(term, str) and mention(question, term)] + if exact: + return 100 + max(exact) + # OEE/RTY/TEEP are discoverable even when the old metric only has a qualified display name. + # This only produces candidates; it never selects between their different business formulas. + tokens = set(re.findall(r"(? dict: + nodes = {n["id"]: n for n in graph.get("nodes", [])} + results = [] + for card in nodes.values(): + cfg = card.get("data", {}).get("config", {}) + if card.get("type") != "metrics" or cfg.get("caliber_from"): + continue + for metric in cfg.get("metrics", []): + findings = [] + + def add(code, message): + if not any(f["code"] == code and f["message"] == message for f in findings): + findings.append({"code": code, "message": message}) + + selected = closure.build(graph, workflow_id, version, card["id"], metric_ids=[metric["id"]]) + for problem in selected.unresolved: + add("dependency", problem["message"]) + raw = metric.get("semantic") + sem = None + if not raw: + add("semantic_missing", "补齐问数定义:常用叫法、业务粒度、汇总规则、对象映射和日期归属") + else: + try: + sem = QuerySemantics.model_validate(raw) + except ValueError: + add("semantic_invalid", "问数定义不完整,请在指标的问数设置中修复") + if sem and not sem.enabled: + add("semantic_disabled", "问数定义尚未启用,验证并确认后才能自动查询") + fields = {f["name"]: f for f in selected.required_entry_fields} + if sem: + mapped = {e.field for e in sem.entities} + if sem.time: + mapped |= {sem.time.start, sem.time.end} + for name in sorted(mapped - fields.keys()): + add("parameter_unused", f"问数参数 {name} 未接入这项指标的取数依赖") + for name, field in fields.items(): + if field.get("required") and field.get("default") is None and name not in mapped: + add("parameter_missing", f"必填参数 {name} 尚未绑定对象或日期") + for field in fields.values(): + desc = str(field.get("description") or "") + if any(word in desc for word in ("含当天", "包含结束", "含结束")): + add("inclusive_end", "旧结束日期包含当天;接入自动日期前,需将取数和说明统一为左闭右开,防止多算或漏算一天") + for query in selected.queries: + qcfg = nodes[query.node_id].get("data", {}).get("config", {}) + sql = (qcfg.get("args") or {}).get("sql", "") + tool = query.tool + source = tool.removeprefix("db_query__") + if source not in sources: + add("source_missing", f"数据源 {source} 不在本机可用目录中,尚不能做真实取数验证") + if (metric.get("contract") or {}).get("shape", "scalar") == "scalar": + if problem := scalar_problem(sql): + add("scalar_unproven", f"取数 {query.node_id}:{problem}") + if "%" in str(metric.get("unit", "")) or "OEE" in str(metric.get("name", "")).upper(): + note = "比率需核对分子、分母与权重,不能通过 SUM、平均值或截取首行机械修复" + else: + note = "先确认业务粒度,再选择求和、去重计数或常量取数;缺数据与真实零值分开处理" + results.append({"card_id": card["id"], "metric_id": metric["id"], "name": metric.get("name", metric["id"]), + "findings": findings, "query_count": len(selected.queries), "guidance": note}) + return {"workflow_id": workflow_id, "revision": version, "metrics": results, + "metrics_total": len(results), "metrics_needing_work": sum(bool(m["findings"]) for m in results), + "runtime_verified": False, + "note": "这是静态迁移检查,不代表查询结果已验证。原定义、已发布版本和权限保持不变;修复后需用独立期望试运行。"} diff --git a/backend/app/ledger/nlchange.py b/backend/app/ledger/nlchange.py index c6f1b0d3..8b0f546f 100644 --- a/backend/app/ledger/nlchange.py +++ b/backend/app/ledger/nlchange.py @@ -721,6 +721,7 @@ _AUTO_FILLERS_SORTED: tuple[str, ...] = tuple(sorted(_AUTO_FILLERS, key=len, rev #: 额外的分析意图词(增长率、占比、同比、环比、拆解、趋势、明细、排行……):**刻意不算通用措辞**——它们留在剩余业务 #: 语义里,使「和指标名一字不差」不成立,于是回旧路径。列在这里是为了让人一眼看到这条边界(测试也核对它俩不相交)。 _AUTO_ANALYSIS_WORDS: tuple[str, ...] = ( + "分析", "为什么", "为何", "原因", "归因", "预测", "建议", "增长率", "增速", "增幅", "降幅", "增长", "占比", "比重", "比例", "百分比", "同比", "环比", "趋势", "走势", "拆解", "分解", "下钻", "明细", "排行", "排名", "对比", "比较", "差异", "变化", "波动", "分布", "构成", ) @@ -755,6 +756,8 @@ async def _auto_cards(session: Any) -> list[dict[str, Any]]: from sqlalchemy import select from app.apps.versions import is_published + from app.apps.tree import node_map + from app.look.calibers import measured_inputs from app.db.models import Workflow, WorkflowVersion rows = (await session.execute( @@ -765,6 +768,7 @@ async def _auto_cards(session: Any) -> list[dict[str, Any]]: for workflow, row in rows: if not is_published(row, current=workflow.published_version) or row.retired_at is not None: continue + graph_nodes = node_map(row.graph or {}) for node in (row.graph or {}).get("nodes") or []: if not isinstance(node, Mapping) or node.get("type") != "metrics": continue @@ -775,10 +779,16 @@ async def _auto_cards(session: Any) -> list[dict[str, Any]]: metrics = [dict(m) for m in config.get("metrics") or [] if isinstance(m, Mapping) and m.get("id")] if not metrics: continue + metric_sources = {} + for metric in metrics: + inputs, reason = measured_inputs(graph_nodes, [metric]) + metric_sources[str(metric["id"])] = ({item["tool"] for item in inputs.values()} + if reason is None else None) out.append({"workflow_id": workflow.id, "workflow_name": str(workflow.name or ""), "version": int(row.version), "card_id": str(node.get("id") or ""), "card_title": str(data.get("label") or node.get("id") or ""), - "caliber": str(config.get("caliber") or ""), "metrics": metrics}) + "caliber": str(config.get("caliber") or ""), "metrics": metrics, + "metric_source_tools": metric_sources}) return out @@ -823,10 +833,10 @@ def _auto_trim(text: str) -> str: return out -def _auto_remainder(question: str, hints: Sequence[str]) -> str: +def _auto_remainder(question: str, hints: Sequence[str], *, remove_dates: bool = True) -> str: """这句话剩下的**业务语义**(归一化后):去掉明确完整的日期/单月、句子里点到的口径/工作流/卡名(`提示的`), 再去掉两端的通用问法、查询、统计措辞。剩下的必须和某个指标 name / 唯一 synonym **一字不差**才谈得上高置信。""" - norm = _normalize(_auto_remove_dates(question)) + norm = _normalize(_auto_remove_dates(question) if remove_dates else question) for hint in sorted({_normalize(item) for item in hints}, key=len, reverse=True): if len(hint) >= 2 and hint + "的" in norm: norm = norm.replace(hint + "的", "") @@ -955,18 +965,22 @@ def _permitted(allowed: Any) -> set[tuple[str, int]] | None: async def auto_route(session: Any, *, question: str, purpose: str | None = None, - allowed: Any = None) -> Route | None: - """普通问数的**保守默认路由**:高置信确定的请求交回一条可以直接交给固定执行的请求,否则 None。 + allowed: Any = None, source_tools: set[str] | None = None) -> Route | None: + """普通问数先匹配已发布的问数定义,再兼容没有语义元数据的旧口径。 + + 声明了 semantic 的指标:按业务时区、支持周期与显式对象词典绑定参数,完整匹配才执行;已命中但含糊的请求 + 返回澄清,避免每次重新探索底表。未匹配或分析问题继续旧探索路径。下面是旧口径的兼容规则。 只在下面每一件都说得清时才命中:按推荐发布版的指标 name / 唯一 synonym 规范化精确匹配**唯一**一个指标(跨卡同名或 一句指到不止一个就不自动);这句话没有含糊的业务条件、也没有和那次口径声明的算法对不上;派生出来的这次入口取值 - 拿得全、类型可证。任何一处不确定(含指不到、派生不出、版本没发布)都交回 None——调用方原样走旧的 Agent 路径, - 不向用户多说什么、也不替它挑一份口径。**零模型**:只看库里的发布版与图,不问模型也不做模糊相似。 + 拿得全、类型可证。旧口径已匹配但日期或对象不明确时返回迁移澄清;未匹配、定义变更和分析请求继续探索。 + **零模型**:只看库里的发布版与图,不问模型也不做模糊相似。 `allowed` 是这个人能用的 `(workflow_id, version)` 集合(资产授权,见 `_permitted`):匹配照旧按**全部发布口径**看, 所以「没匹配到」和「命中了但没权」分得开——唯一命中却不在 allowed 里交回**终止型**的 `ROUTE_REFUSED` / `ROUTE_NOT_PERMITTED`,调用方照实说没有授权,**不回退旧的 Agent 路径**(那条路会去翻底表)。allowed 为 None 是调用方 - 已放行(搭建者及以上),不做这道闸。歧义、参数不全、没匹配到仍然交回 None,由调用方走旧路。""" + 已放行(搭建者及以上),不做这道闸。澄清只列有权查看的候选,不借迁移建议暴露未授权定义。 + source_tools 非空参数表示调用方明确限定的数据源工具集合,匹配及澄清前即收窄候选;None 保留全局匹配。""" from app.ledger import derive, querycore text = str(question or "").strip() @@ -975,10 +989,25 @@ async def auto_route(session: Any, *, question: str, purpose: str | None = None, purpose = str(purpose or querycore.DEFAULT_PURPOSE) permitted = _permitted(allowed) cards = await _auto_cards(session) + if source_tools is not None: + # Scope applies before clarification as well as execution. Unrelated or removed + # databases must not hijack an otherwise exploratory question by metric name. + scoped = [] + for card in cards: + dependencies = card.get("metric_source_tools", {}) + metrics = [metric for metric in card["metrics"] + if dependencies.get(str(metric["id"])) + and dependencies[str(metric["id"])] <= source_tools] + if metrics: + scoped.append({**card, "metrics": metrics}) + cards = scoped + semantic_route = await _semantic_route(session, cards, text, permitted, purpose) + if semantic_route is not None: + return semantic_route hints = [item for card in cards for item in (card["workflow_name"], card["caliber"], card["card_title"])] candidates = _auto_match(cards, _auto_remainder(text, hints), _normalize(text)) if len(candidates) != 1: - return None + return _legacy_clarification(cards, text, permitted) chosen = candidates[0] # 授权闸在最前面:唯一命中但这个人的授权范围里没有这一版时,结果只能是「没授权」,不能落到下面任何一条会把这句话 # 交给旧 Agent 路径(去翻底表)的分支上 @@ -999,7 +1028,7 @@ async def auto_route(session: Any, *, question: str, purpose: str | None = None, return None inputs = _auto_inputs(selected, text) if inputs is None: - return None + return _legacy_clarification(cards, text, permitted) return Route(ROUTE_EXECUTE, ROUTE_AUTO, f"自动命中固定口径「{chosen['caliber'] or chosen['card_title']}」的指标,直接按固定口径执行,不重复解析/规划", request={"workflow_id": chosen["workflow_id"], "card_id": chosen["card_id"], @@ -1007,6 +1036,115 @@ async def auto_route(session: Any, *, question: str, purpose: str | None = None, "version": chosen["version"], "purpose": purpose}) +def _legacy_clarification(cards: Sequence[Mapping[str, Any]], text: str, permitted: set | None) -> Route | None: + """A known legacy metric with object/date language needs migration, not another schema expedition. + Keep analytical and unrelated requests on the exploration path. + """ + from app.ledger.metric_matching import score + + if any(word in text for word in _AUTO_ANALYSIS_WORDS): + return None + if not re.search(r"昨天|昨日|今天|前天|上个月|本月|工厂|产线|(? Route | None: + """Semantic bindings precede the legacy exact-name path. A known but incomplete request + asks for clarification rather than re-discovering the database with an Agent. + Only explicitly authored bindings consume entity identifiers and relative dates. + """ + from pydantic import ValidationError + from app.ledger import derive, params, semantic + from app.apps.forms import input_schema, validate_form + + hits = [] + for card in cards: + for metric in card["metrics"]: + raw = metric.get("semantic") + if not isinstance(raw, Mapping): + continue + terms = [*_auto_metric_terms(metric), *(raw.get("aliases") or [])] + if any(semantic.mention(text, term) for term in terms if isinstance(term, str)): + hits.append({**card, "metric": metric, "metric_id": metric["id"]}) + if not hits: + return None + allowed = [h for h in hits if permitted is None or (h["workflow_id"], h["version"]) in permitted] + if not allowed: + return Route(ROUTE_REFUSED, ROUTE_NOT_PERMITTED, NOT_PERMITTED) + # Never reveal inaccessible candidates. An exact metric name narrows an ambiguous alias. + analysis_text = text + for hit in allowed: + terms = [*_auto_metric_terms(hit["metric"]), *(hit["metric"]["semantic"].get("aliases") or [])] + for term in sorted((t for t in terms if isinstance(t, str) and t), key=len, reverse=True): + analysis_text = re.sub(re.escape(term), " ", analysis_text, flags=re.I) + if any(word in analysis_text for word in _AUTO_ANALYSIS_WORDS): + return None + exact = [h for h in allowed if semantic.mention(text, str(h["metric"].get("name") or ""))] + if exact: + allowed = exact + hinted = [h for h in allowed if _auto_hinted(h, _normalize(text))] + if hinted: + allowed = hinted + if len(allowed) != 1: + names = list(dict.fromkeys(str(h["metric"].get("name") or "指标") for h in allowed)) + return Route(ROUTE_CLARIFY, ROUTE_AMBIGUOUS, "有多种指标口径,请指定要查询的一种:" + "、".join(names[:8])) + chosen = allowed[0] + try: + model = semantic.QuerySemantics.model_validate(chosen["metric"]["semantic"]) + except ValidationError: + return Route(ROUTE_CLARIFY, ROUTE_AMBIGUOUS, "这份指标的问数定义尚不完整,请先由维护者确认") + if not model.enabled: + return Route(ROUTE_CLARIFY, ROUTE_AMBIGUOUS, "已找到这份指标,但对象、时间和汇总规则尚未确认,暂不能自动取数") + values, remaining, problems = semantic.bind(text, model) + hints = [chosen["workflow_name"], chosen["caliber"], chosen["card_title"]] + remainder = _auto_remainder(remaining, hints, remove_dates=False) + terms = [*_auto_metric_terms(chosen["metric"]), *model.aliases] + if remainder not in {_normalize(term) for term in terms}: + # Analytical questions still use the established analysis tools. Extra filters may never + # be discarded just because the metric and identifier matched. + if any(word in remaining for word in _AUTO_ANALYSIS_WORDS): + return None + problems.append("还有未确认的查询条件,请明确对象、日期和指标口径") + if problems: + return Route(ROUTE_CLARIFY, ROUTE_AMBIGUOUS, ";".join(dict.fromkeys(problems))) + try: + pinned = await derive.read_pinned(session, chosen["workflow_id"], chosen["version"], chosen["card_id"]) + selected = derive.select_pinned(pinned, metric_ids=[chosen["metric_id"]]) + except derive.NotDerivable: + return Route(ROUTE_CLARIFY, ROUTE_AMBIGUOUS, "已找到指标,但它的取数依赖尚未就绪,请先修复该口径") + fields = list(selected.entry_fields) + declared = {f["name"] for f in fields} + if set(values) - declared: + return Route(ROUTE_CLARIFY, ROUTE_AMBIGUOUS, "指标的对象或日期映射没有接入取数参数,请先修复口径") + values = params.resolve(fields, values) + if params.check_nodes(selected.fetches, fields=fields, values=values) or validate_form(input_schema(fields), values): + return Route(ROUTE_CLARIFY, ROUTE_AMBIGUOUS, "这份指标还有必填参数未提供,请补齐后查询") + return Route(ROUTE_EXECUTE, ROUTE_AUTO, "已按确认的对象、时间和指标口径匹配,直接执行固定查询", + request={"workflow_id": chosen["workflow_id"], "card_id": chosen["card_id"], + "version": chosen["version"], "metric_ids": [chosen["metric_id"]], + "inputs": values, "purpose": purpose}) + + __all__ = ["KIND_AMBIGUOUS", "KIND_MISSING", "KIND_NO_MATCH", "NOT_PERMITTED", "PROPOSAL_AMBIGUOUS", "PROPOSAL_NONE", "ROUTE_AMBIGUOUS", "ROUTE_AUTO", "ROUTE_CLARIFY", "ROUTE_CONDITION_MISMATCH", "ROUTE_EXECUTE", "ROUTE_EXPLORE", "ROUTE_METRICS_REQUIRED", "ROUTE_NO_CALIBER", "ROUTE_NOT_PERMITTED", diff --git a/backend/app/ledger/semantic.py b/backend/app/ledger/semantic.py new file mode 100644 index 00000000..5bde9560 --- /dev/null +++ b/backend/app/ledger/semantic.py @@ -0,0 +1,153 @@ +"""Published query semantics. These are author-maintained bindings, never guesses from SQL. + +One metric describes one executable grain. Aggregation remains in the reviewed implementation; +the query router cannot invent a rollup, reinterpret an identifier, or discard a filter. +""" +from __future__ import annotations + +import re +from datetime import datetime, timedelta, timezone +from typing import Literal +from zoneinfo import ZoneInfo, ZoneInfoNotFoundError + +from pydantic import BaseModel, ConfigDict, Field, model_validator + + +class Member(BaseModel): + model_config = ConfigDict(extra="forbid", str_strip_whitespace=True) + value: str = Field(min_length=1, max_length=100) + label: str = Field(default="", max_length=100) + aliases: list[str] = Field(default_factory=list, max_length=30) + + +class EntityBinding(BaseModel): + model_config = ConfigDict(extra="forbid", str_strip_whitespace=True) + field: str = Field(min_length=1, pattern=r"^[A-Za-z_][A-Za-z0-9_]*$") + label: str = Field(min_length=1, max_length=50) + entity_key: str | None = Field(default=None, pattern=r"^[a-z][a-z0-9_]{0,63}$") + members: list[Member] = Field(min_length=1, max_length=1000) + + +class TimeBinding(BaseModel): + model_config = ConfigDict(extra="forbid") + start: str = Field(pattern=r"^[A-Za-z_][A-Za-z0-9_]*$") + end: str = Field(pattern=r"^[A-Za-z_][A-Za-z0-9_]*$") + timezone: str + basis: Literal["calendar_day", "production_day"] + periods: list[Literal["day", "month"]] = Field(default_factory=lambda: ["day"], min_length=1, max_length=2) + description: str = Field(min_length=1, max_length=500) + + @model_validator(mode="after") + def valid(self): + try: + ZoneInfo(self.timezone) + except (ZoneInfoNotFoundError, ValueError): + raise ValueError("必须提供有效的业务时区") from None + if self.start == self.end: + raise ValueError("开始和结束参数不能相同") + return self + + +class QuerySemantics(BaseModel): + model_config = ConfigDict(extra="forbid", str_strip_whitespace=True) + version: Literal[1] = 1 + enabled: bool = False + aliases: list[str] = Field(default_factory=list, max_length=30) + grain: str = Field(min_length=1, max_length=200) + aggregation: str = Field(min_length=1, max_length=1000) + time: TimeBinding | None = None + entities: list[EntityBinding] = Field(default_factory=list, max_length=8) + + @model_validator(mode="after") + def distinct_fields(self): + names = [e.field for e in self.entities] + if self.time: + names += [self.time.start, self.time.end] + if len(names) != len(set(names)): + raise ValueError("语义参数绑定不能重复") + return self + + +def now() -> datetime: + return datetime.now(timezone.utc) + + +def interval(text: str, binding: TimeBinding, *, at: datetime | None = None): + """Resolve one complete day/month or one relative calendar label, in the declared timezone. + + Production-day parameters are date labels; SQL owns the approved shift attribution. + Multiple dates are ambiguous, not silently reduced to the first date. + """ + from app.ledger import nlchange + + relative = list(re.finditer(r"今天|昨天|昨日|前天", text)) + spans = nlchange._auto_date_spans(text) + unique = [] + for start, end in spans: + if not unique or start >= unique[-1][1]: + unique.append((start, end)) + if len(relative) + len(unique) != 1: + return None, text + if relative: + if "day" not in binding.periods: + return None, text + match = relative[0] + day = (at or now()).astimezone(ZoneInfo(binding.timezone)).date() + day -= timedelta(days={"今天": 0, "昨天": 1, "昨日": 1, "前天": 2}[match.group()]) + dates = (day.isoformat(), (day + timedelta(days=1)).isoformat()) + start, end = match.span() + else: + start, end = unique[0] + phrase = text[start:end] + period = "day" if (nlchange._AUTO_DAY_CN.fullmatch(phrase) or nlchange._AUTO_DAY_DASH.fullmatch(phrase)) else "month" + if period not in binding.periods: + return None, text + dates = nlchange._auto_date_interval(phrase) + return dates, text[:start] + " " + text[end:] + + +def mention(text: str, phrase: str) -> bool: + """ASCII identifiers must match as a whole: 1050 is not 10501, OEE is not TEEP/OEEfoo.""" + if not phrase.strip(): + return False + left = r"(? tuple[dict, str, list[str]]: + """Consume only recognized dates and explicitly registered entity values. Leave every other word.""" + values: dict = {} + problems: list[str] = [] + remaining = text + if semantic.time: + dates, remaining = interval(remaining, semantic.time) + if dates is None: + periods = "、".join({"day": "单日", "month": "整月"}[p] for p in semantic.time.periods) + problems.append(f"这份指标支持{periods}查询,请提供相应日期;其他周期需选择匹配口径") + else: + values.update(zip((semantic.time.start, semantic.time.end), dates)) + # Match on the original remainder, so aliases shared by different entity types stay ambiguous. + matches = [] + for entity in semantic.entities: + found = [(m, term) for m in entity.members for term in [m.value, m.label, *m.aliases] + if mention(remaining, term)] + ids = {m.value for m, _ in found} + if len(ids) != 1: + problems.append(f"请明确一个{entity.label}") + continue + values[entity.field] = next(iter(ids)) + matches.append((entity, found)) + owners: dict[str, set[str]] = {} + for entity, found in matches: + for _, term in found: + owners.setdefault(term.lower(), set()).add(entity.field) + if any(len(fields) > 1 for fields in owners.values()): + problems.append("这个编号对应多个对象类型,请明确对象") + for entity, found in matches: + for term in sorted({t for _, t in found}, key=len, reverse=True): + left = r"(? dict[str, Any]: + items, rules, warnings = [], [], [] + hashes = {} + occurrences = {} + for document in documents: + doc_id = str(document["document_id"]) + section, headers = "", [] + text = str(document.get("content") or "") + hashes[doc_id] = hashlib.sha256(text.encode()).hexdigest() + offset = 0 + for number, line in enumerate(text.splitlines(keepends=True), 1): + clean = line.strip() + start = offset + offset += len(line) + if clean.startswith("#"): + section = clean.lstrip("# ") + headers = [] + continue + if not clean.startswith("|"): + headers = [] + if clean: + rules.append({"document_id": doc_id, "section": section, "text": clean, + "line": number, "span": [start, offset]}) + continue + cells = [c.strip().replace("
", "\n") for c in re.split(r"(? bool: + expected = catalog.get("document_hashes") or {} + actual = {str(d["document_id"]): hashlib.sha256(str(d.get("content") or "").encode()).hexdigest() + for d in documents} + return bool(expected) and expected == actual + + +def implementation_scope(catalog: dict, selected: list[str] | None = None) -> tuple[list[str], list[dict]]: + """Default to all unambiguous, available definitions. Never guess between same-name definitions.""" + from collections import Counter + + metrics = [i for i in catalog.get("items", []) if i["kind"] == "metric"] + eligible = {i["id"]: i for i in metrics if i["status"] != "unavailable"} + if selected is not None: + wanted = list(dict.fromkeys(selected)) + if not wanted or set(wanted) - eligible.keys(): + raise ValueError("只能选择有来源、可实现的指标定义;无数据源的定义不能自动推算") + names = [eligible[key]["name"] for key in wanted] + if len(names) != len(set(names)): + raise ValueError("所选项包含同名的不同定义,请确认要采用的口径后再处理") + return wanted, [] + counts = Counter(i["name"] for i in eligible.values()) + wanted, issues = [], [] + for item in metrics: + reason = ("原文注明无数据源,待补充来源" if item["status"] == "unavailable" else + "存在同名的不同定义,需确认采用的口径" if counts[item["name"]] > 1 else "") + if reason: + issues.append({"definition_id": item["id"], "name": item["name"], "reason": reason}) + else: + wanted.append(item["id"]) + return wanted, issues + + +def _compact_definition(item: dict[str, Any], chunk_hints: dict[str, str] | None = None) -> dict[str, Any]: + redundant = {"符号", "量", "指标", "指标名称", "名称", "公式", "源字段/来源", "缺失/边界处理", "口径/说明", "实现"} + extra = {k: v for k, v in (item.get("fields") or {}).items() if v and k not in redundant} + out: dict[str, Any] = { + "id": item["id"], + "name": item["name"], + "kind": item["kind"], + } + if item.get("section"): + out["section"] = item["section"] + if item.get("formula"): + out["formula"] = item["formula"] + if item.get("policy"): + out["policy"] = item["policy"] + if item.get("dependencies"): + out["dependencies"] = item["dependencies"] + if extra: + out["extra_fields"] = extra + if item.get("kind") == "metric": + quote = (item.get("source") or {}).get("quote") or "" + if quote: + out["quote"] = quote + chunk_id = (chunk_hints or {}).get(item["id"]) + if chunk_id: + out["chunk"] = chunk_id + return out + + +def implementation_guidance(catalog: dict, selected: list[str], + chunk_hints: dict[str, str] | None = None) -> str: + """Carry source policies and all symbolic definitions into implementation, including overrides. + Selected IDs control scope; formula dependencies don't become user-facing metrics by accident. + """ + import json + + wanted = set(selected) + documents = {item["source"]["document_id"] for item in catalog.get("items", []) if item["id"] in wanted} + items = [_compact_definition(item, chunk_hints) for item in catalog.get("items", []) + if item["id"] in wanted or (item["kind"] == "input" and item["source"]["document_id"] in documents)] + rules = [{"section": r["section"], "text": r["text"]} if r.get("section") else r["text"] + for r in catalog.get("rules", []) if r["document_id"] in documents and r.get("text")] + return ("先按以下已选择的定义建立依赖,再绑定数据目录;不要把每条明细取第一行当成周期指标。\n" + "只能实现选中的指标;基础量是依赖,不要额外作为输出指标。原文规则与例外必须保留。\n" + "每条输出指标带 source_definition_id,写所选定义的 id。一行定义有多个量时可以拆成多条,均指回该定义,名称区分清楚。\n" + "每条指标的 citations 必须逐字抄原文(不少于 8 字),chunk 写逐段原文方括号里的段落 id(定义给出 chunk 时可直接用)。\n" + "配置覆盖、生产日、实体映射等缺少依据时列出未决问题,不猜、不换计算方法。\n" + "加权平均与比值之和必须区别;未开班显示为零不等于计算值为零。\n" + "声明为单值的查询最外层必须收敛到一行;不能用 LIMIT、ROWNUM 或取第一行掩盖多行。\n" + + json.dumps({"definitions": items, "global_rules": rules}, ensure_ascii=False)) diff --git a/backend/app/ledger/semantic_sql.py b/backend/app/ledger/semantic_sql.py new file mode 100644 index 00000000..b09dcecb --- /dev/null +++ b/backend/app/ledger/semantic_sql.py @@ -0,0 +1,73 @@ +"""Conservative cardinality proof for newly generated scalar metric queries. + +This is an authoring check, not a business-formula verifier. Runtime contracts still check +actual rows, truncation and nulls. Unsupported query shapes require an explicit rewrite. +""" +from __future__ import annotations + +import re + +import sqlglot +from sqlglot import exp + + +def scalar_problem(sql: str) -> str | None: + body = re.sub(r"\{\{\s*input\.[A-Za-z_][\w-]*\s*\}\}", "?", sql) + try: + tree = sqlglot.parse_one(body) + except sqlglot.errors.ParseError: + return "无法证明取数为单行,请将最终查询改为无分组聚合" + if tree.find(exp.Limit, exp.Offset) or any(c.name.upper() == "ROWNUM" for c in tree.find_all(exp.Column)): + return "标量指标不能用 LIMIT / OFFSET / ROWNUM 截取明细来伪装单值" + + def single(query: exp.Expression, inherited: dict, visiting: frozenset[str] = frozenset()) -> bool: + if isinstance(query, exp.Subquery): + return single(query.this, inherited, visiting) + if not isinstance(query, exp.Select): + return False + if any(query.args.get(key) for key in ("group", "having", "qualify")): + return False + ctes = dict(inherited) + with_ = query.args.get("with_") + if with_: + ctes.update({c.alias.casefold(): c.this for c in with_.expressions}) + + def scoped(node): + # Subqueries have their own cardinality; windows are not collapsing aggregates. + yield node + if not isinstance(node, (exp.Subquery, exp.Select, exp.Window, exp.AggFunc)): + for child in node.iter_expressions(): + yield from scoped(child) + + projections = [n for projection in query.expressions for n in scoped(projection)] + if any(isinstance(n, exp.AggFunc) for n in projections): + return not any(isinstance(n, (exp.Column, exp.Star, exp.Window)) for n in projections) + # A WHERE clause may remove the only row even when every source is scalar. + if query.args.get("where"): + return False + sources = [] + from_ = query.args.get("from_") + if from_: + sources.append(from_.this) + for join in query.args.get("joins") or []: + # Conditional/inner joins can remove a row. CROSS of scalar sources stays scalar. + if join.args.get("on") or join.args.get("using"): + return False + sources.append(join.this) + for source in sources: + if isinstance(source, exp.Subquery): + if not single(source, ctes, visiting): + return False + elif isinstance(source, exp.Table): + name = source.name.casefold() + if name == "dual" and not source.db: + continue + if name not in ctes or name in visiting or not single(ctes[name], ctes, visiting | {name}): + return False + else: + return False + return True + + if not single(tree, {}): + return "标量指标的最终取数未收敛为单行:请先按业务规则聚合,再计算比率;不能取明细第 0 行" + return None diff --git a/backend/tests/fixtures/oee_metric_definitions.md b/backend/tests/fixtures/oee_metric_definitions.md new file mode 100644 index 00000000..0128ce21 --- /dev/null +++ b/backend/tests/fixtures/oee_metric_definitions.md @@ -0,0 +1,107 @@ +# TEEP / OEE 及相关指标公式总表 + +> 依据:项目当前代码(整理日期 2026-11-03)。 +> 范围:日报、周报、月报、看板共用的 OEE、TEEP、设备计划利用率及相关效率指标。 +> 说明:日报/周月报的比率类指标可由「配置 → 计算口径」里的 `metric_definitions` 覆盖; +> 下表是**代码内置默认口径**,配置公式生效时以配置为准(`scripts/seed_config.py:192`、`common/formula.py`)。 + +## 0. 符号约定 + +| 符号 | 含义 | 源字段/来源 | +|---|---|---| +| `T_open` | 计划开班时长(秒) | `ATTRIBUTE01` | +| `T_stop` | 计划内停机时长(秒) | `ATTRIBUTE05` | +| `T_run_plan` | 计划运行时长 = `T_open − T_stop` | 计算值 | +| `T_open_act` | 实际开班时长(秒) | `ATTRIBUTE02` | +| `T_run_act` | 实际运行时长(秒) | `ATTRIBUTE03` | +| `T_stop_act` | 实际停机时长(秒) | `ATTRIBUTE04` | +| `T_stop_unp` | 计划外停机时长(秒) | `ATTRIBUTE06` | +| `V_plan` | 计划线速(秒/圈) | `ATTRIBUTE12` | +| `M_plan` | 开班量产模架数 | `ATTRIBUTE14` | +| `M_trial` | 开班试制模架数 | `ATTRIBUTE15` | +| `M_actual` | 实际生产模架数 | `ATTRIBUTE31` | +| `M_scrap` | 报废模架数 | `MES_BHGSJ` × `MES_JZHMJXS.MJXS` | +| `M_good` | 良品/有效模架数 = `M_actual − M_scrap` | 计算值 | +| `M_line` | 产线模架数 | 产线主数据 `IOT_DN_PRODUCTIONLINE.BEAT` | +| `N_cycle` | 班次总圈数 | `ATTRIBUTE54` | +| `N_no_stop` | 无停机圈数 | `ATTRIBUTE55` | +| `OEE_src` | 源系统 OEE | `ATTRIBUTE36` | +| `OEE_eff` | 有效 OEE | `common/teep.py:190` | +| `T_nat` | 自然时长(秒) | 班次 43,200;天 86,400;周期 = 天数 × 86,400;多条线合并时再 × 产线条数 | + +## 1. 基础量 + +| 量 | 公式 | 缺失/边界处理 | 实现 | +|---|---|---|---| +| 计划运行时长 | `T_run_plan = max(0, T_open − T_stop)` | `T_open`/`T_stop` 任一缺失 → `None`,不写 0 | `daily-report/api_daily_report_generator_modified.py:1560`、`common/teep.py:72` | +| 理论圈数 | `planned_cycle_count = T_run_plan ÷ V_plan` | `V_plan ≤ 0` → OEE 不算 | `daily-report/api_daily_report_generator_modified.py:1496` | +| 计划每圈模架数 | `M_cycle = M_plan + M_trial` | 任一缺失 → OEE 不算;`≤ 0` → OEE 不算 | 同上 `:1479` | +| 理论模架数 | `M_theory = planned_cycle_count × M_cycle` | `≤ 0` → OEE 不算 | 同上 `:1497` | +| 报废模架数 | `M_scrap = Σ(报废零件数 × MJXS)` | 无报废记录 → 0;缺 `MJXS` → 按默认系数 **0.5** 折算并点名 | 同上 `:1417-1459` | +| 良品模架数 | `M_good = M_actual − M_scrap` | 负数 → OEE 不算 | 同上 `:1498`、`:1504` | +| 模架位次 | `M_plan_slots = (M_plan + M_trial) × N_cycle`;`M_line_slots = M_line × N_cycle` | 发泡线专用;装配线无此项 | `common/teep.py:114` | +| 自然时长 | `T_nat = 43,200`(班次)/ `86,400`(天)/ `周期天数 × 86,400`(周月) | 多线合并时 `T_nat = 单线秒数 × 产线条数` | `common/teep.py:49`、`weekly-report/api_weekly_report_generator.py:1176` | + +## 2. OEE 系列 + +| 指标 | 公式 | 口径/说明 | 实现 | +|---|---|---|---| +| 发泡线公式 OEE(单班) | `OEE_formula = M_good ÷ M_theory × 100`
展开:`(M_actual − M_scrap) ÷ [((T_open − T_stop) ÷ V_plan) × (M_plan + M_trial)] × 100` | 本项目**不是**经典 `时间开动率 × 性能开动率 × 合格率` 三率相乘;OEE > 100% 属正常超产,> 200% 判采集异常 | `daily-report/api_daily_report_generator_modified.py:1398-1512` | +| 装配线 OEE | `OEE_assembly = OEE_src`(`ATTRIBUTE36`) | 装配线无模架、无计划线速,公式无从算起 | `common/teep.py:206-209` | +| 有效 OEE(报表统一口径) | `OEE_eff = OEE_formula`(发泡线);`OEE_eff = OEE_src`(装配线) | 发泡线算不出时返回 `None`,**不回退** `ATTRIBUTE36` | `common/teep.py:190-209` | +| 班次日/单班 OEE | 已开班取 `OEE_eff`;未开班仪表盘显示 0% | 未开班不是「OEE=0」,只是不展示源表默认值 | `daily-report/api_daily_report_generator_modified.py:2306-2326` | +| 日 OEE(全天聚合) | `Σ(OEE_eff_i × T_open_act_i) ÷ Σ T_open_act_i`;仅 1 个有效班次时直接取该值;权重全缺时退回算术平均 | 按实际开班时长加权,不按计划产量 | `daily-report/api_daily_report_generator_modified.py:2209-2227` | +| 周/月报 03 表班次 OEE | 同「有效 OEE」,并按 `T_open_act` 加权到早班/晚班 | 与日报逐班值同源 | `weekly-report/api_weekly_report_generator.py:755-776`、`:1479-1513` | +| 周/月报周期总表 OEE | `Σ(OEE_src_i × T_open_act_i) ÷ Σ T_open_act_i` | **代码现状**:周期总表取 `ATTRIBUTE36` 加权,不是 `OEE_eff`;与 TEEP 的 OEE 因子可能分叉,见文末「口径提醒」 | `weekly-report/api_weekly_report_generator.py:1153-1173` | +| 周期公式 OEE | `Σ M_good ÷ Σ M_theory × 100`,只统计 OEE 输入齐全的班次 | 用于和主 OEE 对账 | `daily-report/api_daily_report_generator_modified.py:2229-2244` | +| OEE 差值 | `OEE_gap = OEE_day − OEE_formula` | 主 OEE 与公式 OEE 的偏差 | `daily-report/api_daily_report_generator_modified.py:2245-2249` | +| OEE 是否算全 | `有效班次数 ÷ 应有班次数`;`有效 == 应有` 时 `oee_calculation_complete = 1` | 有效班次 = OEE 输入齐全的已开班班次 | `daily-report/api_daily_report_generator_modified.py:2278-2280` | + +## 3. TEEP、设备计划利用率与两个「U」 + +| 指标 | 公式 | 口径/说明 | 实现 | +|---|---|---|---| +| 模架装载率(发泡线) | `U_load = Σ((M_plan + M_trial) × N_cycle) ÷ Σ(M_line × N_cycle) × 100`
单班圈数约掉:`(M_plan + M_trial) ÷ M_line × 100` | 一圈都没转时退回各班等权:`Σ(M_plan + M_trial) ÷ ΣM_line × 100`;缺 `M_line` 或缺开班模架数 → `None`,**不**退回不乘装载率 | `common/teep.py:127-151` | +| 模架装载率(装配线) | 恒 `100` | 装配线没有模架项;乘进利用率不改变结果 | `common/teep.py:141-143` | +| 时间利用率 | `U_time = Σ T_run_plan ÷ T_nat × 100` | `T_nat` 按粒度取;未开班也能算 | `common/teep.py:262-264` | +| 设备计划利用率(发泡线) | `U_plan = U_load × (Σ T_run_plan ÷ T_nat)`
即 `U_load × U_time ÷ 100` | 百分数乘比例,结果仍是百分数;只看计划值,停产日会被日历时长如实拉低 | `common/teep.py:169-187` | +| 设备计划利用率(装配线) | `U_plan = Σ T_run_plan ÷ T_nat × 100` | `U_load = 100`,公式退化为时间利用率 | 同上 | +| 单班 TEEP | `TEEP = OEE_eff × U_plan ÷ 100` | 两个因子都是百分数,乘完再 ÷100;任一因子缺失 → `None`,不写 0 | `common/teep.py:230-251` | +| 周期 TEEP(周/月/组级) | `TEEP = Σ(OEE_eff_i × T_open_act_i) ÷ Σ T_open_act_i × U_plan ÷ 100` | 周期 OEE 因子按实际开班时长加权;利用率分子分母跨线各自相加,`T_nat` 按产线条数放大 | `common/teep.py:212-241`、`weekly-report/api_weekly_report_generator.py:1184-1187` | +| 单班 TEEP 等价式(核对用) | `TEEP = M_good ÷ (T_nat ÷ V_plan × M_line) × 100` | 即「合格模架 ÷ 满载产能(日历时间口径)」;可用于反查 TEEP 是否被装载率放大 | `common/teep.py:19-26`、`docs/报告生成技能说明.md:245` | +| TEEP 构成 | `U_plan = U_load × U_time`;`TEEP = OEE_eff × U_plan` | 报表「班次信息概览」按这个构成展示,便于评审复核 | `common/teep.py:254-295` | + +## 4. 其他相关指标 + +| 指标 | 公式 | 口径/说明 | 实现 | +|---|---|---|---| +| 计划完成率(班次) | `C = ATTRIBUTE07` 直读 | **禁止**用 `实际产量 ÷ 计划产量` 重算;装配线 009 实际为 0 时按 007 报工 ÷ 009 计划重算 | `daily-report/api_daily_report_generator_modified.py:1586-1592` | +| 计划完成率(日/周期) | `Σ(C_i × plan_output_i) ÷ Σ plan_output_i`;权重全缺时算术平均 | 按计划产量加权,不是简单平均 | `daily-report/api_daily_report_generator_modified.py:2196-2201` | +| 时间开动率 / 设备利用率 | `T_run_act ÷ T_open_act × 100` | 两者在项目里同公式、不同展示名 | `daily-report/api_daily_report_generator_modified.py:2282-2284` | +| RTY(日/周期) | `Σ rty_good ÷ Σ rty_input × 100` | 来自 `MES_RTY` 零件行相加;班次展示值直读 `ATTRIBUTE39` | `daily-report/api_daily_report_generator_modified.py:2283-2286`、`:1683` | +| 质量合格率 | `Σ rty_good ÷ Σ rty_input × 100` | 与 RTY 同式,用于 OEE 质量维度展示 | 同上 `:2283` | +| 停机率 | `T_stop_act ÷ T_open_act × 100` | 分母是实际开班时长,含计划内停机 | `daily-report/api_daily_report_generator_modified.py:1663`、`:2287` | +| 计划外停机率 | `T_stop_unp ÷ T_open_act × 100` | 只算计划外停机 | 同上 `:1664`、`:2288` | +| 报废率 | `scrap ÷ actual_output × 100` | 日报/看板口径;AI 简报另直读 `ATTRIBUTE38` | 同上 `:1685`、`:2290` | +| 件均不良 | `(rty_rework + rty_scrap) ÷ rty_input` | 不含报废以外的其他不良来源 | 同上 `:2291` | +| 放空率 | `ATTRIBUTE34` 直读 | 源表有值直接用 | `daily-report/api_daily_report_generator_modified.py:1618` | +| 每圈放空工位 | `unplanned_empty_mold ÷ N_cycle` | 用于判断「每圈固定工位放空 / 偶尔放空 / 范围性放空」 | `common/empty_mold.py:188-190`、`:147-157` | +| 有停机圈占比 | `(N_cycle − N_no_stop) ÷ N_cycle × 100` | 判断放空与停机相关性 | `common/empty_mold.py:138-144` | +| 放空损失机时 | `unplanned_empty_mold ÷ (M_plan + M_trial) × V_plan` | 单位秒;漏掉 `÷每圈模架数` 会把损失放大到不可能的量级 | `common/empty_mold.py:115-122` | +| 放空损失产量 | `unplanned_empty_mold × (actual_output ÷ M_actual)` | 用本班自身「每模位产出」折算,不引入全局常数 | `common/empty_mold.py:125-135` | +| 计划外放空重算(仅兜底) | `round(T_run_act ÷ actual_speed, 1) × (M_plan + M_trial) − M_actual` | **只在 `ATTRIBUTE33` 缺失时使用**,且负数截为 `None`/0;源表有值以源表为准 | `daily-report/api_daily_report_generator_modified.py:1635-1661` | +| 人员出勤率 | `attendance_actual ÷ attendance_std × 100` | 标准人数优先 `ATTRIBUTE49`,为空时按 `(M_plan + M_trial)` 查配置分档;实际出勤 0 视为「没采到」 | `ai-trend-api/make_metrics.py:240-261` | +| 缺勤率 / 人员缺口 | `100 − 出勤率`;`attendance_actual − attendance_std` | 超员时为负,不截断 | `ai-trend-api/make_metrics.py:262-269` | +| 计划缺口(KPI 口径) | `plan_output − actual_output` | 与排产单口径 `CPSL − ACTIVE_QTY` **不是同一个数**,不能互相折算 | `docs/日报口径与术语表.md:11-12`、`:27-34` | +| 缺口桥 | `gap = 计划外停机损失 + 计划内停机超时损失 + 开班时长不足损失 + 速度与空转损失 (+ 未解释)` | 各桶按计划节拍 `rate = plan_output ÷ T_run_plan` 折成产量当量;代数闭合 | `common/completion_bridge.py:1-39`、`:360-408` | +| 模架产出缺口 | `max(M_theory − M_actual, 0)` | 理论模架与实际生产模架的差 | `daily-report/api_daily_report_generator_modified.py:1499-1502` | +| 人均效率 / TEEP 波动 | 无数据源 | 业务方已确认上游不提供,报表按「缺失即不成立」处理,不推算 | `docs/大屏字段与009指标对照表.md:165` | + +## 口径提醒(核对时最容易出错的地方) + +1. **发泡线与装配线是两套 OEE 口径**:发泡线用模架/理论模架公式,算不出就是 `NA`,绝不回退 `ATTRIBUTE36`;装配线才用 `ATTRIBUTE36`。两类线的 OEE 不应横向比较。 +2. **TEEP 的 OEE 因子是「有效 OEE」**,按实际开班时长加权;周/月报周期总表的 OEE 在代码里取的是 `ATTRIBUTE36` 加权。两者在发泡线公式值与源值不一致时会分叉,核对 TEEP 时请以 `common/teep.py` 的 `source_oee_wavg` 为准。 +3. **模架装载率 > 100% 在物理上不可能**:实测 1050/3950/3951-NONPIP 曾出现 170~190%,原因是产线主数据 `BEAT` 与实际每圈模架数不符。装载率每超一倍,设备计划利用率与 TEEP 会被等比例放大。 +4. **缺失不写 0**:TEEP、设备计划利用率、OEE 的任一因子缺失都返回 `None`(报表显示 `NA`/灰圈),不是 0。未开班班次只影响 OEE 展示,设备计划利用率仍然照算——「排了班却没开」正是它要暴露的。 +5. **告警阈值**:设备计划利用率 > 95% 提示核查;时间利用率 > 95% 提示计划内停机是否漏扣;TEEP > 100% 判物理不成立;OEE > 200% 判采集异常(`common/teep.py:269-286`、`docs/日报口径与术语表.md:45-48`)。 +6. **配置可覆盖**:日报 `scope=daily`、周月报 `scope=period` 的比率类指标可由「计算口径」页的公式覆盖;表内为代码内置默认值。改口径后历史报表重渲会跟着变,但已入库的旧数据不会自动回溯。 diff --git a/backend/tests/fixtures/recipes/p2_baseline.json b/backend/tests/fixtures/recipes/p2_baseline.json index 7f5c9b58..33c7d302 100644 --- a/backend/tests/fixtures/recipes/p2_baseline.json +++ b/backend/tests/fixtures/recipes/p2_baseline.json @@ -13,6 +13,7 @@ }, "engine_ver": "recipe/1", "ok": true, + "sqlite_writer_version": 3053004, "db_sha256": "986c9128e60b653852330cc496215e634cf37086947c6c9e709cc54737e7f313", "table_hashes": { "日客流": "36452a4d651d017f2d7865e63b1543f6afb939f43c2c61c3672a67df421a4b2b", diff --git a/backend/tests/sqlite_baseline.py b/backend/tests/sqlite_baseline.py new file mode 100644 index 00000000..859989f3 --- /dev/null +++ b/backend/tests/sqlite_baseline.py @@ -0,0 +1,17 @@ +"""Compare SQLite golden bytes across runtimes without changing stored artifacts.""" +from __future__ import annotations + +import hashlib +import sqlite3 +from pathlib import Path + + +def baseline_file_hash(path: Path, *, writer_version: int) -> str: + data = bytearray(path.read_bytes()) + assert data[:16] == b"SQLite format 3\x00" + major, minor, patch = sqlite3.sqlite_version_info + # Offset 96 is the SQLite library version that last wrote the file. All other + # bytes, including page layout, schema and data, remain part of the assertion. + assert int.from_bytes(data[96:100], "big") == major * 1_000_000 + minor * 1_000 + patch + data[96:100] = writer_version.to_bytes(4, "big") + return hashlib.sha256(data).hexdigest() diff --git a/backend/tests/test_authoring_recovery.py b/backend/tests/test_authoring_recovery.py new file mode 100644 index 00000000..74cbf7f8 --- /dev/null +++ b/backend/tests/test_authoring_recovery.py @@ -0,0 +1,256 @@ +"""Fault injection at graph acceptance and safe missing-definition continuation.""" +import asyncio + +from sqlalchemy import select + +from app.api import ledger +from app.db.base import SessionLocal +from app.db.models import CaliberDraft, DataSource, Workflow +from app.ledger import authoring +from tests.test_ledger_docdraft_api import client, datasource, documents, extract, fake_model, fake_trial, plan_reply +from tests.test_definition_batches import imported, reply + + +async def test_graph_acceptance_is_atomic_and_resume_does_not_duplicate( + client, documents, datasource, fake_trial, monkeypatch): + doc = await documents('atomic-acceptance') + source = await datasource() + model = fake_model(monkeypatch, plan_reply(f'{doc}-2')) + capture = authoring._capture_field_baseline + + async def interrupted(**kwargs): + async with SessionLocal() as session: + saved = await session.get(CaliberDraft, kwargs['draft'].id) + assert saved.workflow_id == kwargs['workflow_id'] + assert saved.payload['finalization_pending'] + raise asyncio.CancelledError + + monkeypatch.setattr(authoring, '_capture_field_baseline', interrupted) + made = await extract(client, doc, source) + draft_id = made['draft_id'] + failed = (await client.get(f'/api/ledger/drafts/{draft_id}')).json() + assert failed['status'] == 'failed' and failed['job']['retryable'] + workflow_id = failed['workflow_id'] + monkeypatch.setattr(authoring, '_capture_field_baseline', capture) + resumed = await client.post(f'/api/ledger/drafts/{draft_id}/retry') + assert resumed.status_code == 200, resumed.text + ready = (await client.get(f'/api/ledger/drafts/{draft_id}')).json() + assert ready['workflow_id'] == workflow_id and ready['status'] == 'ready' + assert not ready['finalization_pending'] and len(model.calls) == 1 + async with SessionLocal() as session: + workflows = (await session.execute(select(Workflow))).scalars().all() + assert len([w for w in workflows if (w.origin or {}).get('draft_id') == draft_id]) == 1 + + +async def test_disabled_source_rejected_in_old_new_retry_and_worker( + client, documents, datasource, monkeypatch): + draft_id, source, chunk, items = await imported(client, datasource, monkeypatch, count=1) + async with SessionLocal() as session: + row = await session.get(DataSource, source) + row.enabled = False + await session.commit() + assert (await client.post(f'/api/ledger/drafts/{draft_id}/implement', + json={'datasource_id': source})).status_code == 422 + doc = await documents('disabled-source') + assert (await client.post('/api/ledger/drafts/extract', + json={'documents': [doc], 'datasource_id': source})).status_code == 422 + async with SessionLocal() as session: + row = await session.get(CaliberDraft, draft_id) + row.status, row.datasource_id = 'drafting', source + await session.commit() + await ledger._extract_in_background(draft_id) + result = (await client.get(f'/api/ledger/drafts/{draft_id}')).json() + assert result['status'] == 'failed' and '停用' in result['error'] + assert (await client.post(f'/api/ledger/drafts/{draft_id}/retry')).status_code == 409 + + +async def test_continue_is_idempotent_missing_only_and_preserves_original( + client, datasource, fake_trial, monkeypatch): + draft_id, source, chunk, items = await imported(client, datasource, monkeypatch, count=2) + await client.post(f'/api/ledger/drafts/{draft_id}/implement', json={'datasource_id': source}) + fake_model(monkeypatch, reply(items[:1], chunk), '{"groups": [], "open_questions": ["缺少字段"]}') + await ledger._extract_in_background(draft_id) + before = (await client.get(f'/api/ledger/drafts/{draft_id}')).json() + assert before['metric_count'] == 1 and before['status'] == 'questions_open' + async with SessionLocal() as session: + workflow = await session.get(Workflow, before['workflow_id']) + original_graph, version = workflow.graph, workflow.version + body = {'expected_revision': version, 'guidance': '补充:计数使用 orders'} + result = await client.post(f'/api/ledger/drafts/{draft_id}/continue', json=body) + assert result.status_code == 202, result.text + candidate = result.json()['draft_id'] + assert candidate != draft_id and result.json()['merge_required'] + assert (await client.post(f'/api/ledger/drafts/{draft_id}/continue', json=body)).json()['draft_id'] == candidate + fake_model(monkeypatch, reply(items[1:], chunk)) + await ledger._extract_in_background(candidate) + ready = (await client.get(f'/api/ledger/drafts/{candidate}')).json() + assert ready['metric_count'] == 1 + assert ready['metrics'][0]['source_definition_id'] == items[1]['id'] + async with SessionLocal() as session: + workflow = await session.get(Workflow, before['workflow_id']) + assert workflow.graph == original_graph and workflow.version == version + assert (await client.post(f'/api/ledger/drafts/{draft_id}/continue', + json={**body, 'expected_revision': version + 1})).status_code == 409 + + merged = await client.post(f'/api/ledger/drafts/{candidate}/merge', json={ + 'expected_revision': version, 'candidate_revision': ready['version']}) + assert merged.status_code == 200, merged.text + combined = merged.json() + assert combined['metric_count'] == 2 and combined['workflow_id'] == before['workflow_id'] + assert combined['version'] == version + 1 + async with SessionLocal() as session: + workflow = await session.get(Workflow, before['workflow_id']) + old_card = next(n for n in original_graph['nodes'] if n['type'] == 'metrics') + new_card = next(n for n in workflow.graph['nodes'] if n['type'] == 'metrics') + assert new_card['data']['config']['metrics'][0] == old_card['data']['config']['metrics'][0] + assert len({m['id'] for m in new_card['data']['config']['metrics']}) == 2 + replay = await client.post(f'/api/ledger/drafts/{candidate}/merge', json={ + 'expected_revision': version, 'candidate_revision': ready['version']}) + assert replay.status_code == 200 and replay.json()['replayed'] + assert replay.json()['version'] == combined['version'] + + +async def test_all_blocked_can_continue_and_merge_without_sharing_candidate_graph( + client, datasource, fake_trial, monkeypatch): + draft_id, source, chunk, items = await imported(client, datasource, monkeypatch, count=1) + await client.post(f'/api/ledger/drafts/{draft_id}/implement', json={'datasource_id': source}) + empty = '{"groups": [], "open_questions": ["缺少字段"]}' + fake_model(monkeypatch, empty, empty) + await ledger._extract_in_background(draft_id) + before = (await client.get(f'/api/ledger/drafts/{draft_id}')).json() + assert before['status'] == 'questions_open' and not before['workflow_id'] + result = await client.post(f'/api/ledger/drafts/{draft_id}/continue', json={ + 'expected_revision': 0, 'guidance': '已补齐字段'}) + assert result.status_code == 202, result.text + candidate = result.json()['draft_id'] + fake_model(monkeypatch, reply(items, chunk)) + await ledger._extract_in_background(candidate) + ready = (await client.get(f'/api/ledger/drafts/{candidate}')).json() + conflict = await client.post(f'/api/ledger/drafts/{candidate}/merge', json={ + 'expected_revision': 0, 'candidate_revision': 99}) + assert conflict.status_code == 409 + merged = await client.post(f'/api/ledger/drafts/{candidate}/merge', json={ + 'expected_revision': 0, 'candidate_revision': ready['version']}) + assert merged.status_code == 200, merged.text + assert merged.json()['metric_count'] == 1 + assert merged.json()['workflow_id'] != ready['workflow_id'] + + +def test_merge_keeps_human_edits_and_rejects_conflicting_parameters(): + import copy + import pytest + from app.ledger.continuation import append_candidate + from app.ledger.draftplan import build_graph + + def graph(key): + return build_graph([{'sql': 'SELECT 1 AS n', 'entry_fields': [ + {'name': 'date', 'type': 'string', 'default': 'yesterday'}], 'metrics': [ + {'id': 'm1', 'name': key, 'source_definition_id': key, 'expression': "cell(nodes.fetch, 0, 'n')"}]}], name=key) + original, candidate = graph('old'), graph('new') + old_card = next(n for n in original['nodes'] if n['type'] == 'metrics') + old_card['data']['config']['metrics'][0]['expression'] = "cell(nodes.fetch, 0, 'n') * 100" + original_copy = copy.deepcopy(original) + merged = append_candidate(original, candidate, prefix='candidate', allowed_definitions={'new'}) + merged_card = next(n for n in merged['nodes'] if n['type'] == 'metrics') + assert original == original_copy + assert merged_card['data']['config']['metrics'][0] == old_card['data']['config']['metrics'][0] + assert 'nodes.candidate_fetch' in merged_card['data']['config']['metrics'][1]['expression'] + next(n for n in candidate['nodes'] if n['type'] == 'input')['data']['config']['fields'][0]['default'] = 'today' + with pytest.raises(ValueError, match='参数'): + append_candidate(original, candidate, prefix='candidate', allowed_definitions={'new'}) + + +async def test_continue_supports_subset_datasource_switch_multi_metric_split_and_field_baseline( + client, datasource, fake_trial, monkeypatch): + import json + import uuid + + draft_id, source1, chunk, items = await imported(client, datasource, monkeypatch, count=3) + source2 = f'ds-{uuid.uuid4().hex[:10]}' + async with SessionLocal() as session: + session.add(DataSource( + id=source2, name=f'mes_{uuid.uuid4().hex[:6]}', kind='sqlite', + description='制造库', schema_cache={'tables': {'orders': {'columns': [ + {'name': 'amount', 'type': 'NUMERIC'}]}}, 'total': 1}, + )) + await session.commit() + + await client.post(f'/api/ledger/drafts/{draft_id}/implement', json={'datasource_id': source1}) + empty = '{"groups": [], "open_questions": ["请确认取数表名"]}' + fake_model(monkeypatch, empty) + await ledger._extract_in_background(draft_id) + initial = (await client.get(f'/api/ledger/drafts/{draft_id}')).json() + assert initial['status'] == 'questions_open' and not initial['workflow_id'] + + bad_subset = await client.post(f'/api/ledger/drafts/{draft_id}/continue', json={ + 'expected_revision': 0, 'guidance': '先验证第一项', 'selected': ['unknown-def-id'], + }) + assert bad_subset.status_code == 422 + + cont1 = await client.post(f'/api/ledger/drafts/{draft_id}/continue', json={ + 'expected_revision': 0, 'guidance': '切换到制造库并先处理第一项', + 'selected': [items[0]['id']], 'datasource_id': source2, + }) + assert cont1.status_code == 202, cont1.text + cand1 = cont1.json()['draft_id'] + + split_reply = json.dumps({ + 'sql': 'SELECT COUNT(*) AS a, SUM(amount) AS b FROM orders', + 'entry_fields': [], + 'metrics': [ + { + 'id': 'm1', 'name': f"{items[0]['name']}(计划口径)", 'source_definition_id': items[0]['id'], + 'expression': "cell(nodes.fetch, 0, 'a')", + 'definition': {'text': '计划口径', 'citations': [{'chunk': chunk, 'quote': '按日汇总计数,不取明细第一行'}]}, + }, + { + 'id': 'm2', 'name': f"{items[0]['name']}(产线口径)", 'source_definition_id': items[0]['id'], + 'expression': "cell(nodes.fetch, 0, 'b')", + 'definition': {'text': '产线口径', 'citations': [{'chunk': chunk, 'quote': '按日汇总计数,不取明细第一行'}]}, + }, + ], + 'uncovered': [], 'open_questions': [], + }, ensure_ascii=False) + fake_model(monkeypatch, split_reply) + await ledger._extract_in_background(cand1) + cand1_view = (await client.get(f'/api/ledger/drafts/{cand1}')).json() + assert cand1_view['metric_count'] == 2 + + merged1 = await client.post(f'/api/ledger/drafts/{cand1}/merge', json={ + 'expected_revision': 0, 'candidate_revision': cand1_view['version'], + }) + assert merged1.status_code == 200, merged1.text + after1 = merged1.json() + assert after1['datasource_id'] == source2 + assert after1['metric_count'] == 2 + assert after1['implementation_progress']['completed'] == 1 + assert after1['implementation_progress']['blocked'] == 2 + assert after1['field_baseline']['status'] == 'known' + assert after1['field_baseline']['coverage']['status'] == 'complete' + + # Switching datasource is rejected once the draft owns a workflow graph. + switch_rejected = await client.post(f'/api/ledger/drafts/{draft_id}/continue', json={ + 'expected_revision': after1['version'], 'guidance': '尝试再换库', 'datasource_id': source1, + }) + assert switch_rejected.status_code == 409 + + # Second subset continuation advances field_baseline cleanly onto existing workflow. + cont2 = await client.post(f'/api/ledger/drafts/{draft_id}/continue', json={ + 'expected_revision': after1['version'], 'guidance': '继续处理第二项', 'selected': [items[1]['id']], + }) + assert cont2.status_code == 202, cont2.text + cand2 = cont2.json()['draft_id'] + fake_model(monkeypatch, reply([items[1]], chunk)) + await ledger._extract_in_background(cand2) + cand2_view = (await client.get(f'/api/ledger/drafts/{cand2}')).json() + merged2 = await client.post(f'/api/ledger/drafts/{cand2}/merge', json={ + 'expected_revision': after1['version'], 'candidate_revision': cand2_view['version'], + }) + assert merged2.status_code == 200, merged2.text + after2 = merged2.json() + assert after2['metric_count'] == 3 + assert after2['version'] == after1['version'] + 1 + assert after2['implementation_progress']['completed'] == 2 + assert after2['field_baseline']['status'] == 'known' + assert after2['field_baseline']['coverage']['status'] == 'complete' + assert len(after2['field_baseline']['metric_ids']) == 3 diff --git a/backend/tests/test_definition_batches.py b/backend/tests/test_definition_batches.py new file mode 100644 index 00000000..c0191dc8 --- /dev/null +++ b/backend/tests/test_definition_batches.py @@ -0,0 +1,249 @@ +"""Default-all implementation, bounded internal batches and durable resume without user batching.""" +import json + +import pytest + +from app.api import ledger +from app.db.base import SessionLocal +from app.db.models import Chunk, Document +from tests.test_ledger_docdraft_api import client, datasource, fake_model, fake_trial # noqa: F401 + + +async def imported(client, datasource, monkeypatch, count=31, conflicts=False): + source = await datasource() + text = '# 合成计数定义\n| 指标 | 公式 | 口径/说明 |\n|---|---|---|\n' + text += ''.join(f'| 计数指标{i} | COUNT(*) | 按日汇总计数,不取明细第一行 |\n' for i in range(count)) + if conflicts: + text += '| 无来源指标 | 无数据源 | 不推算 |\n| 同名指标 | COUNT(*) | 范围甲 |\n| 同名指标 | SUM(x) | 范围乙 |\n' + async with SessionLocal() as session: + doc = Document(title='自动处理定义', content=text, status='ready', chunk_count=1) + session.add(doc) + await session.flush() + chunk = Chunk(document_id=doc.id, ordinal=1, content=text) + session.add(chunk) + await session.commit() + doc_id, chunk_id = doc.id, chunk.id + response = await client.post('/api/ledger/drafts/definitions', json={'documents': [doc_id]}) + assert response.status_code == 201, response.text + draft_id = response.json()['draft_id'] + view = (await client.get(f'/api/ledger/drafts/{draft_id}')).json() + + async def enqueue(_): + pass + monkeypatch.setattr(ledger.draft_jobs, 'start', enqueue) + return draft_id, source, chunk_id, view['semantic_inventory']['items'] + + +def reply(items, chunk): + return json.dumps({'sql': 'SELECT COUNT(*) AS count FROM orders', 'entry_fields': [], + 'metrics': [{'id': f'm{index}', 'name': item['name'], 'source_definition_id': item['id'], + 'expression': "cell(nodes.fetch, 0, 'count')", + 'definition': {'text': item['name'], 'citations': [{'chunk': chunk, 'quote': '按日汇总计数,不取明细第一行'}]}} + for index, item in enumerate(items)], 'uncovered': [], 'open_questions': []}, ensure_ascii=False) + + +@pytest.mark.parametrize('count', [31, 67]) +async def test_all_definitions_run_past_four_rounds_without_manual_batches(client, datasource, monkeypatch, fake_trial, count): + draft_id, source, chunk, items = await imported(client, datasource, monkeypatch, count=count, conflicts=True) + response = await client.post(f'/api/ledger/drafts/{draft_id}/implement', json={'datasource_id': source}) + assert response.status_code == 202, response.text + eligible = items[:count] + model = fake_model(monkeypatch, *[reply(eligible[i:i+6], chunk) for i in range(0, count, 6)]) + await ledger._extract_in_background(draft_id) + made = (await client.get(f'/api/ledger/drafts/{draft_id}')).json() + assert made['metric_count'] == count, made + assert len(model.calls) == (count + 5) // 6 and made['progress']['round'] == len(model.calls) + assert made['implementation_mode'] == 'all' and len(made['definition_issues']) == 3 + assert made['implementation_progress'] == {'total': count, 'completed': count, 'pending': 0, 'blocked': 0, 'active_names': []} + assert made['status'] == 'questions_open' and not made['uncovered'] + assert [p['definition_ids'] for p in made['prompts'] if p['purpose'] == 'plan'] == [ + [i['id'] for i in eligible[n:n+6]] for n in range(0, count, 6)] + + +async def test_partial_selection_is_optional_and_can_exceed_six(client, datasource, monkeypatch, fake_trial): + draft_id, source, chunk, items = await imported(client, datasource, monkeypatch, count=15) + chosen = items[2:15] + response = await client.post(f'/api/ledger/drafts/{draft_id}/implement', json={ + 'datasource_id': source, 'selected': [i['id'] for i in chosen]}) + assert response.status_code == 202, response.text + model = fake_model(monkeypatch, *[reply(chosen[n:n+6], chunk) for n in range(0, 13, 6)]) + await ledger._extract_in_background(draft_id) + made = (await client.get(f'/api/ledger/drafts/{draft_id}')).json() + assert made['status'] == 'ready' and made['metric_count'] == 13 and len(model.calls) == 3 + assert {m['source_definition_id'] for m in made['metrics']} == {i['id'] for i in chosen} + + +async def test_unresolved_batch_does_not_stop_other_definitions(client, datasource, monkeypatch, fake_trial): + draft_id, source, chunk, items = await imported(client, datasource, monkeypatch, count=13) + await client.post(f'/api/ledger/drafts/{draft_id}/implement', json={'datasource_id': source}) + empty = json.dumps({'groups': [], 'open_questions': ['首组缺少字段映射'], 'uncovered': []}) + model = fake_model(monkeypatch, empty, reply(items[6:12], chunk), + reply(items[12:] + items[:5], chunk), empty) + await ledger._extract_in_background(draft_id) + made = (await client.get(f'/api/ledger/drafts/{draft_id}')).json() + assert made['status'] == 'questions_open' and len(model.calls) == 4 + assert made['metric_count'] == 12 and made['uncovered'] == [items[5]['name']] + assert made['implementation_progress']['blocked'] == 1 and made['implementation_progress']['pending'] == 0 + assert any(items[5]['name'] in q for q in made['open_questions']) + + +async def test_failed_job_resumes_only_unfinished_definitions(client, datasource, monkeypatch, fake_trial): + from app.ledger import draftplan, docdraft + + draft_id, source, chunk, items = await imported(client, datasource, monkeypatch, count=13) + await client.post(f'/api/ledger/drafts/{draft_id}/implement', json={'datasource_id': source}) + fake_model(monkeypatch, reply(items[:6], chunk)) + original = draftplan.plan_groups + calls = 0 + + async def fail_second(*args, **kwargs): + nonlocal calls + calls += 1 + if calls == 2: + raise docdraft.DraftUnavailable('模型连接中断') + return await original(*args, **kwargs) + + monkeypatch.setattr(draftplan, 'plan_groups', fail_second) + await ledger._extract_in_background(draft_id) + failed = (await client.get(f'/api/ledger/drafts/{draft_id}')).json() + assert failed['status'] == 'failed' and failed['metric_count'] == 6 + async with SessionLocal() as session: + await ledger.draft_jobs.finish(session, draft_id, 'failed', '模型连接中断') + response = await client.post(f'/api/ledger/drafts/{draft_id}/retry') + assert response.status_code == 200, response.text + monkeypatch.setattr(draftplan, 'plan_groups', original) + model = fake_model(monkeypatch, reply(items[6:12], chunk), reply(items[12:], chunk)) + await ledger._extract_in_background(draft_id) + made = (await client.get(f'/api/ledger/drafts/{draft_id}')).json() + assert made['status'] == 'ready' and made['metric_count'] == 13 and len(model.calls) == 2 + assert made['implementation_progress']['completed'] == 13 + assert len({m['source_definition_id'] for m in made['metrics']}) == 13 + + +async def test_oee_overlapping_chunks_and_wide_prefixed_catalog_keep_visible_citations_and_diagnostics( + client, monkeypatch, fake_trial): + from pathlib import Path + import uuid + from app.db.models import DataSource + from app.ledger import authoring + + fixture_path = Path(__file__).parent / 'fixtures' / 'oee_metric_definitions.md' + text = fixture_path.read_text(encoding='utf-8') + lines = text.splitlines(keepends=True) + # Build 16 overlapping chunks similar to production KB chunking (~16k chars total). + overlapping_chunks: list[str] = [] + stride = max(1, len(lines) // 16) + for idx in range(16): + start = max(0, idx * stride - 6) + end = len(lines) if idx == 15 else min(len(lines), (idx + 2) * stride + 6) + overlapping_chunks.append(''.join(lines[start:end])) + + tables: dict[str, dict] = {} + for idx in range(42): + tables[f'vi_ike_sap_wide_{idx:02d}'] = { + 'columns': [{'name': f'sap_col_{c:02d}', 'type': 'VARCHAR2'} for c in range(60)] + } + tables['vi_ike_mes_bhgsj'] = {'columns': [{'name': 'scrap_qty', 'type': 'NUMBER'}]} + tables['vi_ike_mes_jzhmjxs'] = {'columns': [{'name': 'mjxs', 'type': 'NUMBER'}]} + tables['vi_ike_iot_dn_productionline'] = {'columns': [{'name': 'beat', 'type': 'NUMBER'}, {'name': 'line_code', 'type': 'VARCHAR2'}]} + tables['vi_ike_iot_kpi_info'] = { + 'columns': [{'name': 'line_code', 'type': 'VARCHAR2'}, {'name': 'shift_date', 'type': 'DATE'}] + + [{'name': f'attribute{c:02d}', 'type': 'NUMBER'} for c in range(1, 56)] + } + + source_id = f'ds-{uuid.uuid4().hex[:10]}' + async with SessionLocal() as session: + src = DataSource( + id=source_id, name=f'bi_{uuid.uuid4().hex[:6]}', kind='oracle', + description='制造数据仓库', schema_cache={'schema': 'OWODS', 'tables': tables, 'total': len(tables)}, + ) + session.add(src) + doc = Document(title='指标公式总表.md', content=text, status='ready', chunk_count=len(overlapping_chunks)) + session.add(doc) + await session.flush() + for ordinal, content in enumerate(overlapping_chunks, start=1): + session.add(Chunk(document_id=doc.id, ordinal=ordinal, content=content)) + await session.commit() + doc_id = doc.id + focused = authoring._focus_tables(tables, text, source=src) + assert 'vi_ike_mes_bhgsj' in focused + assert 'vi_ike_iot_dn_productionline' in focused + assert 'vi_ike_iot_kpi_info' in focused + + async def enqueue(_): + pass + monkeypatch.setattr(ledger.draft_jobs, 'start', enqueue) + + created = await client.post('/api/ledger/drafts/definitions', json={'documents': [doc_id]}) + assert created.status_code == 201, created.text + draft_id = created.json()['draft_id'] + view = (await client.get(f'/api/ledger/drafts/{draft_id}')).json() + metric_items = [i for i in view['semantic_inventory']['items'] if i['kind'] == 'metric' and i['status'] != 'unavailable'] + assert len(metric_items) >= 40 + target = next(i for i in metric_items if i['name'] == '装配线 OEE') + chosen = [target, *[i for i in metric_items if i['id'] != target['id']][:11]] + + # First batch implements `target` even if model omits citations (verified provenance fallback attaches it), + # while remaining 11 items stay unresolved with a model question (testing >10 unresolved non-spam behavior). + first_reply = json.dumps({ + 'groups': [{ + 'sql': 'SELECT AVG(attribute36) AS oee_assembly FROM OWODS.vi_ike_iot_kpi_info', + 'entry_fields': [], + 'metrics': [{ + 'id': 'm1', 'name': target['name'], 'source_definition_id': target['id'], + 'expression': "cell(nodes.fetch, 0, 'oee_assembly')", + 'definition': {'text': '装配线直接取源系统 OEE', 'citations': []}, + }], + }], + 'open_questions': ['其余指标需确认产线过滤编码与早晚班归属字段'], + 'uncovered': [], + }, ensure_ascii=False) + later_reply = json.dumps({ + 'groups': [], + 'open_questions': ['其余指标需确认产线过滤编码与早晚班归属字段'], + 'uncovered': [], + }, ensure_ascii=False) + fake_model(monkeypatch, first_reply, later_reply) + + started = await client.post(f'/api/ledger/drafts/{draft_id}/implement', json={ + 'datasource_id': source_id, 'selected': [i['id'] for i in chosen], + }) + assert started.status_code == 202, started.text + await ledger._extract_in_background(draft_id) + + made = (await client.get(f'/api/ledger/drafts/{draft_id}')).json() + assert made['metric_count'] == 1 + assert made['metrics'][0]['source_definition_id'] == target['id'] + assert made['metrics'][0]['definition']['citations'] + for trace in made['prompts']: + assert trace['visible_chars'] and all(chars > 0 for chars in trace['visible_chars'].values()) + assert target['id'] not in made['definition_diagnostics'] + unresolved_ids = {i['id'] for i in chosen if i['id'] != target['id']} + assert set(made['definition_diagnostics']) == unresolved_ids + assert not any('尚未生成可用实现,请补充字段映射或确认口径' in q for q in made['open_questions']) + + +async def test_pure_validation_failure_without_model_questions_marks_draft_failed( + client, datasource, monkeypatch, fake_trial): + draft_id, source, _chunk, items = await imported(client, datasource, monkeypatch, count=2) + bad = json.dumps({ + 'groups': [{ + 'sql': 'SELECT COUNT(*) AS n FROM nonexistent_table', + 'entry_fields': [], + 'metrics': [{ + 'id': 'm1', 'name': items[0]['name'], 'source_definition_id': items[0]['id'], + 'expression': "cell(nodes.fetch, 0, 'n')", + 'definition': {'text': items[0]['name'], 'citations': []}, + }], + }], + 'open_questions': [], + 'uncovered': [], + }, ensure_ascii=False) + fake_model(monkeypatch, bad) + await client.post(f'/api/ledger/drafts/{draft_id}/implement', json={'datasource_id': source}) + await ledger._extract_in_background(draft_id) + made = (await client.get(f'/api/ledger/drafts/{draft_id}')).json() + assert made['status'] == 'failed' + assert 'nonexistent_table' in (made['error'] or '') + assert not made['open_questions'] + assert made['definition_diagnostics'][items[0]['id']]['kind'] == 'validation_failed' diff --git a/backend/tests/test_ledger_nlchange.py b/backend/tests/test_ledger_nlchange.py index 970217f0..03cd6884 100644 --- a/backend/tests/test_ledger_nlchange.py +++ b/backend/tests/test_ledger_nlchange.py @@ -729,17 +729,16 @@ async def test_a_single_month_and_a_single_day_map_to_the_half_open_window(clien assert second.input == {"start": "2026-09-02", "end": "2026-09-03"} -async def test_a_name_shared_across_cards_is_ambiguous_and_falls_back(client, model_unconfigured): - """同名跨卡不自动:交回 None,answer_ops 照旧走旧路径(模型没配好时就照旧 400)。""" +async def test_a_name_shared_across_cards_clarifies_without_model(client, model_unconfigured): + """同名跨卡不自动:已知口径需澄清,不重新规划取数。""" shared = "同名歧义甲" for _ in range(2): asset = await _auto_asset(metric_name=shared) await publish(client, asset) async with SessionLocal() as session: - assert await nlchange.auto_route(session, question=f"{shared} 2026年9月是多少") is None - with pytest.raises(HTTPException) as caught: - await _ops(f"{shared} 2026年9月是多少") - assert caught.value.status_code == 400 + assert (await nlchange.auto_route(session, question=f"{shared} 2026年9月是多少")).mode == nlchange.ROUTE_CLARIFY + ops = await _ops(f"{shared} 2026年9月是多少") + assert len(ops) == 1 and ops[0]["op"] == "reply" async def test_a_missing_non_date_parameter_falls_back(client, model_unconfigured): @@ -1004,7 +1003,7 @@ async def test_natural_questions_use_declared_defaults_when_no_date_is_given(cli assert first.input == {"start": "2026-09-01", "end": "2026-10-01"}, "没给日期就用声明的缺省值" # 相对说法(上个月)不是「销售额」:即使字段有缺省值也不直达 async with SessionLocal() as session: - assert await nlchange.auto_route(session, question=f"上个月{AUTO_NAME}是多少") is None + assert (await nlchange.auto_route(session, question=f"上个月{AUTO_NAME}是多少")).mode == nlchange.ROUTE_CLARIFY async def test_a_hit_that_produces_nothing_is_reported_as_a_failure_for_a_user(client, engine_up, monkeypatch): diff --git a/backend/tests/test_recipe_contract.py b/backend/tests/test_recipe_contract.py index fd729e91..9619285a 100644 --- a/backend/tests/test_recipe_contract.py +++ b/backend/tests/test_recipe_contract.py @@ -12,6 +12,8 @@ from dataclasses import MISSING, asdict from pathlib import Path import pytest + +from tests.sqlite_baseline import baseline_file_hash from pydantic import ValidationError from app.data import recipe_parsers as P @@ -652,7 +654,7 @@ def test_p2_baseline_reproduces(): ex = recipe_engine.execute(T.Recipe.model_validate(FLOW), raw, name, xlsx_scan.scan(raw), str(db)) table_versions.settle_journal(db) assert ex.ok - assert raw_store.sha256_file(db) == P2_BASELINE["db_sha256"] + assert baseline_file_hash(db, writer_version=P2_BASELINE["sqlite_writer_version"]) == P2_BASELINE["db_sha256"] assert ex.table_hashes == P2_BASELINE["table_hashes"] assert {t.name: t.rows for t in ex.tables} == P2_BASELINE["rows"] ids = P2_BASELINE["ids"] diff --git a/backend/tests/test_recipe_engine_p3.py b/backend/tests/test_recipe_engine_p3.py index f827a6bf..03e664f0 100644 --- a/backend/tests/test_recipe_engine_p3.py +++ b/backend/tests/test_recipe_engine_p3.py @@ -20,6 +20,8 @@ from typing import Any import pytest +from tests.sqlite_baseline import baseline_file_hash + from app.data import raw_store, recipe_engine, table_versions, xlsx_cells, xlsx_scan from app.data.names import canon from app.data.recipe_types import FIX_KINDS, Extraction, Recipe @@ -88,7 +90,7 @@ def test_d00_build_is_byte_identical_to_phase_2(tmp_path): ex = run((raw, name), FLOW, str(db)) table_versions.settle_journal(db) assert ex.ok and not ex.problems - assert raw_store.sha256_file(db) == BASELINE["db_sha256"] + assert baseline_file_hash(db, writer_version=BASELINE["sqlite_writer_version"]) == BASELINE["db_sha256"] assert ex.table_hashes == BASELINE["table_hashes"] assert {t.name: t.rows for t in ex.tables} == BASELINE["rows"] assert ex.rows_excluded == [] @@ -103,7 +105,7 @@ def test_rules_whose_anchors_do_not_appear_change_nothing(tmp_path): ex = run((raw, name), data, str(db)) table_versions.settle_journal(db) assert ex.ok and not ex.problems - assert raw_store.sha256_file(db) == BASELINE["db_sha256"] + assert baseline_file_hash(db, writer_version=BASELINE["sqlite_writer_version"]) == BASELINE["db_sha256"] assert ex.table_hashes == BASELINE["table_hashes"] assert ex.rows_excluded == [] and ex.ignored_columns == {} and "ignored" not in ex.ledger[0].roles diff --git a/backend/tests/test_semantic_pipeline.py b/backend/tests/test_semantic_pipeline.py new file mode 100644 index 00000000..723f1c05 --- /dev/null +++ b/backend/tests/test_semantic_pipeline.py @@ -0,0 +1,360 @@ +"""Document definitions -> reviewed semantic bindings -> one deterministic, authorized query. + +Synthetic shifts only. No production connection or live model. Independent expected OEE: +(50 * 8 + 20 * 4) / 12 = 40, not the arithmetic mean (35). +""" +from __future__ import annotations + +import copy +import sqlite3 +from datetime import datetime, timezone +from pathlib import Path + +import pytest +from sqlalchemy import select + +from app.db.base import SessionLocal +from app.db.models import Chunk, Document, Run +from app.ledger import nlchange, semantic +from app.ledger.semantic_import import inventory, implementation_guidance +from app.ledger.semantic_sql import scalar_problem +from tests import ledger_samples as S +from tests.publication_samples import publish +from tests.test_ledger_nlchange import ( # noqa: F401 - reuse isolated runtime and tracked cleanup + client, cleanup_own_rows, engine_up, _auto_asset, _ops, _grant_app, _acting, _no_internals, runs_total, +) +from tests.test_ledger_docdraft_api import datasource, fake_trial, fake_model # noqa: F401 + +SOURCE = Path(__file__).parent / 'fixtures' / 'oee_metric_definitions.md' +TIME = {"start": "start", "end": "end", "timezone": "Asia/Shanghai", "basis": "production_day", + "periods": ["day"], "description": "按测试表已确认的生产日归属,夜班不按结束日期另算"} +SEMANTIC = {"version": 1, "enabled": True, "aliases": ["OEE"], "grain": "工厂、生产日单值", + "aggregation": "按有效班次的实际开班时长加权;无有效值返回空;权重全缺时等权", + "time": TIME, "entities": [{"field": "plant", "label": "工厂", "members": [ + {"value": "1050", "label": "测试工厂甲", "aliases": ["甲厂"]}, + {"value": "3950", "label": "测试工厂乙", "aliases": []}]}]} +SQL = """SELECT CASE WHEN COUNT(oee) = 1 THEN MAX(oee) + WHEN COUNT(CASE WHEN oee IS NOT NULL THEN hours END) = 0 THEN AVG(oee) + ELSE SUM(oee * hours) / NULLIF(SUM(CASE WHEN oee IS NOT NULL THEN hours END), 0) END AS oee + FROM shifts WHERE plant = '{{ input.plant }}' + AND production_day >= '{{ input.start }}' AND production_day < '{{ input.end }}'""" + + +def oee_graph(*, model=None, sql=SQL): + return S.chain( + S.node("start", "input", fields=[{"name": name, "type": "string", "required": True} + for name in ("start", "end", "plant")]), + S.node("fetch", "tool", tool=S.tool(), args={"sql": sql}), + S.node("card", "metrics", caliber="合成 OEE 样例", metrics=[{ + "id": "oee", "name": "日 OEE(合成验证)", "unit": "%", "decimals": 2, + "expression": "cell(nodes.fetch, 0, 'oee')", "contract": {"shape": "scalar"}, + "semantic": copy.deepcopy(model if model is not None else SEMANTIC)}]), + S.node("report", "report", instructions="禁止在固定问数中运行本报告节点"), + S.node("out", "output", fields=[{"name": "答案", "value": "{{ nodes.report.text }}"}])) + + +@pytest.fixture +def no_model(monkeypatch): + from app.api import copilot + from app.providers import mock_model + + def boom(*a, **kw): + raise AssertionError("semantic fast path must not plan, discover schemas or write reports with a model") + + monkeypatch.setattr(copilot, "_op_model", boom) + monkeypatch.setattr(mock_model.MockChatModel, "_decide", boom) + monkeypatch.setattr(semantic, "now", lambda: datetime(2026, 11, 10, 2, tzinfo=timezone.utc)) + + +@pytest.fixture +async def oee_asset(client, engine_up, tmp_path, no_model): + with sqlite3.connect(tmp_path / "querycore.db") as db: + db.execute("CREATE TABLE shifts (plant TEXT, production_day TEXT, oee REAL, hours REAL)") + db.executemany("INSERT INTO shifts VALUES (?, ?, ?, ?)", [ + ("1050", "2026-11-09", 50, 8), ("1050", "2026-11-09", 20, 4), + ("1050", "2026-11-09", None, 12), # missing OEE must not dilute valid weights + ("3950", "2026-11-09", 90, 8), ("1050", "2026-11-08", 99, 8), + ("1050", "2026-11-07", 60, None), ("1050", "2026-11-07", 20, None), + ]) + asset = await _auto_asset(graph=oee_graph()) + await publish(client, asset) + return asset + + +def test_actual_document_is_inventory_not_37_invented_scalar_metrics(): + source = SOURCE.read_text() + catalog = inventory([{"document_id": "doc", "content": source}]) + metrics = [i for i in catalog["items"] if i["kind"] == "metric"] + symbols = [i for i in catalog["items"] if i["kind"] == "input"] + assert len(symbols) == 19 + assert len(metrics) == 48 + assert len({i["id"] for i in catalog["items"]}) == len(catalog["items"]) + by_name = {i["name"]: i for i in metrics} + assert by_name["人均效率 / TEEP 波动"]["status"] == "unavailable" + assert "不回退" in by_name["有效 OEE(报表统一口径)"]["policy"] + assert "0.5" in by_name["报废模架数"]["policy"] + for item in catalog["items"]: + a, b = item["source"]["span"] + assert source[a:b].strip() == item["source"]["quote"] + guidance = implementation_guidance(catalog, [by_name["日 OEE(全天聚合)"]["id"]]) + assert "权重全缺" in guidance and "配置可覆盖" in guidance and "T_open_act" in guidance + assert '"name": "放空率"' not in guidance + + +@pytest.mark.parametrize("sql", [ + "SELECT oee FROM shifts", "SELECT 86400 AS seconds FROM productionlines", + "SELECT plant, SUM(oee) FROM shifts GROUP BY plant", "SELECT oee FROM shifts LIMIT 1", + "SELECT oee FROM shifts WHERE ROWNUM = 1", "SELECT SUM(oee) OVER () FROM shifts", + "SELECT SUM(oee) FROM shifts HAVING SUM(oee)>0", "SELECT SUM(oee), plant FROM shifts", + "SELECT SUM(oee) FROM shifts UNION ALL SELECT SUM(oee) FROM shifts", +]) +def test_detail_grouped_truncated_or_window_queries_cannot_be_scalar(sql): + assert scalar_problem(sql) + + +@pytest.mark.parametrize("sql", [SQL, "SELECT COUNT(*) FROM shifts", "SELECT 86400 FROM dual", + "WITH x AS (SELECT SUM(hours) AS hours FROM shifts) SELECT hours * 2 AS value FROM x", + "SELECT SUM(good) / NULLIF(SUM(total),0) * 100 FROM shifts", +]) +def test_single_row_aggregate_or_scalar_projection_is_accepted(sql): + assert scalar_problem(sql) is None + + +@pytest.mark.parametrize("sql", ["SELECT 86400 AS seconds", "SELECT 86400 AS seconds FROM DUAL"]) +def test_documented_constants_do_not_require_a_business_table_scan(sql): + from app.ledger.draftplan import normalize_plan + + text = "自然时长按单天 86400 秒计算,不能按产线表复制多行" + plan, problems = normalize_plan({"sql": sql, "metrics": [{ + "id": "natural", "name": "自然时长", "expression": "cell(nodes.fetch, 0, 'seconds')", + "definition": {"citations": [{"chunk": "source", "quote": text}]}, + }]}, {"source": {"content": text, "document_id": "doc"}}, allowed_tables={"productionlines"}, strict_scalar=True) + assert plan["sql"] == sql and not problems + + +def test_yesterday_uses_business_timezone_and_month_requires_its_own_permission(): + binding = semantic.TimeBinding.model_validate(TIME) + # UTC still Nov 9; in Shanghai it is already Nov 10. + at = datetime(2026, 11, 9, 17, tzinfo=timezone.utc) + assert semantic.interval("昨天 1050 OEE", binding, at=at)[0] == ("2026-11-09", "2026-11-10") + assert semantic.interval("2026年11月 OEE", binding)[0] is None + binding.periods = ["month"] + assert semantic.interval("2026年11月 OEE", binding)[0] == ("2026-11-01", "2026-12-01") + assert semantic.interval("昨天和前天 OEE", binding)[0] is None + assert not semantic.mention("10501 的 OEE", "1050") + assert not semantic.mention("昨天1050的OEEfoo", "OEE") + + +def test_authorized_agent_sees_the_same_published_entity_time_and_aggregation_rules(): + from app.look.calibers import metric_options, metric_briefs + + options = metric_options(oee_graph(), "card", workflow_id="wf", version=1) + text = metric_briefs(options.selectable) + assert "1050" in text and "工厂" in text and "Asia/Shanghai" in text + assert "实际开班时长加权" in text and "production_day" in text + + +async def test_ordinary_question_reaches_querycore_with_one_query_and_no_llm(oee_asset, monkeypatch): + import time + from app.ledger import querycore + + executed = [] + original = querycore.execute + + async def measured(*args, **kwargs): + outcome = await original(*args, **kwargs) + executed.append(outcome) + return outcome + + monkeypatch.setattr(querycore, "execute", measured) + started = time.perf_counter() + [final] = await _ops("给我昨天 1050 的 OEE") + elapsed = time.perf_counter() - started + assert final["op"] == "final", final + result = final["execution"] + assert result["via"] == "querycore" and result["ok"] + assert result["metrics"][0]["value"] == 40 + assert len(executed) == 1 and len(executed[0].queries) == 1 + assert executed[0].measurement.counts["executed_sql"] == 1 + assert executed[0].measurement.counts["llm_calls"] == 0 + async with SessionLocal() as session: + run = await session.get(Run, final["run_id"]) + assert run.input == {"start": "2026-11-09", "end": "2026-11-10", "plant": "1050"} + assert run.origin["service"] == "querycore" + print(f"\nSynthetic OEE ask-data: {elapsed:.3f}s; 1 query; 0 LLM; expected=40 actual=40") + + +@pytest.mark.parametrize("question", ["昨天 OEE", "昨天 10501 的 OEE", "昨天 1050 和 3950 的 OEE", + "昨天 1050 的 OEE 排除返工", "2026年11月 1050 的 OEE", "昨天前天 1050 的 OEE", "昨天产线1050的OEE"]) +async def test_uncertain_semantic_request_clarifies_without_exploration(oee_asset, question): + before = await runs_total() + [reply] = await _ops(question) + assert reply["op"] == "reply" and reply["route"]["mode"] == "clarify", reply + assert await runs_total() == before + + +async def test_missing_day_stays_missing_and_weight_fallback_is_explicit(oee_asset): + [empty] = await _ops("2026-11-06 1050 的 OEE") + assert empty["execution"]["metrics"][0]["value"] is None + [fallback] = await _ops("2026-11-07 1050 的 OEE") + assert fallback["execution"]["metrics"][0]["value"] == 40 + + +async def test_user_authorization_and_tool_parent_run_are_preserved(oee_asset): + before = await runs_total() + with _acting("semantic-denied"): + [refused] = await _ops("昨天 1050 的 OEE") + assert refused["op"] == "error" and refused["code"] == nlchange.ROUTE_NOT_PERMITTED + assert await runs_total() == before + await _grant_app(oee_asset["workflow_id"], 1, who="semantic-user") + with _acting("semantic-user"): + ops = await _ops("昨天 1050 的 OEE") + assert ops[0]["op"] == "final", ops + _no_internals(ops, workflow_id=oee_asset["workflow_id"]) + async with SessionLocal() as session: + parent = await session.get(Run, ops[0]["run_id"]) + rows = (await session.execute(select(Run).where(Run.workflow_id == oee_asset["workflow_id"]))).scalars().all() + assert "caliber_query" in str(parent.graph) + assert any((r.origin or {}).get("parent_run_id") == parent.id for r in rows) + + +async def test_overlapping_oee_aliases_require_selection(client, oee_asset): + other = oee_graph() + other["nodes"][2]["data"]["config"]["metrics"][0]["name"] = "源系统 OEE(合成验证)" + asset = await _auto_asset(graph=other) + await publish(client, asset) + before = await runs_total() + [reply] = await _ops("昨天 1050 的 OEE") + assert reply["op"] == "reply" and "多种" in reply["text"] + assert await runs_total() == before + + +@pytest.mark.parametrize("case", ["disabled", "no_time", "unbound_entity"]) +async def test_semantic_metadata_cannot_discard_a_date_or_unbound_filter(client, no_model, case): + model = copy.deepcopy(SEMANTIC) + if case == "disabled": + model["enabled"] = False + elif case == "no_time": + model["time"] = None + else: + model["entities"][0]["field"] = "unused_plant" + asset = await _auto_asset(graph=oee_graph(model=model)) + await publish(client, asset) + before = await runs_total() + [reply] = await _ops("2026-11-09 1050 的 OEE") + assert reply["op"] == "reply" and reply["route"]["mode"] == "clarify" + assert await runs_total() == before + + +async def test_semantic_edit_uses_existing_revision_and_invalidates_definition(client): + asset = await _auto_asset(graph=oee_graph()) + before = copy.deepcopy(SEMANTIC) + before["enabled"] = False + body = {"operations": [{"op": "set", "metric_id": "oee", "path": "/semantic", "value": before}], + "expected_revision": 1, "request_id": "semantic-edit", "target": {"card_id": "card"}} + response = await client.patch(f"/api/ledger/drafts/{asset['draft_id']}", json=body) + assert response.status_code == 200, response.text + data = response.json() + assert data["revision"] == 2 and data["change"]["definition_hash"]["changed"] + assert not data["metrics"][0]["semantic"]["enabled"] + response = await client.patch(f"/api/ledger/drafts/{asset['draft_id']}", json={**body, "request_id": "stale"}) + assert response.status_code == 409 + + +async def test_definitions_import_needs_no_database_or_model_and_preserves_unavailable(client, no_model): + async with SessionLocal() as session: + doc = Document(title="公式总表", content=SOURCE.read_text()) + session.add(doc) + await session.commit() + doc_id = doc.id + response = await client.post("/api/ledger/drafts/definitions", json={"documents": [doc_id]}) + assert response.status_code == 201, response.text + draft_id = response.json()["draft_id"] + response = await client.get(f"/api/ledger/drafts/{draft_id}") + assert response.status_code == 200, response.text + draft = response.json() + assert draft["workflow_id"] is None and draft["metrics"] == [] and draft["status"] == "path_pending" + assert len(draft["semantic_inventory"]["items"]) == 67 + assert draft["definition_count"] == 48 and draft["metric_count"] == 0 + unavailable = next(i for i in draft["semantic_inventory"]["items"] if i["status"] == "unavailable") + response = await client.post(f"/api/ledger/drafts/{draft_id}/implement", + json={"selected": [unavailable["id"]], "datasource_id": "missing"}) + assert response.status_code == 422 and "无数据源" in response.text + + +@pytest.mark.parametrize("first_query", ["SELECT COUNT(*) AS count FROM orders", "SELECT amount AS count FROM orders"]) +async def test_selected_definition_survives_implementation_and_rejects_duplicate_job(client, datasource, fake_trial, monkeypatch, first_query): + import json + from app.api import ledger + + source_id = await datasource() + content = """# 测试口径 +| 指标 | 公式 | 口径/说明 | +|---|---|---| +| 订单数 | COUNT(*) | 按日汇总,不把明细第一行当周期 | +| 人均效率 | 无数据源 | 不推算 | +""" + async with SessionLocal() as session: + doc = Document(title="测试定义表", content=content, chunk_count=1) + session.add(doc) + await session.flush() + chunk = Chunk(document_id=doc.id, ordinal=1, content=content) + session.add(chunk) + await session.commit() + doc_id, chunk_id = doc.id, chunk.id + response = await client.post("/api/ledger/drafts/definitions", json={"documents": [doc_id]}) + draft_id = response.json()["draft_id"] + draft = (await client.get(f"/api/ledger/drafts/{draft_id}")).json() + item = draft["semantic_inventory"]["items"][0] + calls = [] + + async def queued(draft_id): + calls.append(draft_id) + + monkeypatch.setattr(ledger.draft_jobs, "start", queued) + body = {"selected": [item["id"]], "datasource_id": source_id} + assert (await client.post(f"/api/ledger/drafts/{draft_id}/implement", json=body)).status_code == 202 + assert (await client.post(f"/api/ledger/drafts/{draft_id}/implement", json=body)).status_code == 409 + assert calls == [draft_id] + generated = { + "sql": "SELECT COUNT(*) AS count FROM orders", "entry_fields": [], "metrics": [{ + "id": "count", "name": "订单数", "expression": "cell(nodes.fetch, 0, 'count')", + "definition": {"text": "订单数", "citations": [{"chunk": chunk_id, "quote": "按日汇总,不把明细第一行当周期"}]}, + "semantic": {"enabled": True, "grain": "日单值", "aggregation": "计数", "aliases": ["订单数量"]}, + }], "open_questions": [], "uncovered": [], + } + if first_query != generated["sql"]: + # One document row may become separately named executable quantities (e.g. M_plan_slots / M_line_slots). + generated["metrics"][0]["name"] = "订单计数量" + generated["metrics"][0]["source_definition_id"] = item["id"] + model = fake_model(monkeypatch, json.dumps({**generated, "sql": first_query}, ensure_ascii=False), + json.dumps(generated, ensure_ascii=False)) + await ledger._extract_in_background(draft_id) + made = (await client.get(f"/api/ledger/drafts/{draft_id}")).json() + assert made["status"] == "ready", (made.get("error"), made.get("dropped"), made.get("open_questions")) + assert made["workflow_id"] and made["metric_count"] == 1 + assert made["selected_definition_ids"] == [item["id"]] + assert made["semantic_inventory"] == draft["semantic_inventory"] + assert made["metrics"][0]["semantic"]["enabled"] is False, "model cannot enable unreviewed query semantics" + assert made["metrics"][0]["source_definition_id"] == item["id"] and not made["uncovered"] + assert made["sql"] == "SELECT COUNT(*) AS count FROM orders" + assert len(model.calls) == (1 if first_query == made["sql"] else 2) + + +async def test_changed_source_cannot_be_implemented_as_the_old_definition(client): + async with SessionLocal() as session: + doc = Document(title="会更新的总表", content=SOURCE.read_text()) + session.add(doc) + await session.commit() + doc_id = doc.id + res = await client.post("/api/ledger/drafts/definitions", json={"documents": [doc_id]}) + draft_id = res.json()["draft_id"] + draft = (await client.get(f"/api/ledger/drafts/{draft_id}")).json() + item = next(i for i in draft["semantic_inventory"]["items"] if i["kind"] == "metric") + async with SessionLocal() as session: + doc = await session.get(Document, doc_id) + doc.content += "\n修改公式" + await session.commit() + res = await client.post(f"/api/ledger/drafts/{draft_id}/implement", + json={"selected": [item["id"]], "datasource_id": "unused"}) + assert res.status_code == 409 and "原文已更新" in res.text diff --git a/deploy/agentlab.env b/deploy/agentlab.env index ce4a3978..97d6335d 100644 --- a/deploy/agentlab.env +++ b/deploy/agentlab.env @@ -119,15 +119,15 @@ ENABLE_BWRAP=0 # 行首的 # 再 ./deploy.sh install。 # # 库的镜像(小版本固定;离线包里带着同一个标签的 postgres-image.tar) -POSTGRES_IMAGE=postgres:16.15 +# POSTGRES_IMAGE=postgres:16.15 # 库的容器名,缺省 -postgres -POSTGRES_CONTAINER=agentlab-postgres +# POSTGRES_CONTAINER=agentlab-postgres # 两个容器共用的 docker 网络,缺省 -net(库的端口不对外,只在这个网络里可见) -DOCKER_NETWORK=agentlab-net +# DOCKER_NETWORK=agentlab-net # 库的数据放在这个 docker 数据卷里,缺省 -pgdata(卸载不删) -POSTGRES_VOLUME=agentlab-pgdata +# POSTGRES_VOLUME=agentlab-pgdata # upgrade 前自动备份库(SQLite 拷到 backups/sqlite/,内置 PostgreSQL 用 pg_dump 存到 backups/postgres/),只留最近几份 -BACKUP_KEEP=5 +# BACKUP_KEEP=5 # ---- 外接 PostgreSQL 与多进程(可选)---- # 库放在你自己的 PostgreSQL 上(库要先建好):去掉 AGENTLAB_DATABASE_URL 行首的 # 填好地址,再 ./deploy.sh install。 diff --git a/frontend/src/api/caliberLedger.ts b/frontend/src/api/caliberLedger.ts index b24b1057..082c1680 100644 --- a/frontend/src/api/caliberLedger.ts +++ b/frontend/src/api/caliberLedger.ts @@ -87,6 +87,23 @@ export const caliberLedger = { extractDraft: (body: { documents: string[]; collection?: string; datasource_id?: string; guidance?: string; fixed_filters?: string }) => post<{ draft_id: string; status: string }>('/ledger/drafts/extract', body), + importDefinitions: (body: { documents: string[]; datasource_id?: string; guidance?: string; fixed_filters?: string }) => + post<{ draft_id: string; status: string }>('/ledger/drafts/definitions', body), + implementDefinitions: (draftId: string, body: { selected?: string[]; datasource_id: string }) => + post<{ draft_id: string; status: string }>(`${draftAt(draftId)}/implement`, body), + continueDefinitions: (draftId: string, body: { + expected_revision: number; guidance: string; selected?: string[]; datasource_id?: string + }) => + post<{ draft_id: string; status: string; original_draft_id: string; merge_required: boolean }>(`${draftAt(draftId)}/continue`, body), + mergeContinuation: (draftId: string, body: { expected_revision: number; candidate_revision: number }) => + post(`${draftAt(draftId)}/merge`, body), + migration: (draftId: string) => get<{ + revision: number; metrics_total: number; metrics_needing_work: number; can_rebuild: boolean; note: string + metrics: { card_id: string; metric_id: string; name: string; query_count: number; guidance: string + findings: { code: string; message: string }[] }[] + }>(`${draftAt(draftId)}/migration`), + prepareMigration: (draftId: string, expected_revision: number) => + post<{ draft_id: string; status: string }>(`${draftAt(draftId)}/migration`, { expected_revision }), /** * 改标题、整体换掉指标、或把答清的问题消掉(resolved 是问题原文)。服务端据此重算状态:还有问题 → questions_open, * 否则 ready;已经接上或驳回的不改状态。 diff --git a/frontend/src/pages/ledger/CaliberDrafts.tsx b/frontend/src/pages/ledger/CaliberDrafts.tsx index 3d92d592..a17ddd68 100644 --- a/frontend/src/pages/ledger/CaliberDrafts.tsx +++ b/frontend/src/pages/ledger/CaliberDrafts.tsx @@ -4,7 +4,7 @@ * 主路径一次到位:选文档 → 模型读文档、对照数据目录产出定义 + 取数 + 入口字段 + 指标表达式 → 服务端校验后落成草稿图 * → 后台自动只读试运行 → 这里展示「定义 + 数 + 原文引用」→ 人去画布审改并发布。发布才需要人签字。 * - 发起:选知识库文档和**必选的数据源**(取数 SQL 只对照它的表和列写),可加一段起草要求;页面轮询详情直到状态离开 - * drafting(最多十五分钟,指标多时服务端会分几轮续跑,进度写在 payload.progress); + * drafting(持续有进度时继续轮询,连续十五分钟无进展才暂停轮询;后台任务独立运行); * - 试运行:草稿顶部显示这次自动试运行的状态和每个指标的值,running 时每两秒取一次,最多等一分钟;必填入口没有值时停在 * needs_input,在入口字段里填了再点「试运行」;失败照实写原因; * - 审读:指标默认收成一行(名字、单位、值和未决问题数),展开看单位、粒度、指标表达式、定义和逐字核对过的引用原文 @@ -49,6 +49,9 @@ import { formatDateTime, formatTime } from '../../lib/format' import { LEDGER_UI } from '../../lib/termsLedger' import { useAtLeast } from '../../store/auth' import { TrialResult, TrialValidation } from './TrialResult' +import { ContinuationPanel, DefinitionInventory, QuerySemanticEditor } from './SemanticPanel' +import { OntologyPanel } from './OntologyPanel' +import { MigrationPanel } from './MigrationPanel' import type { DataSourceName, KbDocument, Run } from '../../types' import type { CaliberDraft, CaliberDraftStage, CaliberDraftStatus, CaliberDraftSummary, DraftAlignment, DraftBinding, DraftChangeReceipt, @@ -517,6 +520,7 @@ export function CaliberDrafts() { /** 这次自动试运行的结果;running 时单独轮询,和草稿抽取的轮询互不相干 */ const [trial, setTrial] = useState(null) + const trialRequestSeq = useRef(0) const [trialPollId, setTrialPollId] = useState(null) const [trialTimedOut, setTrialTimedOut] = useState(false) const [trialError, setTrialError] = useState(null) @@ -602,6 +606,7 @@ export function CaliberDrafts() { const [expectSource, setExpectSource] = useState('human_confirmed') /** 验证记录先摆最近几条,其余收起来 */ const [recordsShown, setRecordsShown] = useState(RECORDS_SHOWN) + const [droppedExpanded, setDroppedExpanded] = useState(false) const canReview = useAtLeast('builder') const loadList = useCallback(() => { @@ -706,6 +711,7 @@ export function CaliberDrafts() { setExpectValues({}) setExpectSource('human_confirmed') setRecordsShown(RECORDS_SHOWN) + setDroppedExpanded(false) setImportRuns(null) setImportsError(null) setImportId(null) @@ -768,28 +774,43 @@ export function CaliberDrafts() { useEffect(() => { if (!pollingId) return - const until = Date.now() + DRAFT_POLL_LIMIT_MS + let until = Date.now() + DRAFT_POLL_LIMIT_MS + let lastProgress = '' let alive = true - const timer = setInterval(() => { + let settled = false + const seq = openSeq.current + let timer: ReturnType + const poll = () => { api.caliberLedger.draft(pollingId).then((draft) => { - if (!alive) return - setDetail((cur) => (cur && cur.id === draft.id ? draft : cur)) + if (!alive || seq !== openSeq.current) return + const currentProgress = JSON.stringify([draft.progress, draft.implementation_progress]) + if (currentProgress !== lastProgress) { + lastProgress = currentProgress + until = Date.now() + DRAFT_POLL_LIMIT_MS + } + setDetail((cur) => (cur && cur.id === draft.id && (cur.revision ?? 0) <= (draft.revision ?? 0) ? draft : cur)) if (draft.status !== 'drafting') { + settled = true setPollingId(null) setPollTimedOut(false) loadList() } else if (Date.now() > until) { + settled = true setPollingId(null) setPollTimedOut(true) loadList() } }, (e) => { - if (!alive) return + if (!alive || seq !== openSeq.current) return + settled = true setPollingId(null) setDetailError(e) + }).finally(() => { + if (alive && !settled && seq === openSeq.current) timer = setTimeout(poll, DRAFT_POLL_MS) }) - }, DRAFT_POLL_MS) - return () => { alive = false; clearInterval(timer) } + } + timer = setTimeout(poll, DRAFT_POLL_MS) + return () => { alive = false; clearTimeout(timer) } }, [pollingId, loadList]) /** 草稿有了试运行就取一次结果;还在 running 就交给下面的轮询 */ @@ -801,14 +822,15 @@ export function CaliberDrafts() { setTrial((cur) => (cur?.run_id === run && detail?.trial?.status !== 'stale' ? cur : { run_id: run, status: detail?.trial?.status ?? 'running', metrics: [], error: detail?.trial?.error ?? '', })) + const request = ++trialRequestSeq.current api.caliberLedger.draftTrial(id).then((out) => { - if (!alive) return + if (!alive || request !== trialRequestSeq.current) return setTrial(out) setTrialError(null) setTrialTimedOut(false) if (out.status === 'running') setTrialPollId(id) }, (e) => { - if (!alive) return + if (!alive || request !== trialRequestSeq.current) return setTrialError(e) setTrialPollId(null) }) @@ -820,35 +842,44 @@ export function CaliberDrafts() { if (!trialPollId) return const until = Date.now() + 60_000 let alive = true - const timer = setInterval(() => { + let settled = false + const seq = openSeq.current + let timer: ReturnType + const poll = () => { + const request = ++trialRequestSeq.current api.caliberLedger.draftTrial(trialPollId).then((out) => { - if (!alive) return + if (!alive || seq !== openSeq.current || request !== trialRequestSeq.current) return setTrial(out) setTrialError(null) if (out.status === 'running') { if (Date.now() > until) { + settled = true setTrialPollId(null) setTrialTimedOut(true) } return } + settled = true setTrialPollId(null) setTrialTimedOut(false) loadList() // 试运行到终态时服务端会把这次跑完的验证记录补记一条:跟着取一次,结论按记录说 loadValidations(trialPollId) - const seq = openSeq.current api.caliberLedger.draft(trialPollId).then((fresh) => { if (seq !== openSeq.current) return - setDetail((cur) => (cur && cur.id === fresh.id ? fresh : cur)) + setDetail((cur) => (cur && cur.id === fresh.id && (cur.revision ?? 0) <= (fresh.revision ?? 0) ? fresh : cur)) }, () => undefined) }, (e) => { - if (!alive) return + if (!alive || seq !== openSeq.current || request !== trialRequestSeq.current) return + settled = true setTrialPollId(null) setTrialError(e) + }).finally(() => { + if (alive && !settled && seq === openSeq.current) timer = setTimeout(poll, 2000) }) - }, 2000) - return () => { alive = false; clearInterval(timer) } + } + timer = setTimeout(poll, 2000) + return () => { alive = false; clearTimeout(timer) } }, [trialPollId, loadList, loadValidations]) /** @@ -902,17 +933,17 @@ export function CaliberDrafts() { return at >= 0 ? at + 1 : null } - const submit = async () => { + const submit = async (definitions = true) => { if (!pickedDocs.length) { toast.warn(D.needDoc); return } setSubmitting(true) try { - const out = await api.caliberLedger.extractDraft({ + const out = await (definitions ? api.caliberLedger.importDefinitions : api.caliberLedger.extractDraft)({ documents: pickedDocs, ...(datasourceId ? { datasource_id: datasourceId } : {}), ...(guidance.trim() ? { guidance: guidance.trim() } : {}), ...(fixedFilters.trim() ? { fixed_filters: fixedFilters.trim() } : {}), }) - toast.ok(D.started) + toast.ok(definitions ? '定义清单已导入,可自动处理全部可实现指标' : D.started) setPickedDocs([]) setGuidance('') setFixedFilters('') @@ -2668,14 +2699,38 @@ export function CaliberDrafts() { } const renderDetail = (draft: CaliberDraft) => { + const viewSeq = openSeq.current const stage = stageOf(draft) const drafting = draft.status === 'drafting' const terminal = stage === 'discarded' - const stageNote = stage === 'review' ? D.stageNote(draft.status) : '' - const questions = draft.open_questions ?? [] - const dropped = draft.dropped ?? [] - const uncovered = draft.uncovered ?? [] const metrics = draft.metrics ?? [] + const hasInventory = !!draft.semantic_inventory + const awaitingImplementation = hasInventory && !draft.workflow_id && metrics.length === 0 + const stageNote = awaitingImplementation && draft.status !== 'path_pending' && !drafting + ? D.stageNoteNeedsGraph + : stage === 'review' ? D.stageNote(draft.status) : '' + const rawQuestions = draft.open_questions ?? [] + const questions = hasInventory + ? rawQuestions.filter((q) => + !q.includes('尚未生成可用实现,请补充字段映射或确认口径') + && !(q.startsWith('另有 ') && q.includes('项指标尚未生成可用实现'))) + : rawQuestions + const rawDropped = draft.dropped ?? [] + const hasSpecificDropped = rawDropped.some((item) => item.name !== 'metrics') + const droppedMap = new Map() + for (const item of rawDropped) { + if (hasSpecificDropped && item.name === 'metrics' && item.reason === '没有一条指标通过校验') continue + const key = `${item.name}\u0000${item.reason}` + const existing = droppedMap.get(key) + if (existing) existing.count += 1 + else droppedMap.set(key, { name: item.name, reason: item.reason, count: 1 }) + } + const dropped = Array.from(droppedMap.values()) + const DROPPED_LIMIT = 8 + const visibleDropped = droppedExpanded || dropped.length <= DROPPED_LIMIT + ? dropped + : dropped.slice(0, DROPPED_LIMIT) + const uncovered = draft.uncovered ?? [] const values = new Map((draft.trial?.status === 'stale' ? [] : trial?.metrics ?? []).map((row) => [row.id, row])) // 第一轮还没落库时服务端写的是空对象:没有轮次就不画,别显示「第 undefined 轮」 const progress = draft.progress && typeof draft.progress.round === 'number' ? draft.progress : null @@ -2700,7 +2755,8 @@ export function CaliberDrafts() { {(drafting || draft.job?.retryable || (draft.status === 'failed' && !draft.workflow_id)) && (
{drafting && } - {!drafting && } + {!drafting && }
)} {jobError != null && } @@ -2713,18 +2769,19 @@ export function CaliberDrafts() { {drafting && pollingId === draft.id && ( {D.drafting} - {progress && ( + {progress && !draft.implementation_progress && ( {D.progress(progress.round, progress.covered, progress.uncovered)} )} - {D.draftingHint} + {draft.implementation_progress + ? '系统会自动处理后续指标并保存进度,离开页面不影响执行。需补充的信息会单独列出。' : D.draftingHint} )} {drafting && pollTimedOut && ( {D.draftingTimeout} - {progress && ( + {progress && !draft.implementation_progress && ( {D.progress(progress.round, progress.covered, progress.uncovered)} @@ -2736,12 +2793,31 @@ export function CaliberDrafts() { )} {draft.catalog_truncated && {D.catalogTruncated}} + { loadList(); if (viewSeq === openSeq.current) void open(id) }} /> + { loadList(); if (viewSeq === openSeq.current) void open(id) }} /> + { loadList(); if (viewSeq === openSeq.current) void open(id) }} /> {renderTrial(draft)} {renderRules(draft)} - {renderValidation(draft)} + {!awaitingImplementation && renderValidation(draft)} {renderImports(draft)} {renderNl(draft)} - {renderEntryFields(draft)} + {!awaitingImplementation && renderEntryFields(draft)} + {metrics.map(metric => { + loadList() + if (viewSeq !== openSeq.current) return + setDetail(cur => cur?.id === fresh.id && (cur.revision ?? 0) <= (fresh.revision ?? 0) ? fresh : cur) + loadValidations(draft.id) + }} />)} + { + loadList() + if (viewSeq !== openSeq.current) return + setDetail(cur => cur?.id === fresh.id && (cur.revision ?? 0) <= (fresh.revision ?? 0) ? fresh : cur) + loadValidations(draft.id) + }} /> {renderSql(draft)}
@@ -2759,6 +2835,10 @@ export function CaliberDrafts() {
{metrics.length ? (
    {metrics.map((metric, index) => renderMetric(metric, index, values))}
+ ) : !drafting && draft.semantic_inventory ? ( +

+ {draft.status === 'path_pending' ? '定义已保留,请在上方选择指标并绑定数据。' : D.inventoryAwaitingHint} +

) : !drafting ? ( } title={D.noMetrics} body={D.noMetricsBody} /> ) : ( @@ -2769,6 +2849,9 @@ export function CaliberDrafts() { {questions.length > 0 && (

{D.questionsOther}

+ {hasInventory && ( +

{D.questionsContinuationHint}

+ )} {questionList(questions, 'top')}
)} @@ -2778,17 +2861,23 @@ export function CaliberDrafts() {

{D.dropped}

{D.droppedHint}

    - {dropped.map((item, i) => ( + {visibleDropped.map((item, i) => (
  • {item.name} - :{item.reason} + :{item.reason}{item.count > 1 ? `(重复 ${item.count} 次)` : ''}
  • ))}
+ {dropped.length > DROPPED_LIMIT && ( + + )} )} - {uncovered.length > 0 && ( + {!hasInventory && uncovered.length > 0 && (

{D.uncovered(uncovered.length)}

{D.uncoveredHint}

@@ -2809,7 +2898,7 @@ export function CaliberDrafts() {
)} - {!drafting && stage === 'review' && draft.status !== 'attached' && ( + {!awaitingImplementation && !drafting && stage === 'review' && draft.status !== 'attached' && (
{D.attach}

{D.attachHint}

@@ -2917,11 +3006,12 @@ export function CaliberDrafts() {
- +
@@ -2986,7 +3078,9 @@ export function CaliberDrafts() { {draft.error && {D.draftingFailed}}
- {D.summary(draft.metric_count, draft.question_count)} + {draft.definition_count != null + ? `${draft.definition_count} 项定义 · ${draft.metric_count} 项已实现 · ${draft.question_count} 个待确认问题` + : D.summary(draft.metric_count, draft.question_count)} {draft.created_at ? ` · ${formatTime(draft.created_at)}` : ''}
diff --git a/frontend/src/pages/ledger/MigrationPanel.tsx b/frontend/src/pages/ledger/MigrationPanel.tsx new file mode 100644 index 00000000..9acb222b --- /dev/null +++ b/frontend/src/pages/ledger/MigrationPanel.tsx @@ -0,0 +1,43 @@ +import { useState } from 'react' +import { api } from '../../api/client' +import { ErrorState, Notice } from '../../components/ui' +import type { CaliberDraft } from '../../typesLedger' + +type Audit = Awaited> + +export function MigrationPanel({ draft, onPrepared }: { draft: CaliberDraft; onPrepared: (id: string) => void }) { + const [audit, setAudit] = useState(null) + const [busy, setBusy] = useState(false) + const [error, setError] = useState(null) + if (!draft.workflow_id) return null + const check = async () => { + setBusy(true); setError(null) + try { setAudit(await api.caliberLedger.migration(draft.id)) } + catch (e) { setError(e) } finally { setBusy(false) } + } + const prepare = async () => { + if (!audit) return + setBusy(true); setError(null) + try { onPrepared((await api.caliberLedger.prepareMigration(draft.id, audit.revision)).draft_id) } + catch (e) { setError(e) } finally { setBusy(false) } + } + return
+
+

旧口径迁移检查

+ +
+

逐项检查明细误作标量、必填参数、对象与日期映射,以及本机数据源是否就绪。

+ {error != null && } + {audit && <> + {audit.note} +

共 {audit.metrics_total} 项,{audit.metrics_needing_work} 项需处理

+ {audit.metrics.map(m =>
+ {m.name} · {m.findings.length ? `${m.findings.length} 项待处理` : '静态检查通过,待试运行'} +
    {m.findings.map((f, i) =>
  • {f.message}
  • )}
+

{m.guidance}

+
)} + {audit.can_rebuild && } +

只缺问数定义时可在下方补齐;取数粒度不符时,从原文分批重建、验证。新草稿验证并发布后,再切换调用口径。

+ } +
+} diff --git a/frontend/src/pages/ledger/SemanticPanel.tsx b/frontend/src/pages/ledger/SemanticPanel.tsx new file mode 100644 index 00000000..30376fa3 --- /dev/null +++ b/frontend/src/pages/ledger/SemanticPanel.tsx @@ -0,0 +1,498 @@ +import { useEffect, useRef, useState } from 'react' +import { api } from '../../api/client' +import { ErrorState, Notice, toast } from '../../components/ui' +import type { DataSourceName } from '../../types' +import type { CaliberDraft, CanonicalEntity, DraftMetric, QuerySemantics } from '../../typesLedger' + +const words = (value: string) => value.split(/[,,\n]/).map(s => s.trim()).filter(Boolean) + +function AliasesInput({ values, onChange, className = 'field' }: { + values: string[]; onChange: (values: string[]) => void; className?: string +}) { + const serialized = values.join(',') + const [text, setText] = useState(serialized) + useEffect(() => setText(serialized), [serialized]) + return setText(e.target.value)} onBlur={() => onChange(words(text))} /> +} + +const isSyntheticUnresolved = (text: string) => + text.includes('尚未生成可用实现,请补充字段映射或确认口径') + || (text.startsWith('另有 ') && text.includes('项指标尚未生成可用实现')) + +function resolveItemDiagnostic( + draft: CaliberDraft, + item: { id: string; name: string }, + isImplemented: boolean, +): { kind: 'needs_info' | 'validation_failed'; reasons: string[]; questions: string[] } | null { + if (isImplemented) return null + const direct = draft.definition_diagnostics?.[item.id] + if (direct && ((direct.reasons?.length ?? 0) > 0 || (direct.questions?.length ?? 0) > 0)) { + const reasons = Array.from(new Set(direct.reasons ?? [])) + const questions = Array.from(new Set(direct.questions ?? [])) + return { + kind: direct.kind === 'validation_failed' || (reasons.length > 0 && questions.length === 0) + ? 'validation_failed' : 'needs_info', + reasons, + questions, + } + } + const dropped = draft.dropped ?? [] + const matchedReasons = Array.from(new Set( + dropped + .filter(d => d.reason && (d.name === item.name || (item.name && d.name.includes(item.name)))) + .map(d => d.reason), + )) + const sharedReasons = matchedReasons.length > 0 ? [] : Array.from(new Set( + dropped + .filter(d => d.reason && (d.name === 'sql' || d.name === 'metrics' || d.name === '重写这一组') + && d.reason !== '没有一条指标通过校验') + .map(d => d.reason), + )).slice(0, 2) + const reasons = matchedReasons.length > 0 ? matchedReasons : sharedReasons + const questions = Array.from(new Set( + (draft.open_questions ?? []).filter(q => !isSyntheticUnresolved(q) && item.name && q.includes(item.name)), + )) + if (!reasons.length && !questions.length) return null + return { + kind: reasons.length > 0 && questions.length === 0 ? 'validation_failed' : 'needs_info', + reasons: reasons.slice(0, 4), + questions: questions.slice(0, 4), + } +} + +export function DefinitionInventory({ draft, sources, onStarted }: { + draft: CaliberDraft; sources: DataSourceName[]; onStarted: (draftId: string) => void +}) { + const [picked, setPicked] = useState(draft.selected_definition_ids ?? []) + const [partial, setPartial] = useState(false) + const [source, setSource] = useState(draft.datasource_id ?? '') + const [filter, setFilter] = useState<'all' | 'unresolved' | 'implemented' | 'excluded'>('all') + const [expanded, setExpanded] = useState(false) + const [busy, setBusy] = useState(false) + const mounted = useRef(false) + useEffect(() => { mounted.current = true; return () => { mounted.current = false } }, []) + const [error, setError] = useState(null) + useEffect(() => { + const available = sources.filter(s => s.enabled !== false) + if (!draft.datasource_id && !source && available.length === 1) setSource(available[0].id) + }, [sources, source, draft.datasource_id]) + const catalog = draft.semantic_inventory + if (!catalog) return null + const canStart = draft.status === 'path_pending' && !draft.workflow_id + const implemented = new Set((draft.metrics ?? []).flatMap(m => [m.name, m.source_definition_id])) + const metrics = catalog.items.filter(i => i.kind === 'metric') + const eligible = metrics.filter(i => i.status !== 'unavailable') + const counts = new Map() + eligible.forEach(i => counts.set(i.name, (counts.get(i.name) ?? 0) + 1)) + const automatic = eligible.filter(i => counts.get(i.name) === 1) + const selectedCount = partial ? picked.length : automatic.length + const sourceAvailable = sources.some(s => s.id === source && s.enabled !== false) + const progress = draft.implementation_progress + const isItemImplemented = (item: typeof metrics[number]) => + implemented.has(item.id) || (counts.get(item.name) === 1 && implemented.has(item.name)) + const isItemExcluded = (item: typeof metrics[number]) => + item.status === 'unavailable' + || (counts.get(item.name) ?? 0) > 1 + || (!!draft.selected_definition_ids?.length && !draft.selected_definition_ids.includes(item.id)) + const implementedItems = metrics.filter(isItemImplemented) + const excludedItems = metrics.filter(i => !isItemImplemented(i) && isItemExcluded(i)) + const unresolvedItems = metrics.filter(i => !isItemImplemented(i) && !isItemExcluded(i)) + const filteredMetrics = canStart || filter === 'all' ? metrics + : filter === 'unresolved' ? unresolvedItems + : filter === 'implemented' ? implementedItems : excludedItems + const COLLAPSE_LIMIT = 12 + const visibleMetrics = canStart || expanded || filteredMetrics.length <= COLLAPSE_LIMIT + ? filteredMetrics + : filteredMetrics.slice(0, COLLAPSE_LIMIT) + const start = async () => { + setBusy(true); setError(null) + try { + await api.caliberLedger.implementDefinitions(draft.id, { ...(partial ? { selected: picked } : {}), datasource_id: source }) + toast.ok('已开始自动处理;进度会持续保存,需要补充的信息会单独列出') + if (mounted.current) onStarted(draft.id) + } catch (e) { setError(e) } finally { setBusy(false) } + } + const noticeTone = !canStart && draft.status !== 'drafting' && progress && progress.completed === 0 && progress.blocked > 0 + ? 'warn' : 'info' + const noticeText = canStart + ? '默认自动处理全部可实现指标,保存进度并继续处理后续项;缺少来源或定义冲突的指标会单独列出。生成后仍需验证口径和计算结果。' + : draft.status === 'drafting' + ? '正在对照数据源分批生成取数实现并自动保存进度;未通过校验或缺少字段映射的指标会记录具体原因。' + : progress && progress.completed === 0 && progress.blocked > 0 + ? `本轮尚未生成可用的取数实现(共 ${progress.blocked} 项待处理)。请查看下方「继续处理未完成指标」中的待补充要点,填写来源表名、字段映射或固定条件后继续生成;也可先勾选少量核心指标先行验证。` + : progress && progress.blocked > 0 + ? `已生成 ${progress.completed} 项指标实现,仍有 ${progress.blocked} 项待补充信息或未通过校验。可在下方「继续处理未完成指标」补充说明后生成独立候选,核对无误再合并回原稿。` + : '所选指标已生成取数实现;请在下方核对试运行结果、完成验证并发布。' + return
+

指标定义清单 · {metrics.length} 项

+ {noticeText} + {progress &&
+

本次处理 {progress.total} 项:已生成 {progress.completed} 项,待处理 {progress.pending} 项,需补充信息 {progress.blocked} 项

+ + {draft.status === 'drafting' && !!progress.active_names.length &&

正在处理:{progress.active_names.join('、')}

} + {!!draft.definition_issues?.length &&

另有 {draft.definition_issues.length} 项因来源缺失或定义冲突暂未自动处理,详见清单。

} +
} + {canStart &&
+ {partial ? `已选择 ${picked.length} 项` : `将自动处理 ${automatic.length} 项`} + +
} + {canStart &&
+ + +
} + {!canStart && metrics.length > 6 && ( +
+ {([ + ['all', `全部(${metrics.length})`, true], + ['unresolved', `待补充或未过(${unresolvedItems.length})`, unresolvedItems.length > 0], + ['implemented', `已有实现(${implementedItems.length})`, implementedItems.length > 0], + ['excluded', `暂不处理(${excludedItems.length})`, excludedItems.length > 0], + ] as const).filter(([, , show]) => show).map(([key, label]) => ( + + ))} +
+ )} + {!!catalog.warnings?.length && {catalog.warnings.map((w, i) =>

第 {w.line} 行:{w.reason}。{w.quote}

)}
} +
+ {visibleMetrics.map(item => { + const itemDone = isItemImplemented(item) + const diagnostic = resolveItemDiagnostic(draft, item, itemDone) + const statusText = item.status === 'unavailable' ? '原文注明无数据源' + : itemDone ? '已有实现 · 待核验' + : (counts.get(item.name) ?? 0) > 1 ? '同名定义 · 需确认口径' + : draft.selected_definition_ids?.length && !draft.selected_definition_ids.includes(item.id) ? '未纳入本次处理' + : draft.status === 'drafting' ? '待处理' + : diagnostic?.kind === 'validation_failed' ? '校验未通过 · 待补充或重试' + : progress ? '未完成 · 需补充信息' : '待实现' + return
+ + {canStart && partial && e.stopPropagation()} + onChange={e => setPicked(e.target.checked ? [...picked, item.id] : picked.filter(id => id !== item.id))} />} + {item.name} + {statusText} + +
+ {(draft.definition_issues ?? []).filter(i => i.definition_id === item.id).map(i => ( +

{i.reason}

+ ))} + {diagnostic && ( +
+ {diagnostic.questions.map((q, idx) => ( +

待确认:{q}

+ ))} + {diagnostic.reasons.map((r, idx) => ( +

未通过原因:{r}

+ ))} +
+ )} + {Object.entries(item.fields).map(([name, value]) =>
{name}:
{value || '未说明'}
)} + {!!item.dependencies.length &&
依赖:{item.dependencies.join('、')}
} +
+
原文 · 第 {item.source.line} 行
{item.source.quote}
+
+ })} + {!canStart && filteredMetrics.length > COLLAPSE_LIMIT && ( + + )} +
+
基础量与全局规则(随选中指标一起用于实现) +
+ {catalog.items.filter(i => i.kind === 'input').map(i =>

{i.name}:{i.formula};{i.policy}

)} + {catalog.rules.map((r, i) =>

{r.text}

)} +
+
+ {error != null && } +
+} + +export function QuerySemanticEditor({ draft, metric, onSaved }: { + draft: CaliberDraft; metric: DraftMetric; onSaved: (fresh: CaliberDraft) => void +}) { + const [value, setValue] = useState(metric.semantic ?? { + version: 1, enabled: false, aliases: metric.synonyms ?? [], grain: metric.grain ?? '', aggregation: '', time: null, entities: [], + }) + const [busy, setBusy] = useState(false) + const [open, setOpen] = useState(false) + const [canonicalEntities, setCanonicalEntities] = useState([]) + const mounted = useRef(false) + useEffect(() => { mounted.current = true; return () => { mounted.current = false } }, []) + useEffect(() => { + if (!open) return + void api.ontology.listEntities().then((res) => { + if (mounted.current) setCanonicalEntities(res.items ?? []) + }).catch(() => {}) + }, [open]) + const [error, setError] = useState(null) + const pending = useRef({ signature: '', id: '' }) + const [dirty, setDirty] = useState(false) + const serverValue = JSON.stringify(metric.semantic ?? { + version: 1, enabled: false, aliases: metric.synonyms ?? [], grain: metric.grain ?? '', aggregation: '', time: null, entities: [], + }) + const editingBase = useRef(serverValue) + const conflict = dirty && editingBase.current !== serverValue + useEffect(() => { + if (!dirty) { + editingBase.current = serverValue + setValue(JSON.parse(serverValue) as QuerySemantics) + } + }, [serverValue, dirty]) + const fields = draft.entry_fields ?? [] + if (!metric.id || !draft.card_id || draft.revision == null) return null + const patch = (change: Partial) => { + if (!dirty) editingBase.current = serverValue + setDirty(true); setValue(current => ({ ...current, ...change })) + } + const save = async () => { + if (conflict) return + const signature = JSON.stringify({ revision: draft.revision, value }) + if (pending.current.signature !== signature) pending.current = { + signature, id: globalThis.crypto?.randomUUID?.() ?? `semantic-${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`, + } + setBusy(true); setError(null) + try { + const fresh = await api.caliberLedger.patchDraft(draft.id, { + operations: [{ op: 'set', metric_id: metric.id!, path: '/semantic', value }], expected_revision: draft.revision!, + request_id: pending.current.id, target: { workflow_id: draft.workflow_id ?? undefined, card_id: draft.card_id! }, + }) + if (!mounted.current) return + setDirty(false) + onSaved(fresh) + toast.ok('问数定义已保存;验证并发布该版本后生效') + pending.current = { signature: '', id: '' } + } catch (e) { setError(e) } finally { setBusy(false) } + } + const selectField = (label: string, current: string, update: (s: string) => void) => + return
setOpen((e.currentTarget as HTMLDetailsElement).open)}> + {metric.name} · 问数定义{metric.semantic?.enabled ? '(已启用)' : '(待确认)'} +
+

这里定义常用问法、日期和对象如何对应已验证的取数参数。汇总规则应与取数实现一致;改动后须重新验证和发布。

+ + +