From be845bb82384c80a4e2725f7434116601ed7e44a Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Thu, 15 Oct 2026 14:32:25 +0800 Subject: [PATCH] =?UTF-8?q?=E8=93=9D=E5=9B=BE=E3=80=8C=E7=9C=8B=E5=93=AA?= =?UTF-8?q?=E4=BA=9B=E6=95=B0=E3=80=8D=E5=AF=B9=E5=88=B0=E7=9A=84=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E6=BA=90=E5=92=8C=E8=A1=A8=E8=A6=81=E7=9C=9F=E7=9A=84?= =?UTF-8?q?=E5=AD=98=E5=9C=A8=EF=BC=9A=E5=8D=A1=E7=89=87=E4=B8=8A=E7=85=A7?= =?UTF-8?q?=E5=AE=9E=E8=AF=B4=E6=98=AF=E5=93=AA=E4=B8=80=E9=A1=B9=E3=80=81?= =?UTF-8?q?=E6=80=8E=E4=B9=88=E6=94=B9=EF=BC=88bp.data=5Fsource=5Funknown?= =?UTF-8?q?=20/=20bp.data=5Ftable=5Funknown=EF=BC=8C=E5=B8=A6=E7=99=BD?= =?UTF-8?q?=E8=AF=9D=EF=BC=89=EF=BC=8C=E6=8C=A1=E4=BD=8F=E8=AF=95=E8=BF=90?= =?UTF-8?q?=E8=A1=8C=E5=92=8C=E5=8F=91=E5=B8=83=E5=8D=95=EF=BC=8C=E5=8A=A9?= =?UTF-8?q?=E6=89=8B=E8=87=AA=E6=9F=A5=E4=B9=9F=E6=8B=BF=E5=BE=97=E5=88=B0?= =?UTF-8?q?=EF=BC=9B=E5=A1=AB=E6=8A=A5=E7=99=BB=E8=AE=B0=E9=87=8C=E6=9C=AC?= =?UTF-8?q?=E5=BA=94=E7=94=A8=E7=9A=84=E8=AE=B0=E5=BD=95=E8=A1=A8=E5=86=99?= =?UTF-8?q?=20@records=E3=80=81=E8=A1=A8=E5=90=8D=E5=9B=BA=E5=AE=9A?= =?UTF-8?q?=EF=BC=8C=E6=8F=90=E7=A4=BA=E8=AF=8D=E5=92=8C=E6=A8=A1=E6=9D=BF?= =?UTF-8?q?=E8=AF=B4=E6=98=8E=E5=86=99=E6=98=8E=E8=BF=99=E4=B8=AA=E7=BA=A6?= =?UTF-8?q?=E5=AE=9A=EF=BC=9B=E3=80=8C=E5=AF=B9=E5=88=B0=E7=9A=84=E8=A1=A8?= =?UTF-8?q?=E3=80=8D=E6=94=B9=E6=88=90=E4=BB=8E=E5=B7=B2=E6=8E=A5=E5=85=A5?= =?UTF-8?q?=E7=9A=84=E6=95=B0=E6=8D=AE=E6=BA=90=E5=92=8C=E8=A1=A8=E9=87=8C?= =?UTF-8?q?=E9=80=89=EF=BC=8C=E5=A1=AB=E6=8A=A5=E7=99=BB=E8=AE=B0=E5=A4=9A?= =?UTF-8?q?=E4=B8=80=E9=A1=B9=E3=80=8C=E6=9C=AC=E5=BA=94=E7=94=A8=E7=9A=84?= =?UTF-8?q?=E8=AE=B0=E5=BD=95=E8=A1=A8=E3=80=8D=EF=BC=8C=E5=8D=A1=E7=89=87?= =?UTF-8?q?=E4=B8=8A=E7=85=A7=E7=99=BD=E8=AF=9D=E6=98=BE=E7=A4=BA=EF=BC=88?= =?UTF-8?q?=E4=BB=A5=E5=89=8D=E8=A6=81=E5=88=B0=E8=AF=95=E8=BF=90=E8=A1=8C?= =?UTF-8?q?=E6=89=8D=E6=8A=A5=E3=80=8C=E7=BB=91=E5=AE=9A=E7=9A=84=E5=B7=A5?= =?UTF-8?q?=E5=85=B7=E4=B8=8D=E5=AD=98=E5=9C=A8=E3=80=8D=EF=BC=8C=E7=94=A8?= =?UTF-8?q?=E6=88=B7=E6=89=BE=E4=B8=8D=E5=88=B0=E5=9C=A8=E5=93=AA=E6=94=B9?= =?UTF-8?q?=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/api/blueprints.py | 22 ++-- backend/app/api/copilot_blueprint.py | 9 +- backend/app/api/personal.py | 8 +- backend/app/blueprints/generation.py | 17 ++- backend/app/blueprints/ops.py | 3 + backend/app/blueprints/publish_plan.py | 5 +- backend/app/blueprints/store.py | 60 ++++++++- backend/app/blueprints/templates/records.py | 3 +- backend/app/blueprints/trial.py | 6 + backend/app/plain.py | 6 + backend/tests/test_blueprint_data_sources.py | 118 ++++++++++++++++++ frontend/src/lib/termsBlueprint.ts | 9 ++ .../src/pages/blueprints/BlueprintCards.tsx | 5 +- frontend/src/pages/blueprints/editors.tsx | 94 ++++++++++++-- 14 files changed, 329 insertions(+), 36 deletions(-) create mode 100644 backend/tests/test_blueprint_data_sources.py diff --git a/backend/app/api/blueprints.py b/backend/app/api/blueprints.py index 1e6e6fa2..0979f507 100644 --- a/backend/app/api/blueprints.py +++ b/backend/app/api/blueprints.py @@ -77,12 +77,12 @@ async def _blueprint(session: AsyncSession, blueprint_id: str) -> Blueprint: return await load(session, blueprint_id) -def blueprint_out(bp: Blueprint) -> dict[str, Any]: - """接口形状（frontend/src/typesBlueprint.ts 的 Blueprint，blueprints/store.blueprint_out）：图本身不下发（只给哈希）， - 关节只给状态，问题带白话。""" - from app.blueprints.store import blueprint_out as shaped +async def blueprint_out(session: AsyncSession, bp: Blueprint) -> dict[str, Any]: + """接口形状（frontend/src/typesBlueprint.ts 的 Blueprint，blueprints/store.blueprint_view）：图本身不下发（只给哈希）， + 关节只给状态，问题带白话，含要查库的检查（对到的数据源和表在不在）。""" + from app.blueprints.store import blueprint_view - return shaped(bp) + return await blueprint_view(session, bp) def _refused(e: Any) -> HTTPException: @@ -180,7 +180,7 @@ async def create_blueprint(payload: BlueprintIn, session: AsyncSession = Depends await record_event(session, bp, "create", {"source": bp.source, "scope": bp.scope}, principal=principal) await session.commit() await session.refresh(bp) - return blueprint_out(bp) + return await blueprint_out(session, bp) def _template_spec(spec: dict[str, Any], template_id: str | None) -> dict[str, Any]: @@ -273,7 +273,7 @@ async def list_blueprints(scope: Literal["shared", "personal"] | None = None, rows += list((await session.execute(select(Blueprint).where(mine).order_by(Blueprint.updated_at.desc()) .limit(_LIST_MAX))).scalars()) rows.sort(key=lambda bp: bp.updated_at or bp.created_at, reverse=True) - return {"items": [blueprint_out(bp) for bp in rows[:_LIST_MAX]]} + return {"items": [await blueprint_out(session, bp) for bp in rows[:_LIST_MAX]]} @router.get("/blueprints/{blueprint_id}") @@ -282,7 +282,7 @@ async def get_blueprint(blueprint_id: str, session: AsyncSession = Depends(get_s from app.blueprints.trial import settle bp = await _blueprint(session, blueprint_id) - return blueprint_out(await settle(session, bp)) + return await blueprint_out(session, await settle(session, bp)) @router.get("/blueprints/{blueprint_id}/conversation") @@ -331,7 +331,7 @@ async def patch_blueprint(blueprint_id: str, payload: BlueprintPatch, session: A raise CodedHTTPException(422, e.message, e.code) from e await session.commit() await session.refresh(bp) - return blueprint_out(bp) + return await blueprint_out(session, bp) # -------------------------------------------------------------------------- @@ -351,7 +351,7 @@ async def instantiate_blueprint(blueprint_id: str, session: AsyncSession = Depen result = await materialize(session, bp, principal=principal) except TrialRefused as e: raise _refused(e) from e - return {"blueprint": blueprint_out(bp), "app_spec": result.app_spec, "problems": result.problems} + return {"blueprint": await blueprint_out(session, bp), "app_spec": result.app_spec, "problems": result.problems} class TrialIn(BaseModel): @@ -376,7 +376,7 @@ async def trial(blueprint_id: str, payload: TrialIn, session: AsyncSession = Dep raise _refused(e) from e except LaunchRefused as e: raise _refused(e) from e - return {"run_id": run.id, "blueprint": blueprint_out(bp), "inputs": dict(run.input or {})} + return {"run_id": run.id, "blueprint": await blueprint_out(session, bp), "inputs": dict(run.input or {})} @router.get("/blueprints/{blueprint_id}/preview") diff --git a/backend/app/api/copilot_blueprint.py b/backend/app/api/copilot_blueprint.py index 6058028e..521d1e94 100644 --- a/backend/app/api/copilot_blueprint.py +++ b/backend/app/api/copilot_blueprint.py @@ -48,7 +48,8 @@ BLUEPRINT_SYSTEM_PROMPT = """\ - 场景：用 bp_template 选模板（见下面的模板清单），选了之后模板会预置几项默认值 - /data 看哪些数，每一项：{"id":"gmv","label":"销售额","unit":"元","table":{"source":"库名","table":"表名"}, "measure":"怎么算（白话或简单聚合）","date":{"column":"业务日期的列","rule":"按支付日"},"filter":"筛选条件","grain":"粒度"} - 对到已发布的口径卡时写 "reuse"（照抄 look 给的复用写法），不写 table + 对到已发布的口径卡时写 "reuse"（照抄 look 给的复用写法），不写 table。table 的 source 只能是上面概览里列出的数据源名， + 不要自己起名；填报登记汇总这个应用自己收上来的记录时，写 {"source":"@records","table":"records"}（记录表建好后系统换成真名） - /cadence 多久更新：{"preset":"daily|weekly|monthly","weekday":1,"day":1,"time":"09:00","tz":"Asia/Shanghai", "period":"yesterday|last_iso_week|last_month|today|this_iso_week|this_month"}（weekly 写 weekday，1 是周一；monthly 写 day） - /people 谁来做，每一步：{"id":"finance","label":"财务会签","who":[{"kind":"user|group|slot|initiator","name":"…"}], @@ -298,7 +299,7 @@ async def blueprint_stream(payload: BlueprintStreamIn, session: AsyncSession = D issues=BLUEPRINT_ISSUES_PROMPT.format(lines="\n".join(f"- {i['path']} {i['message']}" for i in missing)) if missing else ""), history) applier = BlueprintApplier(spec, calibers=calibers) - checker = BlueprintChecker(applier) + checker = BlueprintChecker(applier, template_id=row.template_id) started = time.monotonic() gen = copilot.Generation(op_model, system=system, applier=applier, checker=checker, looks=looks, started=started, look_limit=BLUEPRINT_LOOK_LIMIT) @@ -329,7 +330,7 @@ async def blueprint_stream(payload: BlueprintStreamIn, session: AsyncSession = D target = await own.get(Blueprint, blueprint_id) saved = await store.apply_ops_and_save(own, target, [*applier.effective, *([asked] if asked else [])], actor, verified, "assistant", partial=True) - out = store.blueprint_out(target) + out = await store.blueprint_view(own, target) conflicts = [{"code": r["code"], "level": "warning", "path": r["op"].get("path"), "message": f"助手提的这一处没有采纳：{r['message']}"} for r in saved.refused] answer = applier.explanation or (asked["question"] if asked else "") @@ -533,7 +534,7 @@ async def fill_joints_stream(payload: FillJointsIn, session: AsyncSession = Depe target.joints = joints target.updated_by = actor await own.commit() - out = store.blueprint_out(target) + out = await store.blueprint_view(own, target) yield _sse({"op": "final", "blueprint": out, "groups": {g: {"status": "filled" if not (results.get(g) or {}).get("error") and not (results.get(g) or {}).get("issues") else "failed", diff --git a/backend/app/api/personal.py b/backend/app/api/personal.py index 5959ad13..35b5647c 100644 --- a/backend/app/api/personal.py +++ b/backend/app/api/personal.py @@ -45,11 +45,11 @@ def _refused(e: Any) -> HTTPException: return HTTPException(e.status_code, e.message) -def blueprint_out(row: Any) -> dict[str, Any]: +async def blueprint_out(session: AsyncSession, row: Any) -> dict[str, Any]: """蓝图的接口形状只有一份（api/blueprints.blueprint_out，带蓝图检查的白话）：存成个人应用、取副本交回的也是它。""" from app.api.blueprints import blueprint_out as out - return out(row) + return await out(session, row) async def _blueprint(session: AsyncSession, blueprint_id: str) -> Any: @@ -89,7 +89,7 @@ async def save_personal(blueprint_id: str, session: AsyncSession = Depends(get_s return JSONResponse(status_code=e.status_code, content={"detail": e.message, "code": e.code, "issues": e.issues}) raise _refused(e) from e - return {"app_id": app.id, "blueprint": blueprint_out(await _blueprint(session, blueprint_id))} + return {"app_id": app.id, "blueprint": await blueprint_out(session, await _blueprint(session, blueprint_id))} @router.post("/blueprints/{blueprint_id}/copy", status_code=201) @@ -101,7 +101,7 @@ async def copy_for_review(blueprint_id: str, session: AsyncSession = Depends(get row = await copy(session, await _blueprint(session, blueprint_id), principal=_principal(x_actor)) except PersonalRefused as e: raise _refused(e) from e - return blueprint_out(row) + return await blueprint_out(session, row) # -------------------------------------------------------------------------- diff --git a/backend/app/blueprints/generation.py b/backend/app/blueprints/generation.py index fc3ab8b0..b8610d9f 100644 --- a/backend/app/blueprints/generation.py +++ b/backend/app/blueprints/generation.py @@ -143,18 +143,31 @@ class BlueprintChecker: 交回它改。缺的项（bp.missing、bp.data_unmapped……）不交回：缺的往往是只有用户知道的事，逼它补只会让它编，由它在 这一轮里 bp_ask 问，或者留给用户在卡片上填。""" - def __init__(self, applier: BlueprintApplier) -> None: + def __init__(self, applier: BlueprintApplier, *, template_id: str | None = None) -> None: self.applier = applier + self.template_id = template_id def lint(self) -> list[dict[str, Any]]: from app.blueprints.ops import blueprint_issues return blueprint_issues(self.applier.spec, for_="trial") + async def sources(self) -> list[dict[str, Any]]: + """对到的数据源和表在不在（要查库，用自己的会话：流式回答时接口的会话可能已经关了）。对到一个不存在的源是助手 + 提错了，交回它改，不留给用户到试运行时才碰上。""" + from app.blueprints.store import data_source_problems + from app.db.base import SessionLocal + + spec = self.applier.spec + template_id = (spec.get("template") or {}).get("id") or self.template_id + async with SessionLocal() as session: + return await data_source_problems(session, spec, template_id=template_id) + async def issues(self) -> list[dict[str, Any]]: refused = [{"code": r["code"], "level": "error", "path": r["path"], "message": r["message"]} for r in self.applier.unreported()] - return [*refused, *(i for i in self.lint() if i["code"] == "bp.template_mismatch" and i["level"] == "error")] + return [*refused, *(i for i in self.lint() if i["code"] == "bp.template_mismatch" and i["level"] == "error"), + *await self.sources()] def repair_request(self, issues: list[dict[str, Any]], looks_text: str) -> str: return BLUEPRINT_REPAIR_PROMPT.format(spec=json.dumps(self.applier.spec, ensure_ascii=False), diff --git a/backend/app/blueprints/ops.py b/backend/app/blueprints/ops.py index 8ce2798f..87f3f859 100644 --- a/backend/app/blueprints/ops.py +++ b/backend/app/blueprints/ops.py @@ -119,6 +119,9 @@ def _table(value: Any, _spec: dict[str, Any]) -> Any: if not isinstance(value, dict) or set(value) - {"source", "table"}: raise BpRefused("对到的表要写明数据源和表名两项") out = {k: str(value.get(k) or "").strip() for k in ("source", "table")} + if out["source"] == "@records": + # 本应用的记录表（填报登记，blueprints/store.SELF_RECORDS）只有一张，表名固定，用户和模型都不用写 + out["table"] = "records" if not all(out.values()) or any(len(v) > 200 for v in out.values()): raise BpRefused("对到的表要写明数据源和表名") return out diff --git a/backend/app/blueprints/publish_plan.py b/backend/app/blueprints/publish_plan.py index f738b3ef..79e00d61 100644 --- a/backend/app/blueprints/publish_plan.py +++ b/backend/app/blueprints/publish_plan.py @@ -304,7 +304,10 @@ async def _preflight(session: AsyncSession, bp: Blueprint, target: _Target, *, l from app.core.artifact_store import graph_hash # 蓝图：缺的项、待确认的项（发布前要人看过，bp.unconfirmed）、还没填的生成关节（同蓝图卡片上显示的那一份） - out: list[dict[str, Any]] = [{**i, "source": "blueprint"} for i in blueprint_issues_of(bp)] + from app.blueprints.store import data_source_issues + + out: list[dict[str, Any]] = [{**i, "source": "blueprint"} + for i in [*blueprint_issues_of(bp), *await data_source_issues(session, bp)]] # 发布的级别：受管级、把受管降为已发布都只给管理员（Q8，同发布接口） if level == "governed" and not principal.at_least("admin"): out.append(_problem(ADMIN_REQUIRED, "以受管级别发布需要管理员", source="plan")) diff --git a/backend/app/blueprints/store.py b/backend/app/blueprints/store.py index 2eed3298..246bffc4 100644 --- a/backend/app/blueprints/store.py +++ b/backend/app/blueprints/store.py @@ -22,6 +22,9 @@ from app.blueprints.ops import BpRefused, apply_bp_op, parse_pointer #: 蓝图不存在（或者是别人的个人蓝图） BLUEPRINT_GONE = "蓝图不存在，可能已被删除" +#: 「看哪些数」里指填报登记这个应用自己的记录表（templates/records.RECORDS_SELF）：建表之前还没有数据源名，实例化时 +#: 换成记录表登记成的数据源 +SELF_RECORDS = "@records" PERSONAL_OFF = "管理员还没有开放个人应用：想给自己做一个看板，可以请管理员打开，或请搭建者做成应用" @@ -319,7 +322,60 @@ def blueprint_issues_of(blueprint: Any) -> list[dict[str, Any]]: return [with_plain(i) for i in issues] -def blueprint_out(blueprint: Any, *, issues: bool = True) -> dict[str, Any]: +async def data_source_problems(session: AsyncSession, spec: dict[str, Any], *, + template_id: str | None) -> list[dict[str, Any]]: + """「看哪些数」对到的数据源和表在不在。卡片的白名单（ops._table）只核形状、不查库：对到一个不存在的源，要到试运行 + 发起时才报「绑定的工具不存在」，用户还找不到在哪改——这里提前在卡片上照实说，挡住试运行和发布（P1）。 + + 本应用的记录表写 @records，只有填报登记认；源存在、表结构探查过、表却不在里面的，同样照实说。没探查过的不猜。""" + from app.data import catalog + from app.db.models import DataSource + + items = [(i, item, (item.get("table") or {}).get("value")) for i, item in enumerate(spec.get("data") or []) + if isinstance(item, dict)] + names = {t["source"] for _, _, t in items if isinstance(t, dict) and t.get("source") and t["source"] != SELF_RECORDS} + rows = {r.name: r for r in (await session.execute(select(DataSource).where(DataSource.name.in_(names)))).scalars()} \ + if names else {} + records = template_id == "records" + hint = "；汇总本应用填报的记录，请选「本应用的记录表」" if records else "" + out: list[dict[str, Any]] = [] + for i, item, table in items: + if not isinstance(table, dict) or not table.get("source"): + continue + label = (item.get("label") or {}).get("value") or item.get("id") + path = f"/data/{i}/table" + source = table["source"] + if source == SELF_RECORDS: + if not records: + out.append({"code": "bp.data_source_unknown", "level": "error", "path": path, + "message": f"「{label}」对到了「本应用的记录表」，只有填报登记才有记录表：请重新选数据源"}) + continue + row = rows.get(source) + if row is None or not row.enabled: + out.append({"code": "bp.data_source_unknown", "level": "error", "path": path, + "message": f"「{label}」对到的数据源「{source}」{'已停用' if row is not None else '不存在'}：请在" + f"「看哪些数」里重新选数据源{hint}"}) + elif (row.schema_cache or {}).get("tables") and not catalog.resolve_table_name(row.schema_cache, table.get("table") or ""): + out.append({"code": "bp.data_table_unknown", "level": "error", "path": path, + "message": f"数据源「{source}」里没有表「{table.get('table')}」：请在「看哪些数」里重新选表"}) + return out + + +async def data_source_issues(session: AsyncSession, blueprint: Any) -> list[dict[str, Any]]: + """data_source_problems 加白话，按这张蓝图的模板判。""" + from app.plain import with_plain + + spec = blueprint.spec or {} + template_id = (spec.get("template") or {}).get("id") or blueprint.template_id + return [with_plain(i) for i in await data_source_problems(session, spec, template_id=template_id)] + + +async def blueprint_view(session: AsyncSession, blueprint: Any) -> dict[str, Any]: + """接口交回的蓝图：blueprint_out 再加上要查库的检查（对到的数据源和表在不在）。""" + return blueprint_out(blueprint, extra=await data_source_issues(session, blueprint)) + + +def blueprint_out(blueprint: Any, *, issues: bool = True, extra: list[dict[str, Any]] | None = None) -> dict[str, Any]: """一张蓝图交给前端的样子（frontend/src/typesBlueprint.ts 的 Blueprint）。关节只给状态和模板声明的说法、类型、分组 （JointState：status、by、note、label、kind、group），不给填的值（值在图上看）。""" joints = {jid: {k: v for k, v in state.items() if k in ("status", "by", "note", "label", "kind", "group")} @@ -335,5 +391,5 @@ def blueprint_out(blueprint: Any, *, issues: bool = True) -> dict[str, Any]: "updated_at": blueprint.updated_at.isoformat() if blueprint.updated_at else None, } if issues: - out["issues"] = blueprint_issues_of(blueprint) + out["issues"] = [*blueprint_issues_of(blueprint), *(extra or [])] return out diff --git a/backend/app/blueprints/templates/records.py b/backend/app/blueprints/templates/records.py index 06ec33ba..dc6b85cc 100644 --- a/backend/app/blueprints/templates/records.py +++ b/backend/app/blueprints/templates/records.py @@ -203,7 +203,8 @@ TEMPLATE = register(TemplateDef( cards={"purpose": "必填：一句话说清登记什么、谁来填", "fields": "必填：每次提交填哪些字段（名称、类型、必填、上下限、选项），可以标敏感", "people": "可选：提交后要谁审批（审批通过才落进记录表）", - "data": "可选：汇总看哪些数（对到本应用的记录表）", + "data": "可选：汇总看哪些数。汇总这个应用收上来的记录，对到的表写数据源 @records、表 records（记录表建好后" + "换成真名）", "cadence": "可选：汇总多久更新一次", "access": "可选：谁能填（只看得到自己的记录）、谁能看全部、谁能管理"}, personal=None, diff --git a/backend/app/blueprints/trial.py b/backend/app/blueprints/trial.py index 29ef7f2e..1142ece8 100644 --- a/backend/app/blueprints/trial.py +++ b/backend/app/blueprints/trial.py @@ -274,6 +274,12 @@ async def start_trial(session: AsyncSession, blueprint: Blueprint, *, inputs: di from app.api.deps import restricted from app.engine.launch import launch_run + from app.blueprints.store import data_source_issues + + # 对到的数据源和表不在：发起时只会报「绑定的工具不存在」，这里先按卡片照实说是哪一项、怎么改 + unknown = [i for i in await data_source_issues(session, blueprint) if i.get("level") == "error"] + if unknown: + raise TrialRefused(409, NOT_READY_TEXT, NOT_READY, unknown) result = await materialize(session, blueprint, principal=principal) if result.blocked: raise TrialRefused(409, NOT_READY_TEXT, NOT_READY, [p for p in result.problems if p.get("level") == "error"]) diff --git a/backend/app/plain.py b/backend/app/plain.py index 4aa4b431..959f61ab 100644 --- a/backend/app/plain.py +++ b/backend/app/plain.py @@ -321,6 +321,12 @@ PLAIN.update({ "bp.data_unmapped": Plain( "有要看的数还没对到表或口径", "对不到数据，就算不出这个数。", _A, "请助手按数据目录对到表（或复用已发布的口径），对到之后由你确认。"), + "bp.data_source_unknown": Plain( + "有要看的数对到的数据源不存在", "试运行和发布时找不到这个数据源，这个数算不出来。", _Y, + "在「看哪些数」里点「改」重新选数据源；填报登记汇总本应用的记录，选「本应用的记录表」。"), + "bp.data_table_unknown": Plain( + "有要看的数对到的表不在那个数据源里", "查不到这张表，这个数算不出来。", _Y, + "在「看哪些数」里点「改」重新选表；表刚建的话，先到「数据」页读取一次表结构。"), "bp.people_unbound": Plain( "有一步还没定谁来处理", "流程走到这一步就没人接手。", _Y, "在「谁来做」里选人、组或岗位。"), "bp.template_mismatch": Plain( diff --git a/backend/tests/test_blueprint_data_sources.py b/backend/tests/test_blueprint_data_sources.py new file mode 100644 index 00000000..337dc4f8 --- /dev/null +++ b/backend/tests/test_blueprint_data_sources.py @@ -0,0 +1,118 @@ +"""「看哪些数」对到的数据源和表在不在（blueprints/store.data_source_problems）。 + +卡片的白名单只核形状、不查库：以前对到一个不存在的源（模型自己起的「本应用」），要到试运行发起时才报「绑定的工具 +不存在」，用户还找不到在哪改。现在： +- 卡片上就照实说是哪一项、怎么改（bp.data_source_unknown / bp.data_table_unknown，带白话），挡住试运行和发布； +- 本应用的记录表写 @records，只有填报登记认，表名固定为 records； +- 模型自查（BlueprintChecker）拿得到这类问题，交回它改。 + +库和数字都是编的。 +""" +from __future__ import annotations + +import sqlite3 +import uuid + +import pytest +from httpx import ASGITransport, AsyncClient + +from app.blueprints.ops import BpRefused, apply_bp_op +from app.blueprints.spec import empty_blueprint +from app.blueprints.store import SELF_RECORDS, data_source_problems +from app.data.introspect import introspect +from app.db.base import SessionLocal +from app.db.models import DataSource +from app.engine.runner import run_manager +from app.main import app +from tests import blueprint_samples as s + + +@pytest.fixture(autouse=True) +async def engine_up(): + await run_manager.setup() + yield + await run_manager.shutdown() + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +async def _source(tmp_path, *, explored: bool = True) -> str: + path = tmp_path / f"shop_{uuid.uuid4().hex[:6]}.db" + with sqlite3.connect(path) as db: + db.execute("CREATE TABLE orders (id INTEGER PRIMARY KEY, amount REAL, paid_at TEXT)") + name = f"shop_{uuid.uuid4().hex[:6]}" + async with SessionLocal() as session: + row = DataSource(name=name, kind="sqlite", database=str(path), readonly=True, enabled=True) + session.add(row) + await session.commit() + if explored: + row.schema_cache = await introspect(row) + await session.commit() + return name + + +def _spec(*tables: dict) -> dict: + spec = empty_blueprint() + spec["data"] = [{"id": f"m{i}", "label": s.item(f"指标{i}"), "table": s.item(t)} for i, t in enumerate(tables)] + return spec + + +def test_the_apps_own_records_table_has_a_fixed_name(): + spec = empty_blueprint() + spec = apply_bp_op(spec, {"op": "bp_add", "path": "/data", "value": {"id": "total"}}, source="user") + spec = apply_bp_op(spec, {"op": "bp_set", "path": "/data/0/table", + "value": {"source": SELF_RECORDS, "table": "生产日报记录"}}, source="assistant") + assert spec["data"][0]["table"]["value"] == {"source": SELF_RECORDS, "table": "records"} + with pytest.raises(BpRefused): + apply_bp_op(spec, {"op": "bp_set", "path": "/data/0/table", "value": {"source": "", "table": "x"}}, source="user") + + +async def test_unknown_sources_and_tables_are_named_on_the_card(tmp_path): + known = await _source(tmp_path) + bare = await _source(tmp_path, explored=False) + spec = _spec({"source": "本应用", "table": "生产日报记录"}, {"source": known, "table": "orders"}, + {"source": known, "table": "refunds"}, {"source": bare, "table": "anything"}, + {"source": SELF_RECORDS, "table": "records"}) + async with SessionLocal() as session: + board = await data_source_problems(session, spec, template_id="periodic_board") + records = await data_source_problems(session, spec, template_id="records") + by_path = {i["path"]: i for i in board} + # 不存在的源：说是哪一项、怎么改 + assert by_path["/data/0/table"]["code"] == "bp.data_source_unknown" and "「本应用」不存在" in by_path["/data/0/table"]["message"] + # 对得上的不报；表结构探查过、表不在里面的照实说；没探查过的不猜 + assert "/data/1/table" not in by_path and "/data/3/table" not in by_path + assert by_path["/data/2/table"]["code"] == "bp.data_table_unknown" and "refunds" in by_path["/data/2/table"]["message"] + # 本应用的记录表只有填报登记认 + assert by_path["/data/4/table"]["code"] == "bp.data_source_unknown" + records_paths = {i["path"]: i for i in records} + assert "/data/4/table" not in records_paths + assert "本应用的记录表" in records_paths["/data/0/table"]["message"], "填报登记提示改选本应用的记录表" + assert all(i["level"] == "error" for i in board + records) + + +async def test_a_trial_on_an_unknown_source_is_refused_with_the_card_problem(client, tmp_path): + source = await s.register_source(s.make_source(tmp_path)) + wf_id = await s.published_workflow(s.board_graph(source), f"周报-{uuid.uuid4().hex[:4]}") + bp = (await client.post("/api/blueprints", json={"source": {"kind": "workflow_version", "id": wf_id}, + "scope": "shared"})).json() + await s.edit_spec(bp["id"], purpose=s.item("每周看销售额"), cadence=s.item(s.WEEKLY), + data=_spec({"source": "本应用", "table": "orders"})["data"]) + got = (await client.get(f"/api/blueprints/{bp['id']}")).json() + issue = next(i for i in got["issues"] if i["code"] == "bp.data_source_unknown") + assert issue["path"] == "/data/0/table" and issue["plain"]["who"] == "you" + r = await client.post(f"/api/blueprints/{bp['id']}/trial", json={"inputs": s.PERIOD}) + assert r.status_code == 409, r.text + assert "本应用" in r.text, "挡在发起之前，照实说是哪一项，不是「绑定的工具不存在」" + + +async def test_the_assistants_self_check_sees_unknown_sources(): + from app.blueprints.generation import BlueprintApplier, BlueprintChecker + + spec = _spec({"source": "本应用", "table": "生产日报记录"}) + checker = BlueprintChecker(BlueprintApplier(spec), template_id="records") + issues = await checker.issues() + assert any(i["code"] == "bp.data_source_unknown" for i in issues) diff --git a/frontend/src/lib/termsBlueprint.ts b/frontend/src/lib/termsBlueprint.ts index ddab443f..3488e295 100644 --- a/frontend/src/lib/termsBlueprint.ts +++ b/frontend/src/lib/termsBlueprint.ts @@ -295,6 +295,15 @@ export const BP_OP_TEXT = { export const BP_EDIT_TEXT = { source: '数据源', table: '表名', + /** 填报登记里「看哪些数」对到这个应用自己收上来的记录（服务端写 @records） */ + selfRecords: '本应用的记录表', + selfRecordsHint: '这个应用每次提交存下的那张表；发布后系统建好表、换成真名', + pickSource: '选一个数据源', + pickTable: '选一张表', + tablesLoading: '正在读取表…', + /** 读不到表的列表（没读取过表结构、或没有权限）时直接填表名 */ + tableManual: '读不到这个数据源的表，请直接填表名', + sourcesFailed: '读不到数据源列表，请直接填数据源名和表名', column: '日期列', rule: '按什么算（例如按支付日）', preset: '频率', diff --git a/frontend/src/pages/blueprints/BlueprintCards.tsx b/frontend/src/pages/blueprints/BlueprintCards.tsx index ef941ebe..63c36fe3 100644 --- a/frontend/src/pages/blueprints/BlueprintCards.tsx +++ b/frontend/src/pages/blueprints/BlueprintCards.tsx @@ -25,7 +25,7 @@ import { chatBusy, useBlueprint } from '../../store/blueprint' import type { Blueprint, BpDataItem, BpField, BpItem, BpOp, BpStep, ItemStatus, JointState, PlainIssue, TemplateInfo, } from '../../typesBlueprint' -import { ValueEditor, valueText, type EditorKind, type EditorScope } from './editors' +import { SELF_RECORDS, ValueEditor, valueText, type EditorKind, type EditorScope } from './editors' import { isItem, missingPaths, visibleCards, type CardKey } from './shared' const STATUS_COLOR: Record = { @@ -115,6 +115,7 @@ export function BlueprintCards({ blueprintId, templates }: { blueprintId: string scope: { fields: spec.fields.map((f) => ({ name: f.name, title: String((f.spec?.value as Record | null)?.title ?? f.name) })), metrics: spec.data.map((d) => ({ id: d.id, title: String(d.label?.value ?? d.id) })), + recordsSelf: (spec.template?.id ?? bp.template_id) === 'records', }, } return ( @@ -353,7 +354,7 @@ function DataBlock({ ctx, item, index }: { ctx: CardCtx; item: BpDataItem; index
- {BLUEPRINT_TEXT.dataTable}：{table.source}.{table.table} + {BLUEPRINT_TEXT.dataTable}：{valueText('table', table)} {item.grain?.value != null && {BLUEPRINT_TEXT.dataGrain}：{String(item.grain.value)}} {date?.column && {BLUEPRINT_TEXT.dataDate}：{valueText('date', date)}}
diff --git a/frontend/src/pages/blueprints/editors.tsx b/frontend/src/pages/blueprints/editors.tsx index c45e492b..d7f4f5af 100644 --- a/frontend/src/pages/blueprints/editors.tsx +++ b/frontend/src/pages/blueprints/editors.tsx @@ -7,7 +7,7 @@ * agentlab.form/1 的一个属性另加是否必填；字段对应的取值是表单字段（form.x）、指标（metric.y）、核对出来的行（rows）或 * 常量（文字加单引号）。说法都是白话：时限写「2 天 3 小时」，存的是 ISO 8601 时长；处理人写名字，不写 kind。 */ -import { useState } from 'react' +import { useEffect, useState } from 'react' import type { ReactNode } from 'react' import { Plus, X } from 'lucide-react' import { PrincipalPicker, type Principal as PickerPrincipal } from '../../components/PrincipalPicker' @@ -17,17 +17,23 @@ import { TIMEOUT_LABEL, WEEKDAY_LABEL, cadenceText, durationIso, durationText, policyText, principalText, } from '../../lib/terms' import type { BpPrincipal } from '../../typesBlueprint' +import type { DataSourceName } from '../../types' +import { api } from '../../api/client' export type EditorKind = | 'text' | 'textarea' | 'table' | 'date' | 'cadence' | 'principals' | 'grantees' | 'policy' | 'duration' | 'timeout' | 'threshold' | 'field' | 'bool' | 'mapping' -/** 编辑框要的选项：「填什么」里的字段、「看哪些数」里的指标（金额线、字段对应从里面选） */ +/** 编辑框要的选项：「填什么」里的字段、「看哪些数」里的指标（金额线、字段对应从里面选）；填报登记能对到本应用的记录表 */ export interface EditorScope { fields: { name: string; title: string }[] metrics: { id: string; title: string }[] + recordsSelf?: boolean } +/** 「看哪些数」里指这个应用自己的记录表（服务端 blueprints/store.SELF_RECORDS），表名固定 */ +export const SELF_RECORDS = '@records' + type Obj = Record const obj = (v: unknown): Obj => (v && typeof v === 'object' && !Array.isArray(v) ? v as Obj : {}) const principals = (v: unknown): BpPrincipal[] => (Array.isArray(v) ? v.filter((p) => p && typeof p.name === 'string') : []) @@ -79,7 +85,9 @@ export function valueText(kind: EditorKind, value: unknown, scope?: EditorScope) if (value == null || value === '') return BLUEPRINT_TEXT.empty const o = obj(value) switch (kind) { - case 'table': return o.source || o.table ? `${o.source ?? ''}.${o.table ?? ''}` : BLUEPRINT_TEXT.empty + case 'table': + if (o.source === SELF_RECORDS) return BP_EDIT_TEXT.selfRecords + return o.source || o.table ? `${o.source ?? ''}.${o.table ?? ''}` : BLUEPRINT_TEXT.empty case 'date': return o.column ? `${o.column}${o.rule ? `（${o.rule}）` : ''}` : BLUEPRINT_TEXT.empty case 'cadence': return cadenceText(o) || BLUEPRINT_TEXT.empty case 'principals': @@ -160,17 +168,85 @@ function TextEditor({ kind, value, busy, onSave, onCancel }: ValueEditorProps) { ) } -function TableEditor({ value, busy, onSave, onCancel }: ValueEditorProps) { +/** + * 对到哪张表：先选数据源（已接入的，填报登记另有「本应用的记录表」），再选表（读这个数据源读取过的表结构）。读不到列表时 + * 退回直接填写——合不合规照旧由服务端判，对到不存在的源、表由蓝图检查在卡片上照实说 + */ +function TableEditor({ value, busy, onSave, onCancel, scope }: ValueEditorProps) { const o = obj(value) const [source, setSource] = useState(String(o.source ?? '')) - const [table, setTable] = useState(String(o.table ?? '')) + const [table, setTable] = useState(String(o.source === SELF_RECORDS ? 'records' : o.table ?? '')) + const [sources, setSources] = useState(null) + const [tables, setTables] = useState(null) + const [loadingTables, setLoadingTables] = useState(false) + const [manual, setManual] = useState(false) + useEffect(() => { + let alive = true + api.datasources.names() + .then((list) => { if (alive) setSources(list.filter((x) => x.enabled !== false)) }) + .catch(() => { if (alive) { setSources([]); setManual(true) } }) + return () => { alive = false } + }, []) + const picked = sources?.find((x) => x.name === source) + useEffect(() => { + setTables(null) + if (!picked) return + let alive = true + setLoadingTables(true) + api.datasources.schema(picked.id) + .then((r) => { if (alive) setTables(Array.isArray(r?.tables) ? r.tables.map(String) : []) }) + .catch(() => { if (alive) setTables([]) }) + .finally(() => { if (alive) setLoadingTables(false) }) + return () => { alive = false } + }, [picked?.id]) // eslint-disable-line react-hooks/exhaustive-deps + const self = source === SELF_RECORDS + const ok = self || (!!source.trim() && !!table.trim()) + const save = () => onSave(self ? { source: SELF_RECORDS, table: 'records' } : { source: source.trim(), table: table.trim() }) + if (manual) { + return ( + +

{BP_EDIT_TEXT.sourcesFailed}

+
+ {(p) => setSource(e.target.value)} />} + {(p) => setTable(e.target.value)} />} +
+ + ) + } return ( - onSave({ source: source.trim(), table: table.trim() })}> +
- {(p) => setSource(e.target.value)} />} - {(p) => setTable(e.target.value)} />} + + {(p) => ( + + )} + + + {(p) => self ? ( + + ) : tables && tables.length ? ( + + ) : ( + setTable(e.target.value)} /> + )} +
+ {self &&

{BP_EDIT_TEXT.selfRecordsHint}

} ) }