diff --git a/backend/app/api/catalog.py b/backend/app/api/catalog.py index 370c412e..6a7ee7ef 100644 --- a/backend/app/api/catalog.py +++ b/backend/app/api/catalog.py @@ -700,8 +700,9 @@ class AssistantMessageIn(BaseModel): class AssistantIn(BaseModel): #: 整段对话(服务端不存);没有使用者的话时助手先开口 messages: list[AssistantMessageIn] = Field(default_factory=list, max_length=catalog_assistant.MAX_MESSAGES) - #: 界面上还没保存的修改,让模型知道 - pending: list[CatalogPatchChange] = Field(default_factory=list, max_length=catalog_assistant.MAX_PENDING) + #: 界面上还没保存的修改,让模型知道。不设上限:宽表上几轮攒下来常常超过一次保存的 200 项,不能因此不让问(背景里只列 + #: 最近的 catalog_assistant.PENDING_SHOWN 项,码值照全部合并) + pending: list[CatalogPatchChange] = Field(default_factory=list) @router.post("/{source_id}/catalog/{table}/assistant") diff --git a/backend/app/data/catalog_assistant.py b/backend/app/data/catalog_assistant.py index e8fd3b28..27d848f5 100644 --- a/backend/app/data/catalog_assistant.py +++ b/backend/app/data/catalog_assistant.py @@ -7,7 +7,8 @@ **只改目录、不改业务库。** 模型只有两个工具: - propose_changes:一批修改提案。按 catalog.plan_patch 核对(取值格式、列名必须在表结构里、公式不收),遮罩列不给码值,已经 - 是这个值又确认过的不提;核对过的作为 proposal 事件交给界面,**不写目录**。没收下的项和原因交回模型(不带任何取值)。 + 是这个值又确认过的不提;有效记录条件是模型写的 SQL,另按起草的规矩核对(_Tools.filter_problem:一个条件、只用这张表的 + 列、不能有子查询……)。核对过的作为 proposal 事件交给界面,**不写目录**。没收下的项和原因交回模型(不带任何取值)。 界面上还没保存的修改(pending)里同一列的码值并进新提的码值:界面上同一路径以后一次为准,不能把先提的码值丢了。 - run_query:只读查询,只有数据源开了数据剖析才给。经剖析的只读守卫和单条时限(catalog_profile._Runner: guard.check(readonly=True),单条时限取剖析设置和数据源查询时限中较小的一个),每次请求最多 MAX_QUERIES 条(守卫拒掉的 @@ -36,7 +37,9 @@ catalog.MODEL_MAX_TOKENS,服务方嫌大时退回 catalog.MODEL_FALLBACK_MAX_T 等模型的时候每 HEARTBEAT_S 秒发一行 SSE 注释(「: keep-alive」,不是事件),免得中间的代理断开。 **对话不存服务端**:每次请求带上整段对话(最多 MAX_MESSAGES 条、每条最多 MESSAGE_MAX 字)。查询结果只在这一次请求里 -看得到,提示词里让模型把要用的结论写进回答。 +看得到,提示词里让模型把要用的结论写进回答。界面上待保存的修改不设上限(宽表上逐列补中文名、含义,几轮下来常常超过一次 +保存的 200 项,不能因此不让问):背景里只列最近的 PENDING_SHOWN 项(界面上新的提案接在后面),写明另有几项;码值照全部 +待保存的修改合并。 写给模型的文字都在名字带 _SYSTEM / _PROMPT 的常量里;这里的 ERROR_* 是给人看的。 """ @@ -56,17 +59,18 @@ from typing import Any from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession -from app.data import catalog, catalog_jobs, catalog_profile +from app.data import catalog, catalog_jobs, catalog_profile, sqlcheck from app.data.catalog_review import SOURCE_LABEL, item_label, slot_path, table_usage_counts +from app.data.catalog_transfer import filter_columns_problem from app.data.draft_model import DRAFT_MODEL_PLACE from app.data.engine import SnapshotTampered, masked_columns logger = logging.getLogger(__name__) -#: 对话最多几条、每条最多几个字;界面上待保存的修改最多几项(同 PATCH 的上限) +#: 对话最多几条、每条最多几个字;界面上待保存的修改在背景里最多列几项(最近的;请求里交多少都收) MAX_MESSAGES = 40 MESSAGE_MAX = 4000 -MAX_PENDING = catalog.PATCH_MAX_CHANGES +PENDING_SHOWN = 200 #: 每次请求最多调用几次模型、最多几条只读查询 MAX_ROUNDS = 8 MAX_QUERIES = 5 @@ -133,7 +137,9 @@ fact/dimension/snapshot/log/config、business_date {"column","rule","timezone"} flow/stock/ratio/identifier/status/attribute、codes 码值 {"码":"含义"});已有的关联关系写 relations.<编号>;新增关联关系\ 写 relations.new,value 写 {"columns":[本表列],"to_table":"目标表","to_columns":[目标列],\ "cardinality":"many_to_one/one_to_one/one_to_many"}。 -- codes 只写要补充或更正的码,原有的码会保留;valid_filter 要写成完整的条件(原有条件也要带上)。""" +- codes 只写要补充或更正的码,原有的码会保留;valid_filter 要写成完整的条件(原有条件也要带上)。 +- valid_filter 只能用这张表的列(不带表名),由比较、IN、BETWEEN、LIKE、IS NULL 用 AND / OR / NOT 连起来,例如 \ +status NOT IN ('C', 'X');不能有子查询、聚合、窗口函数、分号,函数只能用 UPPER、LOWER、TRIM、COALESCE。""" #: 查询工具的说明(开了剖析时接在系统提示后面)、没开剖析时的那句 ASSISTANT_QUERY_PROMPT = """只读查询:run_query 在这个数据源上执行一条只读 SQL({dialect} 的写法),最多返回 {rows} 行,\ @@ -180,6 +186,7 @@ ASSISTANT_PROMPT_LABELS = { "unnamed": "目录里还没有中文名和含义的列:{cols}(共 {n} 列{more})", "unnamed_more": ",这里列出前 {n} 个", "pending": "界面上还没保存的修改(使用者已经接受、还没保存):", + "pending_more": "(另有 {n} 项更早的待保存修改没有列出:太多了。它们同样已经接受,不要重复提)", "pending_item": "- {path}:{value}(依据:{reason})", "status_proposed": "推断", "status_verified": "有确证", @@ -509,7 +516,6 @@ async def prepare(session: AsyncSession, row: Any, source: Any, table: str, if unnamed: more = labels["unnamed_more"].format(n=UNNAMED_SHOWN) if len(unnamed) > UNNAMED_SHOWN else "" lines.append(labels["unnamed"].format(cols="、".join(unnamed[:UNNAMED_SHOWN]), n=len(unnamed), more=more)) - pending = list(pending) pending_codes: dict[str, dict[str, Any]] = {} shown_pending: list[str] = [] for change in pending: @@ -527,7 +533,10 @@ async def prepare(session: AsyncSession, row: Any, source: Any, table: str, reason = str(change.get("reason") or "").strip() shown_pending.append(labels["pending_item"].format(path=path, value=_value_text(value), reason=reason or "—")) if shown_pending: - lines += ["", labels["pending"], *shown_pending] + lines += ["", labels["pending"]] + if len(shown_pending) > PENDING_SHOWN: + lines.append(labels["pending_more"].format(n=len(shown_pending) - PENDING_SHOWN)) + lines += shown_pending[-PENDING_SHOWN:] if can_query: tools = ASSISTANT_QUERY_PROMPT.format(dialect=row.kind, rows=QUERY_ROWS, limit=MAX_QUERIES) else: @@ -652,6 +661,18 @@ class _Tools: return change return {**change, "value": {**before, **value}} + def filter_problem(self, condition: Any) -> str | None: + """模型提的有效记录条件哪里不合规(交回模型的一句话),合规是 None。 + + 按模型起草时的规矩核对(sqlcheck.valid_filter_problem:一个条件、只用这张表的列、不能有子查询、聚合、窗口、分号, + 只能用比较、IN、BETWEEN、LIKE、IS NULL 和几个函数);sqlcheck 里还没有这条规矩时,至少核对用到的列都是这张表的、 + 都在表结构里、没有子查询(catalog_transfer.filter_columns_problem,导入时核对人工写的条件也用它)。""" + meta = self.ctx.tables.get(self.ctx.table) or {} + strict = getattr(sqlcheck, "valid_filter_problem", None) + if strict is not None: + return strict(condition, meta, kind=self.ctx.kind, table=self.ctx.table) + return filter_columns_problem(condition, meta, kind=self.ctx.kind, table=self.ctx.table) + async def propose(self, changes: Any) -> tuple[dict[str, Any] | None, str]: """(给界面的 proposal 事件, 交回模型的结果)。提案不写目录。""" from app.db.base import SessionLocal @@ -674,6 +695,9 @@ class _Tools: if change.state == "same": dropped.append(prompts["same"].format(where=_where(key, change))) continue + if key == ("t", "valid_filter") and (problem := self.filter_problem(change.after)) is not None: + dropped.append(problem) + continue kept.append(change) said: list[str] = [] event: dict[str, Any] | None = None @@ -883,5 +907,5 @@ async def stream(model: Any, model_id: str, ctx: AssistantContext, __all__ = ["AssistantContext", "CODE_FAILED", "CODE_MODEL", "CODE_ROUNDS", "CODE_TIMEOUT", "CODE_TRUNCATED", - "HEARTBEAT_S", "MAX_MESSAGES", "MAX_PENDING", "MAX_QUERIES", "MAX_ROUNDS", "MESSAGE_MAX", "PROPOSE_TOOL", + "HEARTBEAT_S", "MAX_MESSAGES", "MAX_QUERIES", "MAX_ROUNDS", "MESSAGE_MAX", "PENDING_SHOWN", "PROPOSE_TOOL", "QUERY_ROWS", "QUERY_TOOL", "SHOWN_ROWS", "conversation_problem", "prepare", "resolve_model", "stream"] diff --git a/backend/tests/test_catalog_assistant.py b/backend/tests/test_catalog_assistant.py index 09a4a769..8c4b4470 100644 --- a/backend/tests/test_catalog_assistant.py +++ b/backend/tests/test_catalog_assistant.py @@ -1,9 +1,11 @@ """表助手(data/catalog_assistant.py,POST /api/datasources/{id}/catalog/{table}/assistant)。 - 开始前:表不在 404;起草模型用不了 409 catalog_draft_model_unavailable(说法同起草任务);对话超过 40 条、一条超过 - 4000 字、待保存的修改超过 200 项 422 + 4000 字 422。界面上待保存的修改不设上限(宽表上攒到 200 项以上也照常对话):背景里只列最近的 200 项、写明另有几项, + 码值照全部待保存的修改合并 - 流:text 增量、query(跑过的只读查询,前 10 行,遮罩列写「已遮罩」)、proposal(按 plan_patch 核对过的提案,形状同助手流 - 的 catalog_patch:公式不收、列名必须在表结构里、遮罩列不给码值)、done(用量、查询条数)、error(中途出错,之后没有 done) + 的 catalog_patch:公式不收、列名必须在表结构里、遮罩列不给码值;有效记录条件要是只用这张表的列的一个条件,有起草的 + 校验规矩 sqlcheck.valid_filter_problem 时按它)、done(用量、查询条数)、error(中途出错,之后没有 done) - 交给模型的背景:表结构、这张表的目录(每项的来源和状态)、关联关系(含别的表指过来的)、使用次数、数据源的起草要求、 最近一次起草留空的列、界面上还没保存的修改;遮罩列只给列名、不给任何值 - 只读查询:只有开了数据剖析的源才给这个工具;经剖析的只读守卫和单条时限;每次请求最多 5 条;最多 50 行交给模型;结果 @@ -207,8 +209,6 @@ async def test_refusals_before_the_stream(client, tmp_path, monkeypatch): assert (await client.post(_url(sid), json=too_many)).status_code == 422 too_long = _ask("长" * 4001) assert (await client.post(_url(sid), json=too_long)).status_code == 422 - pending = [{"path": "label", "value": "订单", "reason": "x"}] * 201 - assert (await client.post(_url(sid), json=_ask("x", pending=pending))).status_code == 422 wrong_role = {"messages": [{"role": "system", "content": "x"}]} assert (await client.post(_url(sid), json=wrong_role)).status_code == 422 last_is_answer = {"messages": [{"role": "user", "content": "x"}, {"role": "assistant", "content": "y"}]} @@ -333,6 +333,8 @@ async def test_background_has_structure_catalog_and_hides_masked_values(client, # 规矩 for rule in ("只改数据目录", "不改业务库", "公式", "拿不准", "原话"): assert rule in system + # 有效记录条件的写法(同起草时的校验规矩),免得提了再被退回 + assert "只能用这张表的列" in system and "不能有子查询" in system and "UPPER、LOWER、TRIM、COALESCE" in system # 表结构、目录(来源和状态)、使用次数、起草要求、数据源说明 for fact in ("order_no", "status", "amount", "member_id → members.id", "主键", "每笔订单一行", "人工填写", "已确认", "模型起草", "已驳回", "还没有被运行用到过", "金额一律按元", "虚构的网店库", "数据剖析:全表 30 行"): @@ -464,6 +466,83 @@ async def test_proposal_codes_build_on_the_pending_ones(client, tmp_path, monkey assert change["after"] == {"1": "已支付", "2": "已退款", "9": "作废"} +async def test_many_pending_changes_do_not_block_the_conversation(client, tmp_path, monkeypatch): + """宽表上逐列补中文名、含义,待保存的修改攒到 200 项以上:照常对话(不 422)。背景里只列最近的 200 项、写明另有几项; + 码值照全部待保存的修改合并(最早那条码值没列进背景,新提的码值也不能把它丢了)。""" + sid = await _source(client, tmp_path) + oldest = {"path": "columns.status.codes", "value": {"2": "已退款"}, "reason": "用户说:2 是退款"} + labels = [{"path": f"columns.c{i:03d}.label", "value": f"第 {i} 列", "reason": f"用户说:第 {i} 列"} + for i in range(201)] + model = Scripted([Turn(calls=[("propose_changes", {"changes": [ + {"path": "columns.status.codes", "value": {"9": "作废"}, "reason": "用户说:9 是作废"}]})]), + Turn(text=["好。"])]) + use(monkeypatch, model) + events, _ = await _talk(client, sid, _ask("9 是作废", pending=[oldest, *labels])) + assert events[-1]["type"] == "done" + system = model.seen[0][0].content + # 最近的 200 项列出来了,最早的两项没列,写明另有 2 项 + assert "columns.c200.label" in system and "columns.c001.label" in system + assert "columns.c000.label" not in system and "用户说:2 是退款" not in system + assert "另有 2 项" in system + [proposal] = [e for e in events if e["type"] == "proposal"] + [change] = proposal["changes"] + assert change["value"] == {"2": "已退款", "9": "作废"} + # 再多也收 + model = Scripted([Turn(text=["好。"])]) + use(monkeypatch, model) + many = [{"path": f"columns.c{i:04d}.meaning", "value": "说明", "reason": "x"} for i in range(1200)] + events, _ = await _talk(client, sid, _ask("接着看", pending=many)) + assert events[-1]["type"] == "done" and "另有 1000 项" in model.seen[0][0].content + + +async def _propose_filters(client, sid: str, monkeypatch, filters: list[str]) -> tuple[list[dict], Scripted]: + """模型一轮提一条有效记录条件(同一条提案里同一个路径只认第一次),最后一轮回答。""" + turns = [Turn(calls=[("propose_changes", {"changes": [ + {"path": "valid_filter", "value": f, "reason": "用户说:统计时排除作废"}]})]) for f in filters] + model = Scripted([*turns, Turn(text=["好。"])]) + use(monkeypatch, model) + events, _ = await _talk(client, sid, _ask("统计时排除作废的单子")) + return events, model + + +async def test_valid_filter_proposals_must_be_a_condition_on_this_table(client, tmp_path, monkeypatch): + """有效记录条件是模型写的 SQL:引用表结构里没有的列、别的表的列、带子查询的不交给界面,原因交回模型;合规的照常提。""" + sid = await _source(client, tmp_path) + events, model = await _propose_filters(client, sid, monkeypatch, [ + "status <> 9 AND ghost_col = 1", + "members.nick IS NOT NULL", + "member_id IN (SELECT id FROM members)", + "status <> 9", + ]) + proposals = [e for e in events if e["type"] == "proposal"] + assert [[c["value"] for c in p["changes"]] for p in proposals] == [["status <> 9"]] + said = [[m for m in seen if isinstance(m, ToolMessage)][-1].content for seen in model.seen[1:5]] + assert "ghost_col" in said[0] and "不在表结构里" in said[0] and "没有可以交给使用者的修改" in said[0] + assert "members.nick" in said[1] + assert "子查询" in said[2] + assert "已把 1 项修改交给使用者确认" in said[3] + + +async def test_valid_filter_proposals_follow_the_drafting_rules(client, tmp_path, monkeypatch): + """有起草的校验规矩(sqlcheck.valid_filter_problem:只能用几个函数、不能有聚合……)时按它核对,说法照它的。""" + from app.data import sqlcheck + + sid = await _source(client, tmp_path) + calls: list[tuple] = [] + + def strict(condition, meta, *, kind, table=None): + calls.append((condition, [c["name"] for c in meta["columns"]], kind, table)) + return "有效行过滤里只能用 UPPER、LOWER、TRIM、COALESCE 这几个函数,不能用 ABS" if "ABS" in condition else None + + monkeypatch.setattr(sqlcheck, "valid_filter_problem", strict, raising=False) + events, model = await _propose_filters(client, sid, monkeypatch, ["ABS(amount) > 0", "status <> 9"]) + proposals = [e for e in events if e["type"] == "proposal"] + assert [[c["value"] for c in p["changes"]] for p in proposals] == [["status <> 9"]] + assert "不能用 ABS" in [m for m in model.seen[1] if isinstance(m, ToolMessage)][-1].content + assert calls[0][0] == "ABS(amount) > 0" and calls[0][2:] == ("sqlite", "orders") + assert "status" in calls[0][1] and "member_id" in calls[0][1] + + async def test_nothing_acceptable_means_no_proposal_event(client, tmp_path, monkeypatch): sid = await _source(client, tmp_path) model = Scripted([Turn(calls=[("propose_changes", {"changes": [{"path": "columns.ghost.label", "value": "x"}]})]),