diff --git a/backend/app/engine/evidence.py b/backend/app/engine/evidence.py index e15404b6..ab6d9595 100644 --- a/backend/app/engine/evidence.py +++ b/backend/app/engine/evidence.py @@ -1719,6 +1719,41 @@ def _resolve_entity(marker: dict[str, Any], catalog: dict[str, Any], snaps: _Sna "kind": ev_kind, "role": "entity", "status": "resolved", "rendered": ref} +#: 字段标记前面这几种连接:「产量表的」「产量表中的」「产量表里的」,或者只隔空白(_short_column) +_SHORT_JOINERS = ("中的", "里的", "的", "") +#: 流式渲染时为 _short_column 留着的已渲染文字有多长:表名再长也够看 +SHORT_LOOKBACK = 160 + + +def _short_column(before: str, ref: str) -> str | None: + """[[c:表.列]] 前面紧挨着(隔空白、「的」「中的」「里的」)刚写过同一张表的名字时,正文里只显示列名:交回列名,不用缩短的 + 交回 None。部署上撞过「取自 某表 的 某表.某列」。 + + before 是标记前面已经渲染出来的正文([[t:表]] 渲染出来就是表名本身,写成文字的表名也算)。表名按 name_key 比(只对 + ASCII 不分大小写),带库名的写法(db.表.列)全名、末段都认;英文名前面还连着字母、数字、下划线的不算(myorders 不是 + orders)。粗体、斜体收尾的星号、下划线跳过。""" + table, dot, column = ref.strip().rpartition(".") + if not dot or not table or not column: + return None + tail = before.rstrip(_BLANK) + for joiner in _SHORT_JOINERS: + if joiner and not tail.endswith(joiner): + continue + head = tail[:len(tail) - len(joiner)].rstrip(_BLANK) if joiner else tail + if not joiner and head == before: + continue # 什么都没隔:「产量表产量表.工厂」不是这种写法,不动 + head = head.rstrip("*_") + for name in dict.fromkeys((table, table.rsplit(".", 1)[-1])): + start = len(head) - len(name) + if start < 0 or name_key(head[start:]) != name_key(name): + continue + if start > 0 and name[0].isascii() and (head[start - 1].isascii() and (head[start - 1].isalnum() + or head[start - 1] == "_")): + continue + return column + return None + + # -------------------------------------------------------------------------- # 引文:[[q:K1|原话]] 要在检索快照的命中片段里逐字出现 # -------------------------------------------------------------------------- @@ -2154,14 +2189,19 @@ def _wordish(ch: str) -> bool: return ch.isdecimal() or (ch.isascii() and ch.isalnum()) -def _render(text: str, catalog: dict[str, Any], snaps: _Snapshots, cells_allowed: bool, prev: str = "") -> str: +def _render(text: str, catalog: dict[str, Any], snaps: _Snapshots, cells_allowed: bool, prev: str = "", + before: str = "") -> str: + """before:这段文字前面已经渲染出来的正文(流式渲染分块时用),字段标记要看前面刚写过的表名(_short_column)。""" text = _tidy(_expand_tables(text, catalog, snaps, cells_allowed, prev), catalog, snaps, cells_allowed, prev) out: list[str] = [] last = 0 for marker in parse_markers(text): out.append(text[last:marker["start"]]) if marker["kind"] != "see": - out.append(resolve_marker(marker, catalog, cells_allowed=cells_allowed, loader=snaps)["rendered"]) + cite = resolve_marker(marker, catalog, cells_allowed=cells_allowed, loader=snaps) + short = _short_column(before + "".join(out), cite["ref"]) \ + if marker["kind"] == "c" and cite["status"] == "resolved" else None + out.append(short or cite["rendered"]) last = marker["end"] out.append(text[last:]) return "".join(out) @@ -2247,6 +2287,8 @@ class StreamRenderer: self.snaps = _Snapshots(loader) self.pending = "" self.prev = "" + #: 已经放出去的正文的末尾一截:字段标记要看前面刚写过的表名(_short_column) + self.shown = "" def _has_unit(self, marker: dict[str, Any]) -> bool: return bool(_rendered_unit(marker, self.catalog, self.snaps, self.cells_allowed)) @@ -2254,8 +2296,9 @@ class StreamRenderer: def _release(self, ready: str) -> str: if not ready: return "" - out = _render(ready, self.catalog, self.snaps, self.cells_allowed, self.prev) + out = _render(ready, self.catalog, self.snaps, self.cells_allowed, self.prev, self.shown) self.prev = ready[-1] + self.shown = (self.shown + out)[-SHORT_LOOKBACK:] return out def feed(self, delta: str) -> str: @@ -2868,6 +2911,9 @@ def compose_doc( continue cite = _stored(resolve_marker(marker, catalog, cells_allowed=cells_allowed, loader=snaps)) ok = cite["status"] == "resolved" + if ok and marker["kind"] == "c" and (short := _short_column("".join(parts), cite["ref"])): + # 紧跟在同一张表名后面:只显示列名,引用上记 short,复核照它核对(老文档没有这一项,照全名) + cite = {**cite, "rendered": short, "short": True} seg = { "kind": _SEG_KIND.get(marker["kind"]) or ("number" if marker["kind"] == "m" or (ok and is_number(cite.get("value"))) @@ -3217,6 +3263,10 @@ def verify_doc( cite = _phase_two_cite(seg, ref, text) \ or resolve_ref(ref, catalog, cells_allowed=cells_allowed, loader=snaps) ok = cite["status"] == "resolved" + if ok and ref.startswith("c:") and (seg.get("cite") or {}).get("short") is True \ + and "." in str(cite.get("rendered") or ""): + # 组装时紧跟在同一张表名后面、只显示了列名(_short_column):照列名核对 + cite = {**cite, "rendered": str(cite["rendered"]).rsplit(".", 1)[-1]} # 反引号里的实体:片段的字带着两个反引号 shown = f"`{cite['rendered']}`" if seg.get("code") and seg.get("kind") == "entity" else cite["rendered"] matches = shown == text @@ -3609,7 +3659,8 @@ ENTITY_RULES = ( "表和字段(系统会核对):提到表名、字段名时写 [[t:表名]]、[[c:表名.字段名]];英文的名字也可以放进反引号" "(`orders`),中文的表名、字段名一律用 [[t:]] / [[c:]]——反引号里的中文名系统不核对;" "只写下面列出的、或者这次查过的数据源里真实存在的名字——本次运行的表结构快照、查询用到的表、" - "查询结果列里都没有的名字,会被标成「可能是编造的名字」。\n" + "查询结果列里都没有的名字,会被标成「可能是编造的名字」。" + "紧跟在同一张表名后面的字段(「[[t:orders]] 的 [[c:orders.amount]]」)只显示字段名,不用为了不重复把字段改写成别的说法。\n" "说明一个数是怎么算出来的(按哪个字段去重、怎么汇总、筛了什么条件)时,句末挂 [[see:Q1]] 指向算出它的那次" "查询(下面每张表后面写了哪几次查询用到它):只挂 [[t:]]、[[c:]] 指不出是哪一次查询算出的这个数。" ) diff --git a/backend/tests/test_evidence_entities.py b/backend/tests/test_evidence_entities.py index 668f2c38..87715859 100644 --- a/backend/tests/test_evidence_entities.py +++ b/backend/tests/test_evidence_entities.py @@ -161,10 +161,11 @@ def test_phase_two_ledgers_build_no_entities(): def test_entity_markers_render_as_the_name(catalog): doc = compose_doc("数据来自 [[t:orders]] 的 [[c:orders.amount]] 和 [[c:order_cnt]]。[[see:t:refunds]]", catalog) - assert doc["markdown"] == "数据来自 orders 的 orders.amount 和 order_cnt。" + # 紧跟在同一张表名后面的字段只显示字段名(不写成「orders 的 orders.amount」),出处照旧是那个字段 + assert doc["markdown"] == "数据来自 orders 的 amount 和 order_cnt。" segs = entity_segments(doc) assert [(s["text"], s["cite"]["kind"], s["state"]) for s in segs] == [ - ("orders", "table", "deterministic"), ("orders.amount", "column", "deterministic"), + ("orders", "table", "deterministic"), ("amount", "column", "deterministic"), ("order_cnt", "column", "deterministic")] assert segs[0]["cite"]["eid"] == catalog["t:orders"]["eid"] assert doc["violations"] == [] and doc["stats"]["entities"] == 3 diff --git a/backend/tests/test_evidence_unicode_names.py b/backend/tests/test_evidence_unicode_names.py index 2135cee4..cca25ed5 100644 --- a/backend/tests/test_evidence_unicode_names.py +++ b/backend/tests/test_evidence_unicode_names.py @@ -97,7 +97,8 @@ def test_new_doc_records_the_syntax_and_resolves_chinese_names(catalog): doc = compose("本月[[t:明细]]的[[c:明细.金额]]按[[c:区域]]汇总,[[v:Q1.r0.区域]]最高。[[see:t:汇总,Q1]]", catalog) assert ENTITY_SYNTAX == 2 and doc["entity_syntax"] == 2 and doc_entity_syntax(doc) == 2 assert '"entity_syntax":2' in canonical_json(doc) # 在文档顶层,参与内容哈希 - assert doc["markdown"] == "本月明细的明细.金额按区域汇总,分区甲最高。" # 显示写作者写的名字本身 + # 显示写作者写的名字本身;紧跟在同一张表名后面的字段只显示字段名(不写成「明细的明细.金额」) + assert doc["markdown"] == "本月明细的金额按区域汇总,分区甲最高。" assert doc["violations"] == [] table, column, shared = entities(doc) assert table["cite"]["alias"] == "t:明细" and table["state"] == "deterministic" @@ -117,7 +118,7 @@ def test_streaming_renders_chinese_names_like_the_full_doc(catalog): streamed = "".join([renderer.feed("本月[[t:明"), renderer.feed("细]]的[[c:明细.金"), renderer.feed("额]]上升。"), renderer.flush()]) assert streamed == render_markers(text, catalog, loader=loader) == compose(text, catalog)["markdown"] \ - == "本月明细的明细.金额上升。" + == "本月明细的金额上升。" @pytest.mark.parametrize("ref", [ diff --git a/backend/tests/test_report_entity_short.py b/backend/tests/test_report_entity_short.py new file mode 100644 index 00000000..04a7cb32 --- /dev/null +++ b/backend/tests/test_report_entity_short.py @@ -0,0 +1,90 @@ +"""字段标记紧跟在同一张表名后面时只显示字段名(engine/evidence._short_column)。 + +部署上撞过「取自 某表 的 某表.某列」:[[c:表.列]] 渲染成「表.列」全名,写作者前面又写了「某表 的」。现在:标记前面紧挨着 +(隔空白、「的」「中的」「里的」)刚写过同一张表的名字(写成文字或 [[t:同一张表]] 都算)时,只显示列名;引用、出处、证据 +标识照旧是那个字段,引用上记 short: true,复核照它核对显示的字。没有这一项的老文档照旧按全名核对。写作规则也写一句。 +表名、列名都是虚构的。 +""" +from __future__ import annotations + +import copy + +import pytest + +from app.engine.evidence import ENTITY_RULES, StreamRenderer, build_catalog, compose_doc, iter_segments, \ + render_markers, verify_doc + +SCHEMA_ART = "6" * 64 +SCHEMA = {"kind": "schema", "node_id": "fetch", "exec": 1, "artifact": SCHEMA_ART, "source": "plant", + "tables": {"产量表": ["工厂", "产量"], "产线表": ["工厂", "产线"], "plant_output": ["factory", "qty"]}} +QUERY = {"kind": "query", "node_id": "fetch", "exec": 1, "artifact": "a" * 64, "tool": "db_query__plant", + "source": "plant", "columns": ["工厂", "产量"], "rows": 1, "truncated": False, "schema_artifact": SCHEMA_ART, + "tables": ["产量表"]} + + +@pytest.fixture +def catalog(): + return build_catalog(nodes={}, ledger=[QUERY, SCHEMA]) + + +def column_segments(doc): + return [s for s in iter_segments(doc) if s.get("ref", "").startswith("c:")] + + +@pytest.mark.parametrize("raw, shown", [ + ("工厂取自[[t:产量表]]的[[c:产量表.工厂]]。", "工厂取自产量表的工厂。"), + ("工厂取自 产量表 的 [[c:产量表.工厂]]。", "工厂取自 产量表 的 工厂。"), # 表名写成文字也算 + ("[[t:产量表]]中的[[c:产量表.产量]]按工厂汇总。", "产量表中的产量按工厂汇总。"), + ("产量表里的 [[c:产量表.产量]] 按工厂汇总。", "产量表里的 产量 按工厂汇总。"), + ("[[t:plant_output]] 的 [[c:plant_output.qty]]。", "plant_output 的 qty。"), + ("[[t:PLANT_OUTPUT]] [[c:plant_output.qty]]。", "PLANT_OUTPUT qty。"), # 只隔空白;英文名不分大小写 +]) +def test_a_column_right_after_its_table_shows_only_the_column(catalog, raw, shown): + doc = compose_doc(raw, catalog) + assert doc["markdown"] == shown and render_markers(raw, catalog) == shown + [seg] = column_segments(doc) + assert seg["state"] == "deterministic" and seg["cite"]["short"] is True + assert seg["cite"]["alias"] == f"c:{seg['ref'][2:]}" or seg["cite"]["alias"].lower() == seg["ref"].lower() + assert verify_doc(doc, catalog)["ok"], verify_doc(doc, catalog)["violations"] + + +@pytest.mark.parametrize("raw, shown", [ + ("工厂取自[[t:产线表]]的[[c:产量表.工厂]]。", "工厂取自产线表的产量表.工厂。"), # 前面是另一张表 + ("[[c:产量表.工厂]]取自产量表。", "产量表.工厂取自产量表。"), # 前面没有表名 + ("产量表,[[c:产量表.工厂]]。", "产量表,产量表.工厂。"), # 隔着逗号:不是紧挨着 + ("myplant_output 的 [[c:plant_output.qty]]。", "myplant_output 的 plant_output.qty。"), # 英文名前面还连着字母 +]) +def test_other_columns_keep_the_full_name(catalog, raw, shown): + doc = compose_doc(raw, catalog) + assert doc["markdown"] == shown and render_markers(raw, catalog) == shown + assert not any(s["cite"].get("short") for s in column_segments(doc)) + + +def test_a_document_written_before_still_verifies_with_the_full_name(catalog): + """改动前组装的文档:字段照全名显示、引用上没有 short。照全名核对,不判成被改过。""" + doc = compose_doc("工厂取自[[t:产量表]]的[[c:产量表.工厂]]。", catalog) + old = copy.deepcopy(doc) + [seg] = column_segments(old) + start = seg["span"][0] + old["markdown"] = old["markdown"].replace("的工厂。", "的产量表.工厂。") + seg["text"], seg["span"] = "产量表.工厂", [start, start + len("产量表.工厂")] + seg["cite"].pop("short") + seg["cite"]["rendered"] = "产量表.工厂" + tail = next(s for s in iter_segments(old) if s["span"][0] == start + len("工厂")) + tail["span"] = [start + len("产量表.工厂"), start + len("产量表.工厂") + len(tail["text"])] + for unit in (u for b in old["blocks"] for u in b["units"]): + unit["span"] = [unit["span"][0], unit["span"][1] + len("产量表.")] + assert verify_doc(old, catalog)["ok"], verify_doc(old, catalog)["violations"] + + +@pytest.mark.parametrize("size", [1, 2, 3, 5, 8, 40]) +def test_the_stream_shows_the_same(catalog, size): + raw = "工厂取自[[t:产量表]]的[[c:产量表.工厂]],[[t:产线表]]的[[c:产量表.工厂]]不同。" + renderer = StreamRenderer(catalog) + out = [renderer.feed(raw[i:i + size]) for i in range(0, len(raw), size)] + [renderer.flush()] + assert "".join(out) == render_markers(raw, catalog) == compose_doc(raw, catalog)["markdown"] \ + == "工厂取自产量表的工厂,产线表的产量表.工厂不同。" + + +def test_the_writer_is_told(): + assert "只显示字段名" in ENTITY_RULES