diff --git a/backend/app/bundle/verifier.js b/backend/app/bundle/verifier.js index ffca5f15..c760352b 100644 --- a/backend/app/bundle/verifier.js +++ b/backend/app/bundle/verifier.js @@ -615,11 +615,14 @@ /** * render_cell(shown(raw))：快照里的一格。kind 是查询时记下的列类型，display 是报告文档记下的单元格显示规则版本 - * （2 版起 ISO 日期时间的 T 换成空格）。DECIMAL 落成的数字文本按数渲染、不经 float；文字压成一行，竖线换成 ¦ + * （2 版起 ISO 日期时间的 T 换成空格；3 版起标识列的数不分千位）。plain 是引用上记下的「这一格按标识列显示」： + * 哪几列是标识列服务端按 SQL 和冻结的数据目录判（cell_format.plain_columns），本页解析不了 SQL，照引用上记的渲染—— + * 它只决定分不分千位，值本身照样逐位比对。DECIMAL 落成的数字文本按数渲染、不经 float；文字压成一行，竖线换成 ¦ */ - function renderCell(raw, token, conv, kind, display) { - if (display !== 1 && display !== 2) throw new Unshown(T.shownDisplay(display)) + function renderCell(raw, token, conv, kind, display, plain = false) { + if (display !== 1 && display !== 2 && display !== 3) throw new Unshown(T.shownDisplay(display)) if (!PY_NUMERIC || !ISO_DATETIME) throw new Unshown(T.shownOld) + const fmt = display >= 3 && plain === true ? 'plain' : 'thousands' let value = raw if (display >= 2 && typeof raw === 'string') { const m = ISO_DATETIME.exec(pyStrip(raw)) @@ -630,12 +633,12 @@ // Python 把全角、阿拉伯文这些数字也当数（int('1٢٣') 是 123），本页只照 ASCII 算 if (/[^\x00-\x7f]/.test(text)) throw new Unshown(T.shownDigits) if (!/^-?\d+$/.test(text) || text.replace('-', '').length <= INT_DIGITS) { - return renderNumeric(decimalOf(text), '', null, 'thousands', conv) + return renderNumeric(decimalOf(text), '', null, fmt, conv) } } // 对象、列表：服务端按 Python 的 json.dumps 写（键的先后、浮点的写法），本页没有移植 if (isObj(value) || Array.isArray(value)) throw new Unshown(T.shownShape) - return renderNumber(value, token, { conv }).replace(/\|/g, '¦') + return renderNumber(value, token, { conv, fmt }).replace(/\|/g, '¦') } /** @@ -1469,7 +1472,7 @@ if (!sameValue(cite.value, info.truth)) return { ...info, result: FAIL, reason: T.numCellDiffers } const display = Object.prototype.hasOwnProperty.call(doc, 'cell_display') ? doc.cell_display : 1 return checkShown(info, seg, () => renderCell(cell.raw, info.truthToken, convOf(cite), - columnKind(art.json, cell.column), display)) + columnKind(art.json, cell.column), display, own(cite, 'plain') === true)) } /** 引用上记的换算（|万、|pct 这类），没写是 null */ diff --git a/backend/app/engine/cell_format.py b/backend/app/engine/cell_format.py new file mode 100644 index 00000000..04dc22fc --- /dev/null +++ b/backend/app/engine/cell_format.py @@ -0,0 +1,183 @@ +"""查询结果里哪几列是标识列：报告里显示这几列的数不分千位（evidence.render_cell 的 plain，单元格显示规则 3 版起）。 + +部署上撞过：产线编码这类数值型的编码列（编码 1203）在报告的表格里写成「1,203」，读的人以为是个量。千分位只该给量用。 +按序看三样信号，第一样说得出的算数；都说不出的照旧分千位： + +1. 查询当时冻结的数据目录（查询快照的 schema_artifact 指向的表结构快照里的 catalog）给源列标的度量类型：identifier + （标识）、status（状态）、attribute（属性）不分千位；flow（流量）、stock（存量）、ratio（比率）是量，分千位； +2. 结果列取自的源列名像编码、编号：CODE、ID、NO 整个词或者在结尾（STATION_CODE、order_id、userNo），带「编码」「编号」 + 「代码」的； +3. 结果列是分组键（GROUP BY 里有它，写列、写别名、写序号都算），不是聚合。 + +SUM、COUNT、AVG 这类聚合出来的列是量，分千位，不看后面几样；MIN、MAX 取的是那一列本身的值，按那一列判。SQL 解析不了、 +结果列在 SQL 里找不到（SELECT *、UNION）的，只按结果列名本身和目录里唯一那张表判。 + +只看查询快照（SQL、列名）和它指向的表结构快照（都按内容寻址、不会变）：同一份快照什么时候判都是同一个结果，组装、 +流式渲染、出口复核一致。离线核对页（bundle/verifier.js）没法解析 SQL，照引用上记下的 plain 渲染（见 evidence._resolve_cell）。 +""" +from __future__ import annotations + +import re +from collections.abc import Callable, Mapping +from typing import Any + +from app.data.names import name_key + +#: 目录里不是量的度量类型：这些列的数是编号、码值、属性，不分千位 +PLAIN_MEASURES = frozenset({"identifier", "status", "attribute"}) +#: 目录里是量的度量类型 +COUNTED_MEASURES = frozenset({"flow", "stock", "ratio"}) +#: 列名像编码、编号：CODE / ID / NO 整个词、或者以 _CODE、_ID、_NO 结尾（不分大小写）；驼峰的 stationCode、orderId、userNo； +#: 带「编码」「编号」「代码」的 +_CODE_NAME = re.compile(r"(?i:(?:^|_)(?:code|id|no)$)|[a-z0-9](?:Code|Id|ID|No|NO)$|编码|编号|代码") +#: 解析 SQL 按这几种读法依次试（快照里没记是哪种库）：先通用读法，读不通再换，取第一种读得通的——同一条 SQL 结果固定 +_READINGS = (None, "mysql", "postgres", "oracle", "sqlite") + + +def code_like(name: Any) -> bool: + """列名像编码、编号（_CODE_NAME）。""" + return isinstance(name, str) and bool(_CODE_NAME.search(name.strip())) + + +def plain_columns(snapshot: Mapping[str, Any], schema: Callable[[str], Any] | None = None) -> frozenset[str]: + """查询快照里要按标识列显示（数不分千位）的结果列名（照快照 columns 的写法）。 + + schema 取表结构快照的内容（参数是 schema_artifact，取不到返回 None），用来读冻结的数据目录；不给就不看目录。 + 不抛异常：分析出了意外就当判断不了（照旧分千位）。""" + columns = [str(c) for c in snapshot.get("columns") or []] + if not columns: + return frozenset() + try: + catalog = _frozen_catalog(snapshot, schema) + select = _outer_select(snapshot.get("sql")) + return frozenset(c for c in columns if _plain(c, select, catalog)) + except Exception: # noqa: BLE001 - 判断不了就照旧分千位，不能让报告渲染崩掉 + return frozenset() + + +def _frozen_catalog(snapshot: Mapping[str, Any], schema: Callable[[str], Any] | None) -> dict[str, dict[str, str]]: + """冻结的数据目录里各列的度量类型：{表名的 name_key: {列名的 name_key: 度量类型}}。表名带库名的，最后一段也记一份。""" + artifact = snapshot.get("schema_artifact") + content = schema(artifact) if schema is not None and isinstance(artifact, str) and artifact else None + frozen = content.get("catalog") if isinstance(content, dict) else None + out: dict[str, dict[str, str]] = {} + for table, entry in (frozen or {}).items() if isinstance(frozen, dict) else (): + notes = entry.get("notes") if isinstance(entry, dict) else None + cols = notes.get("columns") if isinstance(notes, dict) else None + measures = {name_key(str(c)): str(item["measure"]["value"]) for c, item in (cols or {}).items() + if isinstance(item, dict) and isinstance(item.get("measure"), dict) + and isinstance(item["measure"].get("value"), str)} + if measures: + for key in {name_key(str(table)), name_key(str(table).rsplit(".", 1)[-1])}: + out.setdefault(key, {}).update(measures) + return out + + +def _outer_select(sql: Any) -> Any: + """最外层的 SELECT（sqlglot 的节点）；解析不了、最外层不是一个 SELECT（UNION 这类）的为 None。""" + from sqlglot import exp + + from app.data.sql_semantics import _parse + + for read in _READINGS: + root, _ = _parse(sql, read) + if root is not None: + return root if isinstance(root, exp.Select) else None + return None + + +def _plain(column: str, select: Any, catalog: dict[str, dict[str, str]]) -> bool: + from sqlglot import exp + + item = _projection(select, column) if select is not None else None + if item is None: + # SELECT *、UNION、解析不了：只看结果列名本身，目录按唯一那张表找 + tables = _tables(select) if select is not None else {} + measure = _measure(catalog, next(iter(tables.values())) if len(tables) == 1 else None, column) + return measure in PLAIN_MEASURES if measure else code_like(column) + value = _unwrap(item.unalias()) + if isinstance(value, exp.AggFunc) and not isinstance(value, (exp.Min, exp.Max, exp.AnyValue)): + return False # SUM / COUNT / AVG……：量 + if isinstance(value, (exp.Min, exp.Max, exp.AnyValue)): + value = _unwrap(value.this) + if isinstance(value, exp.Column): + tables = _tables(select) + table = tables.get(name_key(value.table)) if value.table else \ + (next(iter(tables.values())) if len(tables) == 1 else None) + measure = _measure(catalog, table, value.name) + if measure in PLAIN_MEASURES: + return True + if measure in COUNTED_MEASURES: + return False + if code_like(value.name): + return True + return _grouped(select, item) + + +def _unwrap(node: Any) -> Any: + from sqlglot import exp + + while isinstance(node, (exp.Paren, exp.Cast, exp.TryCast)): + node = node.this + return node + + +def _projection(select: Any, column: str) -> Any: + """结果列 column 对应的那一项选取（按别名或列名，先一字不差、再按 name_key）。找不到、或者对上不止一项的为 None。""" + items = [p for p in select.expressions if p.alias_or_name] + for same in (lambda a: a == column, lambda a: name_key(a) == name_key(column)): + hits = [p for p in items if same(p.alias_or_name)] + if hits: + return hits[0] if len(hits) == 1 else None + return None + + +def _tables(select: Any) -> dict[str, str]: + """这一层 FROM / JOIN 里的实表：{别名或表名的 name_key: 表名（带库名的照写）}。子查询、CTE 不算实表。""" + from sqlglot import exp + + with_ = select.args.get("with_") or select.args.get("with") + ctes = {name_key(c.alias_or_name) for c in with_.expressions} if with_ is not None else set() + sources = [(select.args.get("from_") or select.args.get("from"))] + sources += [j for j in select.args.get("joins") or []] + out: dict[str, str] = {} + for source in sources: + table = getattr(source, "this", None) + if not isinstance(table, exp.Table) or name_key(table.name) in ctes: + continue + name = ".".join(p for p in (table.db, table.name) if p) + out[name_key(table.alias_or_name)] = name + return out + + +def _measure(catalog: dict[str, dict[str, str]], table: str | None, column: str) -> str | None: + if not table or not catalog: + return None + for key in (name_key(table), name_key(table.rsplit(".", 1)[-1])): + if key in catalog: + return catalog[key].get(name_key(column)) + return None + + +def _grouped(select: Any, item: Any) -> bool: + """这一项选取是分组键：GROUP BY 里写了同样的表达式、它的别名，或者它的序号（GROUP BY 1）。""" + from sqlglot import exp + + group = select.args.get("group") + if group is None: + return False + value = _unwrap(item.unalias()) + position = next((k for k, p in enumerate(select.expressions, start=1) if p is item), None) + alias = name_key(item.alias) if item.alias else None + for key in group.expressions: + key = _unwrap(key) + if isinstance(key, exp.Literal) and key.is_int and int(key.this) == position: + return True + if alias and isinstance(key, exp.Column) and not key.table and name_key(key.name) == alias: + return True + if key == value: + return True + return False + + +__all__ = ["COUNTED_MEASURES", "PLAIN_MEASURES", "code_like", "plain_columns"] diff --git a/backend/app/engine/evidence.py b/backend/app/engine/evidence.py index 522894d0..5f3f7a8e 100644 --- a/backend/app/engine/evidence.py +++ b/backend/app/engine/evidence.py @@ -79,9 +79,11 @@ ENTITY_SYNTAXES = frozenset({1, 2}) #: eid、证据下钻都不变 #: - 3：整表标记 rows= 的结束行越界、起始行在范围内时，截到最后一行（以前整张表算解析不了）。截完照样展开成一格格的 #: 单元格引用、按格复核，所以这一条只影响组装；复核 3 版以前的文档时，没展开的整表标记照老规矩判，不然老文档里那句 -#: 「没有第 N 行」重新解析得了，会被当成文档被改过（render_mismatch） +#: 「没有第 N 行」重新解析得了，会被当成文档被改过（render_mismatch）。另外标识列（编码、编号、分组键，见 +#: cell_format.plain_columns）的数不分千位：编码 1203 不写成「1,203」。真有格子因此改了写法的文档记 cell_display: 3， +#: 引用上记 plain: true（离线核对页解析不了 SQL，照它渲染） #: 新组装的文档里真有格子因为显示规则改了写法时才记 cell_display（参与内容哈希），记的是渲染得出这些格子的最低一版 -#: （只有日期时间被改写的记 2：3 版只多了组装时的整表截行，不改格子的写法），没有的文档和以前一字不差。复核必须按 +#: （只有日期时间被改写的记 2：标识列没有因此改写法的，3 版的整表截行不改格子的写法），没有的文档和以前一字不差。复核必须按 #: 文档生成时的那一版：老文档存的是带 T 的字，拿新规则重新渲染对不上，会被当成文档被改过（render_mismatch） CELL_DISPLAY = 3 CELL_DISPLAYS = frozenset({1, 2, 3}) @@ -704,10 +706,11 @@ def render_metric(metric: dict[str, Any], conv: str | None = None) -> str: ) -def render_cell(value: Any, conv: str | None = None, *, kind: str | None = None) -> str: +def render_cell(value: Any, conv: str | None = None, *, kind: str | None = None, plain: bool = False) -> str: """查询快照里的一格渲染成字。 - - 数按千分位，小数位照值本来的样子（最多 10 位、去尾零）；DECIMAL 落盘后的数字文本 + - 数按千分位，小数位照值本来的样子（最多 10 位、去尾零）；plain 时不分组（标识列：编码 1203 不写成「1,203」， + 哪几列是标识列见 cell_format.plain_columns）。DECIMAL 落盘后的数字文本 （"45678.50"，小数位为 0 的 "45678"，规则见 expressions.cell_value）按数算，用 Decimal 渲染，不经 float 丢精度 - kind 是这一列查询时记下的类型：text 列按文本，VARCHAR 的 "2026" 不渲染成「2,026」； @@ -715,16 +718,17 @@ def render_cell(value: Any, conv: str | None = None, *, kind: str | None = None) - NULL 显示「—」：这是快照里的事实（那一格就是空的），不是缺值，更不是 0 - 文本压成一行，竖线换成 ¦：它常常落在表格里，原样的 | 会多切出一格 """ + fmt = "plain" if plain else DEFAULT_FORMAT if isinstance(value, str) and cell_value(value, kind) is not value: try: - return _render_numeric(Decimal(value.strip()), unit="", decimals=None, fmt=DEFAULT_FORMAT, conv=conv) + return _render_numeric(Decimal(value.strip()), unit="", decimals=None, fmt=fmt, conv=conv) except RenderError: raise except (ArithmeticError, ValueError) as e: raise RenderError(f"「{value}」无法按规则显示") from e if isinstance(value, (dict, list)): value = json.dumps(value, ensure_ascii=False, default=str) - return render_number(value, fmt=DEFAULT_FORMAT, conv=conv).replace("|", "¦") + return render_number(value, fmt=fmt, conv=conv).replace("|", "¦") # -------------------------------------------------------------------------- @@ -1321,6 +1325,19 @@ class _Snapshots: #: 这一次解析里的单元格要按哪一版显示规则才渲染得出来（有格子因为某一版的规则改了写法，就至少是那一版）： #: 组装时据此决定文档记不记、记哪一版 cell_display self.needed = 1 + #: 每份查询快照里的标识列（cell_format.plain_columns），按快照的工件 id 记 + self.plains: dict[str, frozenset[str]] = {} + + def plain(self, artifact: Any, snapshot: Any) -> frozenset[str]: + """这份查询快照里按标识列显示（数不分千位）的结果列。3 版以前的显示规则没有这一条，一律分千位。""" + if self.display < 3 or not isinstance(snapshot, dict): + return frozenset() + key = str(artifact or "") + if key not in self.plains: + from app.engine.cell_format import plain_columns + + self.plains[key] = plain_columns(snapshot, lambda a: self._fetch(a, "表结构快照")[0]) + return self.plains[key] def shown(self, raw: Any) -> Any: """按显示规则要显示的原值：2 版起 ISO 日期时间的 T 换成空格。没改就是原值本身。""" @@ -1343,6 +1360,7 @@ class _Snapshots: return self twin = _Snapshots(self.loader, entities=self.entities_on, syntax=syntax, display=display) twin.memo = self.memo + twin.plains = self.plains return twin def get(self, alias: str, artifact: Any) -> tuple[Any, str | None]: @@ -1441,7 +1459,14 @@ def _resolve_cell(marker: dict[str, Any], catalog: dict[str, Any], snaps: _Snaps try: raw, column = locate_cell(snapshot, parsed["row"], parsed["column"]) kind = column_kind(snapshot, column) - rendered = render_cell(snaps.shown(raw), conv, kind=kind) + shown = snaps.shown(raw) + rendered = render_cell(shown, conv, kind=kind) + plain = column in snaps.plain(entry.get("artifact"), snapshot) \ + and render_cell(shown, conv, kind=kind, plain=True) != rendered + if plain: + # 标识列（3 版起）：数不分千位。真改了写法才记——文档的 cell_display 记成 3，引用上记 plain，离线核对页照它渲染 + rendered = render_cell(shown, conv, kind=kind, plain=True) + snaps.needed = max(snaps.needed, 3) except CellError as e: return _unresolved(marker, alias, "cell", f"{alias} {e}") except RenderError as e: @@ -1452,6 +1477,8 @@ def _resolve_cell(marker: dict[str, Any], catalog: dict[str, Any], snaps: _Snaps "rendered": rendered} if conv: cite["conv"] = conv + if plain: + cite["plain"] = True return cite @@ -3636,6 +3663,7 @@ def _query_prompt(entry: dict[str, Any], snaps: _Snapshots, cells_allowed: bool, return lines columns = [str(c) for c in snapshot.get("columns") or []][:_PROMPT_COLS] rows = snapshot["rows"] + plain = snaps.plain(entry.get("artifact"), snapshot) example = None for r, record in enumerate(rows[:shown_rows]): cells = [] @@ -3643,7 +3671,8 @@ def _query_prompt(entry: dict[str, Any], snaps: _Snapshots, cells_allowed: bool, try: raw, _ = locate_cell(snapshot, r, c) kind = column_kind(snapshot, c) - shown, numeric = render_cell(snaps.shown(raw), kind=kind), is_number(cell_value(raw, kind)) + shown = render_cell(snaps.shown(raw), kind=kind, plain=c in plain) + numeric = is_number(cell_value(raw, kind)) except (CellError, RenderError): shown, numeric = MISSING, False cells.append(f"{c}={shown}") @@ -3798,7 +3827,8 @@ def _field_prompt(path: str, value: Any, ev: dict[str, Any], aliases: dict[str, if why: raise CellError(why) raw, column = locate_cell(snapshot, row, column) - shown = render_cell(snaps.shown(raw), kind=column_kind(snapshot, column)) + shown = render_cell(snaps.shown(raw), kind=column_kind(snapshot, column), + plain=column in snaps.plain(ev.get("artifact"), snapshot)) except (CellError, RenderError): try: shown = render_cell(value) diff --git a/backend/tests/test_bundle_shown.py b/backend/tests/test_bundle_shown.py index af720153..a29efb6f 100644 --- a/backend/tests/test_bundle_shown.py +++ b/backend/tests/test_bundle_shown.py @@ -60,7 +60,7 @@ function render(c) { try { const value = v.parseJson(c.json) const token = v.tokenAt(v.numberTokens(c.json), []) - if (c.cell) return { shown: v.renderCell(value, token, c.conv, c.kind, c.display) } + if (c.cell) return { shown: v.renderCell(value, token, c.conv, c.kind, c.display, c.plain) } const decimals = v.parseJson(c.decimals) return { shown: v.renderNumber(value, token, { unit: c.unit, decimals, decimalsToken: v.tokenAt(v.numberTokens(c.decimals), []), fmt: c.format, conv: c.conv }) } @@ -137,9 +137,10 @@ def py_number(value: Any, unit: str, decimals: Any, fmt: Any, conv: Any) -> str return None -def py_cell(raw: Any, conv: Any, kind: Any, display: int) -> str | None: +def py_cell(raw: Any, conv: Any, kind: Any, display: int, plain: bool = False) -> str | None: try: - return render_cell(_Snapshots(display=display).shown(raw), conv, kind=kind) + # 3 版起标识列的数不分千位：服务端按 SQL 判（cell_format），引用上记 plain，页面照它渲染 + return render_cell(_Snapshots(display=display).shown(raw), conv, kind=kind, plain=plain and display >= 3) except RenderError: return None @@ -190,9 +191,9 @@ def cell_cases() -> list[dict[str, Any]]: text = json.dumps(raw, ensure_ascii=False) for kind in CELL_KINDS: for conv in CELL_CONVS: - for display in (1, 2): + for display, plain in ((1, False), (2, False), (3, False), (3, True), (2, True)): cases.append({"cell": True, "json": text, "kind": kind, "conv": conv, "display": display, - "py": py_cell(json.loads(text), conv, kind, display)}) + "plain": plain, "py": py_cell(json.loads(text), conv, kind, display, plain)}) return cases @@ -331,7 +332,8 @@ def test_every_format_in_the_monthly_sample_passes_as_exported(): "m:conv_rate": "2.35%", "m:growth": "-8.2%", "m:orders": "12840单", "m:avg": "5.81元", "m:avg|.3": "5.811元", "m:avg|int": "6元", "m:share|pct": "34.6%", "m:export": "15,230.4美元", "v:Q1.r0.region": "华南", "v:Q1.r0.gmv": "45,678.5", "v:Q1.r0.gmv|万": "4.57万", "v:Q1.r1.region": "华东", - "v:Q1.r1.gmv": "-1,200.75", "v:Q1.r0.order_no": "202,609,071,234,567,890", + # 订单号是标识列（列名以 _no 结尾）：3 版显示规则起不分千位，超过 2^53 也一位不差 + "v:Q1.r1.gmv": "-1,200.75", "v:Q1.r0.order_no": "202609071234567890", "v:Q1.r0.paid_at": "2026-09-07 08:21:39", } @@ -353,8 +355,8 @@ def test_every_format_in_the_monthly_sample_passes_as_exported(): ("v:Q1.r0.gmv", "54,678.5", "45678.5", "45,678.5"), # DECIMAL 落成的数字文本 ("v:Q1.r1.gmv", "1,200.75", "-1200.75", "-1,200.75"), # 负数单元格 ("v:Q1.r0.gmv|万", "5.57万", "45678.5", "4.57万"), # 单元格换算成万 - ("v:Q1.r0.order_no", "202,609,071,234,567,891", "202609071234567890", - "202,609,071,234,567,890"), # 超过 2^53 的整数 + # 超过 2^53 的整数；标识列不分组，应显示的字和引用的值一样，原因里不另写 + ("v:Q1.r0.order_no", "202609071234567891", "202609071234567890", None), ("v:Q1.r0.paid_at", "2026-09-08 08:21:39", "2026-09-07T08:21:39", "2026-09-07 08:21:39"), # 日期时间 ]) def test_changing_only_the_shown_text_fails_and_names_both(ref, text, cited, expected): @@ -391,13 +393,27 @@ def test_a_format_the_page_does_not_know_cannot_pass(ref, cite, text): assert hit["result"] == "na" and hit["reason"].startswith("无法核对显示文字："), hit +def test_identifier_cells_read_without_separators_and_the_page_follows(): + """标识列（订单号这类编号，列名以 _no 结尾）的数不分千位：文档记 cell_display: 3、引用上记 plain，页面照它渲染、 + 原样通过；把 plain 去掉的伪造页，页面按千分位渲染，和正文显示的字对不上，不通过。""" + from tests.bundle_samples import formats_bundle + + bundle = formats_bundle() + art, doc = doc_of(bundle) + seg = segment(doc, "v:Q1.r0.order_no") + assert doc["cell_display"] == 3 and seg["text"] == "202609071234567890" and seg["cite"]["plain"] is True + assert item(verdicts(bundle), "v:Q1.r0.order_no")["result"] == "pass" + seg["cite"].pop("plain") + assert item(verdicts(reseal(bundle, art, doc)), "v:Q1.r0.order_no")["result"] == "fail" + + def test_an_unknown_cell_display_version_cannot_pass(): """文档记下的单元格显示规则版本认不出（服务端复核记 bad_schema）：单元格的显示文字判无法核对；口径卡的照常核对。""" from tests.bundle_samples import formats_bundle bundle = formats_bundle() art, doc = doc_of(bundle) - doc["cell_display"] = 3 + doc["cell_display"] = 4 got = verdicts(reseal(bundle, art, doc)) cells = [n for n in got["numbers"] if n["ref"].startswith("v:")] assert cells and all(n["result"] == "na" and n["reason"].startswith("无法核对显示文字：") for n in cells) @@ -521,7 +537,7 @@ def test_numeric_text_cells_show_their_own_digits_in_the_panel(): table["rows"][0][2] = "202609071234567891" got = verdicts(formats_bundle(table=table)) order = item(got, "v:Q1.r0.order_no") - assert order["result"] == "pass" and order["text"] == "202,609,071,234,567,891", order + assert order["result"] == "pass" and order["text"] == "202609071234567891", order assert order["truthText"] == "202609071234567891", order assert item(got, "v:Q1.r0.gmv")["truthText"] == "45678.50" # 不是数字文本的格子不另写原文：照旧按值写 diff --git a/backend/tests/test_report_identifier_cells.py b/backend/tests/test_report_identifier_cells.py new file mode 100644 index 00000000..22484941 --- /dev/null +++ b/backend/tests/test_report_identifier_cells.py @@ -0,0 +1,121 @@ +"""报告里标识列的数不分千位（engine/cell_format.plain_columns，单元格显示规则 3 版）。 + +部署上撞过：产线编码这类数值型的编码列在报告的表格里写成「1,203」，读的人以为是个量。现在按序看三样信号：查询当时冻结的 +数据目录给源列标的度量类型（identifier / status / attribute 不分，flow / stock / ratio 分）、源列名像编码编号（CODE、 +_ID、ID、NO、编码、编号、代码）、是分组键而不是聚合；SUM、COUNT、AVG 出来的是量，照旧分千位；判断不了的照旧分千位。 +真有格子因此改了写法的文档记 cell_display: 3、引用上记 plain；老文档按记下的版本复核，照旧分千位。数据是虚构的。 +""" +from __future__ import annotations + +import pytest + +from app.core import artifact_store +from app.engine import evidence as ev +from app.engine.cell_format import code_like, plain_columns +from app.engine.evidence import build_catalog, compose_doc, verify_doc, writer_catalog + +SCHEMA = {"source": "plant", "tables": {"plant_output": {"columns": []}}, + "catalog": {"plant_output": {"version": 2, "notes": {"columns": { + "batch": {"measure": {"value": "identifier", "source": "human", "status": "confirmed"}}, + "capacity": {"measure": {"value": "stock", "source": "human", "status": "confirmed"}}}}}}} +SQL = ("SELECT p.STATION_CODE AS 产线, YEAR(p.made_on) AS 年份, p.capacity AS 产能, SUM(p.qty) AS 产量, " + "COUNT(*) AS 工单数, MAX(p.batch) AS 批次 FROM plant_output p " + "GROUP BY p.STATION_CODE, YEAR(p.made_on), p.capacity") +COLUMNS = ["产线", "年份", "产能", "产量", "工单数", "批次"] + + +def _store(obj: dict) -> str: + text = artifact_store.canonical_json(obj) + artifact = artifact_store.content_hash(text) + path = artifact_store._path_of(artifact) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return artifact + + +def _snap(schema_artifact: str) -> dict: + return {"columns": COLUMNS, "column_types": {c: "number" for c in COLUMNS}, + "rows": [[1203, 2031, 7001, 15200, 1450, 88001], [1207, 2031, 6400, 9300, 1210, 88002]], + "row_count": 2, "truncated": False, "elapsed_ms": 2, "sql": SQL, "source": "plant", + "schema_artifact": schema_artifact} + + +AMOUNTS = {"columns": ["金额"], "column_types": {"金额": "number"}, "rows": [[12345]], "row_count": 1, + "truncated": False, "elapsed_ms": 1, "sql": "SELECT o.amount * 2 AS 金额 FROM orders o", "source": "plant"} + + +def _query(artifact: str, columns: list[str], call: str) -> dict: + return {"kind": "query", "node_id": "fetch", "exec": 1, "artifact": artifact, "via": "b" * 64, "call_id": call, + "tool": "db_query__plant", "source": "plant", "columns": columns, "rows": 2, "truncated": False} + + +@pytest.fixture +def catalog(): + snap = _store(_snap(_store(SCHEMA))) + ledger = [_query(snap, COLUMNS, "call_1"), _query(_store(AMOUNTS), ["金额"], "call_2")] + return build_catalog(nodes={}, ledger=ledger, inputs={}) + + +TEXT = ("[[v:Q1.r0.年份]] 年 [[v:Q1.r0.产线]] 线产量 [[v:Q1.r0.产量]]，工单 [[v:Q1.r0.工单数]] 张，产能 [[v:Q1.r0.产能]]，" + "批次 [[v:Q1.r0.批次]]；另一项金额 [[v:Q2.r0.金额]]。[[see:Q1,Q2]]") + + +def _shown(doc) -> dict[str, str]: + return {s["ref"]: s["text"] for b in doc["blocks"] for u in b["units"] for s in u["segments"] + if s.get("ref", "").startswith("v:")} + + +def test_identifier_columns_read_without_separators(catalog): + doc = compose_doc(TEXT, catalog) + assert doc["violations"] == [], doc["violations"] + shown = _shown(doc) + assert shown["v:Q1.r0.产线"] == "1203" # 源列名像编码（STATION_CODE） + assert shown["v:Q1.r0.年份"] == "2031" # 分组键 + assert shown["v:Q1.r0.批次"] == "88001" # 目录标的是标识（MAX 取的是那一列本身的值） + assert shown["v:Q1.r0.产能"] == "7,001" # 分组键，但目录标的是存量：是量，照旧分千位 + assert shown["v:Q1.r0.产量"] == "15,200" # SUM：量 + assert shown["v:Q1.r0.工单数"] == "1,450" # COUNT：量 + assert shown["v:Q2.r0.金额"] == "12,345" # 判断不了：照旧分千位 + assert doc["cell_display"] == 3 + plain = {s["ref"] for b in doc["blocks"] for u in b["units"] for s in u["segments"] + if (s.get("cite") or {}).get("plain")} + assert plain == {"v:Q1.r0.产线", "v:Q1.r0.年份", "v:Q1.r0.批次"} + assert verify_doc(doc, catalog)["ok"] + + +def test_tables_and_the_writer_catalog_read_the_same_way(catalog): + doc = compose_doc("[[table:Q1 cols=产线,产量 rows=0-1]]", catalog) + assert doc["violations"] == [] and "| 1203 | 15,200 |" in doc["markdown"] and "| 1207 | 9,300 |" in doc["markdown"] + text = writer_catalog(catalog).text + assert "产线=1203" in text and "产量=15,200" in text, text + + +def test_documents_composed_before_still_verify_with_separators(catalog, monkeypatch): + monkeypatch.setattr(ev, "CELL_DISPLAY", 2) + old = compose_doc(TEXT, catalog) + monkeypatch.undo() + assert _shown(old)["v:Q1.r0.产线"] == "1,203" and "cell_display" not in old + assert verify_doc(old, catalog)["ok"], "按文档记下的版本复核：照旧分千位，不判成被改过" + + +def test_a_document_edited_back_to_separators_is_caught(catalog): + doc = compose_doc("产线 [[v:Q1.r0.产线]]。", catalog) + seg = next(s for b in doc["blocks"] for u in b["units"] for s in u["segments"] if s.get("ref") == "v:Q1.r0.产线") + start = seg["span"][0] + doc["markdown"] = doc["markdown"].replace("1203", "1,203") + seg["text"], seg["span"] = "1,203", [start, start + 5] + assert "render_mismatch" in {v["code"] for v in verify_doc(doc, catalog)["violations"]} + + +@pytest.mark.parametrize("name, yes", [ + ("STATION_CODE", True), ("order_id", True), ("ID", True), ("user_no", True), ("orderId", True), ("stationCode", True), + ("产线编码", True), ("工单编号", True), ("物料代码", True), + ("qty", False), ("piano", False), ("video", False), ("income", False), ("产量", False), +]) +def test_code_like_names(name, yes): + assert code_like(name) is yes + + +def test_unparseable_sql_falls_back_to_the_column_name(): + snap = {"columns": ["station_id", "total"], "sql": "this is not sql", "rows": []} + assert plain_columns(snap) == frozenset({"station_id"}) diff --git a/scripts/check-bundle.mjs b/scripts/check-bundle.mjs index 6ac7b7b4..40c7ee30 100644 --- a/scripts/check-bundle.mjs +++ b/scripts/check-bundle.mjs @@ -864,7 +864,8 @@ await section('样本页：各种形状', async () => { check('formats：点开换算成万的数字，引用的值、口径卡里的值照原值写，注明与按显示规则渲染的结果一致', detail.includes('74613.65') && detail.includes('与按显示规则渲染的结果一致') && !detail.includes('按显示规则应为'), detail.replace(/\s+/g, ' ').slice(0, 160)) - const order = page.locator('button.num').filter({ hasText: '202,609,071,234,567,890' }) + // 订单号是标识列（列名以 _no 结尾）：3 版显示规则起不分千位 + const order = page.locator('button.num').filter({ hasText: /^202609071234567890$/ }) detail = '找不到订单号' let rowCell = '' if (await order.count()) {