diff --git a/backend/app/engine/evidence.py b/backend/app/engine/evidence.py index ecd3d662..f501da08 100644 --- a/backend/app/engine/evidence.py +++ b/backend/app/engine/evidence.py @@ -1639,8 +1639,76 @@ def resolve_ref(ref: str, catalog: dict[str, Any], *, cells_allowed: bool = True return resolve_marker(_parse_ref(ref), catalog, cells_allowed=cells_allowed, loader=loader) +#: 数字标记后面重复写的单位：最多隔几个空格、单位最长几个字。流式渲染放出一个数字标记之前， +#: 要等到它后面至少有 UNIT_LOOKAHEAD 个字（或者一个换行）才能判断，多出来的 1 是单位后面那个字 +_UNIT_GAP = 2 +_UNIT_MAX = 10 +UNIT_LOOKAHEAD = _UNIT_GAP + _UNIT_MAX + 1 +_UNIT_KINDS = frozenset({"m", "i", "v"}) +#: 渲染结果末尾的单位：最后一个数字后面的字（12,345人次 → 人次，4.57万元 → 万元，12.5% → %） +_UNIT_TAIL = re.compile(r"\d([^\d\s][^\d]*)$") +_BLANK = " \t\u3000" +_SEE_RUN = re.compile(r"\[\[" + _MARKER_GUARD + r"[ \t]*see[ \t]*:[^\[\]\n]*?\]\]" + r"(?:[" + _BLANK + r"]*\[\[" + _MARKER_GUARD + r"[ \t]*see[ \t]*:[^\[\]\n]*?\]\])*") + + +def _rendered_unit(marker: dict[str, Any], catalog: dict[str, Any], snaps: _Snapshots, + cells_allowed: bool) -> str: + cite = resolve_marker(marker, catalog, cells_allowed=cells_allowed, loader=snaps) + if cite["status"] != "resolved" or not is_number(cite.get("value")): + return "" + tail = _UNIT_TAIL.search(str(cite.get("rendered") or "")) + return tail.group(1) if tail and len(tail.group(1)) <= _UNIT_MAX else "" + + +def _tidy(text: str, catalog: dict[str, Any], snaps: _Snapshots, cells_allowed: bool, prev: str = "") -> str: + """渲染前收拾两处排版，数和引用都不动： + + - 数字标记后面重复写的单位去掉。[[m:visits]] 渲染出来已经带单位（12,345人次），写作者常常 + 还跟着写一遍（[[m:visits]] 人次），读者看到的是「12,345人次 人次」。单字的单位后面紧跟着字时 + 不动：写作者写的可能是另一个单位（口径卡是「人」，正文写「人次」），硬删会把不一致藏起来； + - [[see:…]] 前面的空白去掉。see 渲染成空，「最高月份 [[see:Q2]]。」会剩下「最高月份 。」。 + see 后面紧跟着字、要靠这个空白隔开的（「A [[see:Q1]]B」）留着；行首的缩进不动。 + + prev 是这段文字前面那个字（流式渲染分块时用），判断空白是不是在行首。 + """ + cuts: list[tuple[int, int]] = [] + for marker in parse_markers(text): + if marker["kind"] not in _UNIT_KINDS: + continue + unit = _rendered_unit(marker, catalog, snaps, cells_allowed) + if not unit: + continue + hit = re.compile(f"[{_BLANK}]{{0,{_UNIT_GAP}}}" + re.escape(unit)).match(text, marker["end"]) + if hit: + after = text[hit.end():hit.end() + 1] + if len(unit) > 1 or not after.isalnum(): + cuts.append((marker["end"], hit.end())) + for run in _SEE_RUN.finditer(text): + start = run.start() + while start > 0 and text[start - 1] in _BLANK: + start -= 1 + before = text[start - 1] if start > 0 else prev + after = text[run.end():run.end() + 1] + # 几个 see 之间的空白也一样渲染成多余的空格 + parts = list(MARKER_RE.finditer(text, run.start(), run.end())) + cuts.extend((m.end(), n.start()) for m, n in zip(parts, parts[1:]) if m.end() < n.start()) + if start < run.start() and before not in ("", "\n") and not (after.isalnum() or after == "["): + cuts.append((start, run.start())) + if not cuts: + return text + out, last = [], 0 + for a, b in sorted(cuts): + if a < last: + continue + out.append(text[last:a]) + last = b + out.append(text[last:]) + return "".join(out) + + def _render(text: str, catalog: dict[str, Any], snaps: _Snapshots, cells_allowed: bool, prev: str = "") -> str: - text = _expand_tables(text, catalog, snaps, cells_allowed, prev) + text = _tidy(_expand_tables(text, catalog, snaps, cells_allowed, prev), catalog, snaps, cells_allowed, prev) out: list[str] = [] last = 0 for marker in parse_markers(text): @@ -1662,7 +1730,11 @@ def render_markers(text: str, catalog: dict[str, Any], *, cells_allowed: bool = # 流式渲染：让 llm.token 里出现的就是最终的数字 # -------------------------------------------------------------------------- -def _safe_cut(text: str) -> int: +#: 文字末尾的空白和 [[see:…]]：流式渲染先攒着，等后面的字来了再一起交给 _tidy +_TRAILING_SEE = re.compile(r"(?:[" + _BLANK + r"]|\[\[" + _MARKER_GUARD + r"[ \t]*see[ \t]*:[^\[\]\n]*?\]\])+$") + + +def _safe_cut(text: str, has_unit: Callable[[dict[str, Any]], bool] | None = None) -> int: """能放出去的前缀有多长：最后一个还可能长成标记的 [[ 之前都能放。 一个标记最长 MARKER_MAX 个字，所以没闭合的时候最多攒 MARKER_MAX - 1 = _HOLD_LIMIT 个字； @@ -1670,6 +1742,9 @@ def _safe_cut(text: str) -> int: 开头可能是另一个 [。 整表标记闭合了也先不放：它展开时要看紧跟在后面的那个字（是不是换行），那个字还没到。 + _tidy 要往后看的也先不放：末尾的空白和 [[see:…]]（后面来的字决定空白去不去掉）， + 以及后面还没攒够 UNIT_LOOKAHEAD 个字的数字标记（后面可能是重复的单位）。has_unit 判断一个 + 标记渲染出来带不带单位，不带的（运行输入里的周次、没有单位的格）不用攒；不给就都攒。 """ cut = len(text) start = text.rfind("[[") @@ -1679,12 +1754,23 @@ def _safe_cut(text: str) -> int: cut = start if cut == len(text) and text.endswith("["): cut = len(text) - 1 - while (start := text.rfind("[[", 0, cut)) != -1: - m = MARKER_RE.match(text, start) - if not (m and m.end() == cut and m.group("kind") == "table"): - break - cut = start - return cut + while True: + held = cut + start = text.rfind("[[", 0, held) + m = MARKER_RE.match(text, start) if start != -1 else None + if m and m.end() == held and m.group("kind") == "table": + held = start + if (trail := _TRAILING_SEE.search(text, 0, held)) and trail.start() < held: + held = trail.start() + numeric = [m for m in MARKER_RE.finditer(text, 0, held) if m.group("kind") in _UNIT_KINDS + and (has_unit is None or has_unit(parse_markers(m.group(0))[0]))] + if numeric and len(text[numeric[-1].end():held]) < UNIT_LOOKAHEAD \ + and "\n" not in text[numeric[-1].end():held]: + held = numeric[-1].start() + if held == cut: + return cut + cut = held + class StreamRenderer: @@ -1709,6 +1795,9 @@ class StreamRenderer: self.pending = "" self.prev = "" + def _has_unit(self, marker: dict[str, Any]) -> bool: + return bool(_rendered_unit(marker, self.catalog, self.snaps, self.cells_allowed)) + def _release(self, ready: str) -> str: if not ready: return "" @@ -1718,7 +1807,7 @@ class StreamRenderer: def feed(self, delta: str) -> str: self.pending += delta or "" - cut = _safe_cut(self.pending) + cut = _safe_cut(self.pending, self._has_unit) ready, self.pending = self.pending[:cut], self.pending[cut:] return self._release(ready) @@ -2070,11 +2159,20 @@ def _context(text: str, start: int, end: int) -> str: #: 粗体：**…** 或 __…__，里面可以有标记（标记里的下划线不算收尾，[[m:refund_rate]] 很常见） _BOLD = re.compile(r"(\*\*|__)(?P(?:\[\[" + _MARKER_GUARD + r"[^\[\]\n]*?\]\]|(?!\1)[^\n])+?)\1") +#: 方向词、因果词：没有数字也没有引用的句子，含这些词才算结论。「同比」「环比」不在里面：它们只说明 +#: 跟谁比，本身不是涨跌；真正的比较结论还会带「增长」「下降」或者数字 _DIRECTIONAL = re.compile( r"增长|增加|上升|提升|提高|上涨|下降|下滑|减少|降低|回落|下跌|反弹|超过|高于|低于|领先|落后|最高|最低" - r"|最多|最少|来自|导致|因为|由于|带动|拉动|拖累|驱动|原因|归因|贡献|占比|环比|同比|翻倍|持平|波动" + r"|最多|最少|来自|导致|因为|由于|带动|拉动|拖累|驱动|原因|归因|贡献|占比|翻倍|持平|波动" r"|(? tuple[str, list[tuple[int, int]]]: @@ -2211,12 +2309,16 @@ def _classify(kind: str | None, segments: list[dict[str, Any]], see: list[dict[s 因果词的算结论；其余的是连接性的话。 自动链接的名字按文字算：它只是把正文重新切了一刀，句子算不算结论和开没开实体层无关。 + + 没有数字、没有引用的定义句（「环比是将本期与上月对比，反映短期波动」）也算连接性的话： + 它在解释概念，方向词只是定义的一部分。带因果词的不算定义句，照旧是结论。 """ if kind: return kind if see or any(s["kind"] not in ("text", "structural") and not _auto_entity(s) for s in segments): return "claim" - if any((s["kind"] == "text" or _auto_entity(s)) and _DIRECTIONAL.search(s["text"]) for s in segments): + words = "".join(s["text"] for s in segments if s["kind"] == "text" or _auto_entity(s)) + if _DIRECTIONAL.search(words) and not (_DEFINITION.search(words) and not _CAUSAL.search(words)): return "claim" return "connective" @@ -2243,7 +2345,8 @@ def compose_doc( """ source = (raw or "").replace("\r\n", "\n").replace("\r", "\n").strip() snaps = _Snapshots(loader, entities=entities) - text, strong = _normalize_strong(_expand_tables(source, catalog, snaps, cells_allowed)) + text, strong = _normalize_strong(_tidy(_expand_tables(source, catalog, snaps, cells_allowed), + catalog, snaps, cells_allowed)) markers = parse_markers(text) for marker in markers: marker["strong"] = any(a <= marker["start"] and marker["end"] <= b for a, b in strong) diff --git a/backend/tests/test_evidence_tidy_claims.py b/backend/tests/test_evidence_tidy_claims.py new file mode 100644 index 00000000..2bae8038 --- /dev/null +++ b/backend/tests/test_evidence_tidy_claims.py @@ -0,0 +1,222 @@ +"""结论句的判定，以及渲染时收拾的两处排版。 + +一次真实运行降档出具：唯一的缺口是一句解释同比、环比是什么意思的话（「同比是将本期与上年同月对比…… +反映月度之间的短期波动」），里面有「同比」「环比」「波动」三个方向词，没挂依据就被算成缺依据的结论句。 +同一份报告里每个数字都显示成「12,345人次 人次」（指标渲染已带单位，写作者又写了一遍）， +「最高月份 [[see:Q2]]。」去掉 see 后剩下「最高月份 。」。 +""" +from __future__ import annotations + +import random + +import pytest + +from app.engine.evidence import ( + UNIT_LOOKAHEAD, + StreamRenderer, + build_catalog, + compose_doc, + iter_segments, + render_markers, + verify_doc, +) + +ART = "b" * 64 + + +def metric(mid, name, value, unit="", decimals=None, fmt="thousands"): + return {"id": mid, "name": name, "unit": unit, "value": value, "decimals": decimals, + "format": fmt, "status": "ok" if value is not None else "missing_input", "expression": "…"} + + +CARD = { + "kind": "metric_set", "caliber": "客流口径", "caliber_version": "v1", "artifact": ART, + "metrics": [ + metric("visits", "客流量", 12345, "人次"), + metric("heads", "到访人数", 1200, "人"), + metric("gmv", "销售额", 45678.5, "元", 1), + metric("yoy", "同比增幅", 12.5, "%", 1, "plain"), + metric("gone", "缺数", None, "人次"), + ], +} +LEDGER = [{"kind": "metric_set", "node_id": "cal", "exec": 1, "artifact": ART, "caliber": "客流口径", + "version": "v1", "metrics": [m["id"] for m in CARD["metrics"]]}] + + +@pytest.fixture +def catalog(): + return build_catalog(nodes={"cal": CARD}, ledger=LEDGER, inputs={"week": "2026-W37"}) + + +def units(doc): + return [(u["kind"], "".join(s["text"] for s in u["segments"] if s["kind"] != "structural")) + for b in doc["blocks"] for u in b["units"]] + + +def kind_of(doc, fragment): + return next(k for k, text in units(doc) if fragment in text) + + +# -------------------------------------------------------------------------- +# 结论句 +# -------------------------------------------------------------------------- + +DEFINITION = ("同比与环比的比较基准不同：**同比**是将本期与上年同月对比（本期为 2026 年 9 月，基期为 2025 年 9 月），" + "剔除季节性因素，反映年度维度的变化；**环比**是将本期与紧邻的上月对比（基期为 2026 年 8 月），" + "反映月度之间的短期波动。") + + +def test_the_definition_that_downgraded_a_real_report_is_not_a_claim(catalog): + doc = compose_doc(f"本期客流 [[m:visits]]。[[see:m:visits]]\n\n{DEFINITION}两者的差异仅在基期。", catalog) + assert kind_of(doc, "比较基准不同") == "connective" + assert doc["stats"]["uncited_claims"] == 0 + assert verify_doc(doc, catalog)["violations"] == [] + + +@pytest.mark.parametrize("sentence", [ + "同比与环比使用同一套汇总口径。", # 只提到同比、环比：说的是跟谁比，不是涨跌 + "增长率是指本期较上期增长的部分占上期的比例。", + "客单价的计算方式为销售额除以订单数，订单越多摊得越低。", +]) +def test_sentences_without_a_conclusion_are_connective(catalog, sentence): + assert kind_of(compose_doc(sentence, catalog), sentence[:6]) == "connective" + + +@pytest.mark.parametrize("sentence", [ + "本期客流同比增长，环比回落。", + "环比波动明显加大。", + "增长主要是将促销提前带来的。", # 有「是将」，但说的是原因 + "同比增幅是指增长的比例，这次的增长由于新客首单。", +]) +def test_real_conclusions_stay_claims(catalog, sentence): + assert kind_of(compose_doc(sentence, catalog), sentence[:6]) == "claim" + + +def test_a_definition_with_a_number_is_still_a_claim(catalog): + doc = compose_doc("同比是将本期 [[m:visits]] 与上年同月对比。", catalog) + assert kind_of(doc, "同比是将") == "claim" + + +# -------------------------------------------------------------------------- +# 重复的单位 +# -------------------------------------------------------------------------- + +@pytest.mark.parametrize(("raw", "shown"), [ + ("客流 [[m:visits]] 人次，创新高。", "客流 12,345人次，创新高。"), + ("客流 [[m:visits]]人次。", "客流 12,345人次。"), + ("销售额 [[m:gmv]] 元，", "销售额 45,678.5元，"), + ("销售额 [[m:gmv|万]] 万元。", "销售额 4.57万元。"), + ("同比 [[m:yoy]] %。", "同比 12.5%。"), + ("客流 [[m:visits]] 人次的水平", "客流 12,345人次的水平"), # 多字单位后面跟着字，照样去掉 +]) +def test_a_repeated_unit_is_dropped(catalog, raw, shown): + doc = compose_doc(raw, catalog) + assert doc["markdown"] == shown + assert render_markers(raw, catalog) == shown + assert doc["stats"]["numbers_cited"] == 1 + assert verify_doc(doc, catalog)["violations"] == [] + + +@pytest.mark.parametrize(("raw", "shown"), [ + # 口径卡的单位是「人」，写作者写的是「人次」：不是同一个单位，不动，让不一致露出来 + ("到访 [[m:heads]] 人次。", "到访 1,200人 人次。"), + ("销售额 [[m:gmv]] 美元。", "销售额 45,678.5元 美元。"), + ("客流 [[m:visits]] 人次。", "客流 12,345人次 人次。"), # 隔了三个空格：不是紧跟着的 + ("缺数 [[m:gone]] 人次。", "缺数 — 人次。"), # 没有值，渲染成「—」，没有单位可比 +]) +def test_other_text_after_a_number_is_left_alone(catalog, raw, shown): + assert compose_doc(raw, catalog)["markdown"] == shown + assert render_markers(raw, catalog) == shown + + +def test_the_segment_still_carries_the_full_rendered_value(catalog): + doc = compose_doc("客流 [[m:visits]] 人次。[[see:m:visits]]", catalog) + number = next(s for s in iter_segments(doc) if s["kind"] == "number") + assert number["text"] == "12,345人次" and number["ref"] == "m:visits" + + +# -------------------------------------------------------------------------- +# see 前面的空白 +# -------------------------------------------------------------------------- + +@pytest.mark.parametrize(("raw", "shown"), [ + ("为最高月份 [[see:m:visits]]。", "为最高月份。"), + ("增长 [[see:m:visits]] [[see:m:yoy]]，", "增长，"), + ("前一句 [[see:m:visits]] 后一句", "前一句 后一句"), + ("前一句 [[see:m:visits]]后一句", "前一句 后一句"), # see 后面紧跟着字：留一个空格隔开 + ("末尾 [[see:m:visits]]", "末尾"), + ("| 客流 | 值 [[see:m:visits]] |", "| 客流 | 值 |"), +]) +def test_blank_before_see_is_dropped(catalog, raw, shown): + assert render_markers(raw, catalog) == shown + + +def test_indentation_before_see_is_kept(catalog): + raw = "- 第一条\n [[see:m:visits]] 续行" + assert render_markers(raw, catalog) == "- 第一条\n 续行" + + +def test_the_report_from_the_downgraded_run(catalog): + raw = ("2026 年 9 月，客流量为 [[m:visits]] 人次，为本次统计区间内的最高月份 [[see:m:visits]]。\n\n" + f"{DEFINITION}\n\n" + "**同比**：本期 [[m:visits]] 人次，同比增幅 [[m:yoy]] [[see:m:visits,m:yoy]]。") + doc = compose_doc(raw, catalog) + assert doc["markdown"] == ( + "2026 年 9 月，客流量为 12,345人次，为本次统计区间内的最高月份。\n\n" + f"{DEFINITION}\n\n" + "**同比**：本期 12,345人次，同比增幅 12.5%。") + assert doc["stats"]["uncited_claims"] == 0 and doc["stats"]["numbers_cited"] == 3 + assert verify_doc(doc, catalog)["violations"] == [] + + +# -------------------------------------------------------------------------- +# 流式渲染：分块输出拼起来仍然等于整篇渲染 +# -------------------------------------------------------------------------- + +def _stream(catalog, raw, sizes): + renderer, out, i = StreamRenderer(catalog), [], 0 + for size in sizes: + if i >= len(raw): + break + out.append(renderer.feed(raw[i:i + size])) + i += size + if i < len(raw): + out.append(renderer.feed(raw[i:])) + out.append(renderer.flush()) + return out + + +@pytest.mark.parametrize("size", [1, 2, 3, 5, 8, UNIT_LOOKAHEAD, 40]) +def test_stream_drops_the_repeated_unit_and_the_blank_like_the_full_render(catalog, size): + raw = "客流 [[m:visits]] 人次，为最高月份 [[see:m:visits]]。到访 [[m:heads]] 人次 [[see:m:heads]]\n末尾 [[m:gmv]] 元" + full = render_markers(raw, catalog) + assert full == "客流 12,345人次，为最高月份。到访 1,200人 人次\n末尾 45,678.5元" + out = _stream(catalog, raw, [size] * len(raw)) + assert "".join(out) == full, out + + +@pytest.mark.parametrize("raw", [ + "为最高月份 [[see:m:visits]]。下一句", + "前一句 [[see:m:visits]]后一句", + "增长 [[see:m:visits]] [[see:m:yoy]]\n下一行", +]) +@pytest.mark.parametrize("size", range(1, 9)) +def test_stream_holds_the_blank_before_see_until_it_knows_what_follows(catalog, raw, size): + # 没有数字标记帮忙攒着：空白和 see 得自己攒到后面的字来了再放 + assert "".join(_stream(catalog, raw, [size] * len(raw))) == render_markers(raw, catalog) + + +def _corpus(n): + rng = random.Random(20260930) + pieces = ["[[m:visits]]", "[[m:gmv|万]]", "[[m:heads]]", "[[m:yoy]]", "[[see:m:visits]]", "[[see:m:yoy]]", + "人次", "人", "次", "万元", "元", "%", " ", " ", "　", "\n", "。", "，", "文字", "[[", "]]", "["] + return ["".join(rng.choice(pieces) for _ in range(rng.randint(1, 16))) for _ in range(n)] + + +@pytest.mark.parametrize("raw", _corpus(150)) +def test_stream_matches_the_full_render_on_random_input(catalog, raw): + rng = random.Random(raw) + full = render_markers(raw, catalog) + for _ in range(3): + sizes = [rng.randint(1, 20) for _ in range(len(raw))] + assert "".join(_stream(catalog, raw, sizes)) == full diff --git a/backend/tests/test_report_node.py b/backend/tests/test_report_node.py index bad7121d..cfb1ba99 100644 --- a/backend/tests/test_report_node.py +++ b/backend/tests/test_report_node.py @@ -257,8 +257,9 @@ async def test_a_stream_that_breaks_mid_marker_leaks_nothing(monkeypatch): assert [e.data["code"] for e in await events(row.id, "log") if e.data.get("code") == "stream_fallback"] \ == ["stream_fallback"] streamed = "".join(t.data["delta"] for t in tokens if t.node_id == "write") - # 断之前放出去的是渲染好的字；半截标记一个字符都不能出去 - assert streamed == "本周（2026-W37）销售额 ", streamed + # 断之前放出去的是渲染好的字；半截标记一个字符都不能出去。末尾的空格也先攒着： + # 后面要是 [[see:…]]，渲染时这个空格要去掉 + assert streamed == "本周（2026-W37）销售额", streamed checked, doc = await report_of(row.id) assert checked["ok"] is True and checked["repairs"] == 0 assert doc["markdown"] == "## 本周概览\n\n本周（2026-W37）销售额 45,678.5元，环比 8.7%；订单 1,234单。"