Files
AgentLab/backend/app/chat/direct_evidence.py
T
ThearchyHelios eddeded2a4 全栈安全加固、性能提速与架构演进:连接池与沙箱止血、双路混合检索、多版本归档及前端分包治理
- 第一批(安全加固与生命周期止血):隔离测试宿主机密钥与 .env 泄漏;修复 SQLite 取消查询挂死、:memory: 连接池报错与 EngineCache 世代竞态;补齐运行与评测删除时的 Checkpointer 线程、Artifact 磁盘文件与无主附件回收;加固三套沙箱路径越界、进程组清理与 Seatbelt 数据目录隔离,封堵 SSRF IPv4-mapped IPv6 元数据绕过与响应体 OOM;修复推理模型多轮 reasoning_content、结构化输出 Token 漏计与缓存折扣计费
- 第二批(数据库/集群/问数提速与首屏分包):补齐核心表索引与 SQLite WAL 调优,列表接口按需 defer 大列,收敛启动扫描并打通跨进程 slot 槽位唤醒;优化跨源全局选表、哈希桶数字绑定、流式 CSV 导入及多方言分析算子;剥离首屏静态引入的 @xyflow/react 与全局样式(首屏主包降至 461.59 kB),补充流式 rAF 攒批、Store 精细订阅与表单无障碍绑定
- 第三批(架构演进与工程自动化闭环):记录库超挂接阈值历史版本自动增量归档至单库多表(解除 SQLite 10 库 ATTACH 硬上限);重构知识库为词面 BM25 与全集合归一化向量矩阵真·双路独立召回并支持游标分批重建;升级多方言感知 SQL 模板词法分析(E'...' / $$...$$ / q'[...]' / # / 嵌套注释)与 Decimal / bool / NaN 参数校验,消除 ledger 层对 api 路由层的反向依赖;拆分 3101 行 CaliberDrafts 页面为 7 个领域子模块,消除检查脚本本机路径硬编码并新增 GitHub Actions CI 门禁与三批回归测试套件
2026-11-13 14:51:36 +08:00

943 lines
38 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""快速直查(Fast Direct Ask)零延迟数字证据封装。
设计原则:
- 零额外模型调用(0 LLM calls,纯 Python 内存处理 < 5ms),不退回深度模式 25–30s 的二次报告重写;
- 对 Agent 单轮直出文本中的显式标记(`[[v:Q1.r0.col]]`、`[[table:Q1 ...]]`、`[[see:Q1]]`)做容错归一与遮罩守卫,
杜绝漏写 `.r0.`、列名大小写差异或未解析占位符 `⟦?...⟧` 暴露给用户;
- 对 Agent 未包裹标记但可唯一无歧义回指到未遮罩查询单元格的裸数字(含 Markdown 表格与正文),确定性升级为
`[[v:Qn.rR.col]]` 引用标记,交由 `compose_doc` 统一生成内容哈希封存的 `report_doc` 工件。
"""
from __future__ import annotations
from collections import Counter
from collections.abc import Mapping
from dataclasses import dataclass
from decimal import Decimal, InvalidOperation, ROUND_HALF_UP
import re
from typing import Any
from app.core.artifact_store import load as load_artifact
from app.engine.evidence import (
MARKER_MAX,
MARKER_RE,
TABLE_MAX_COLS,
_INLINE_SEE,
_SYNTAX_BLANK,
_structural_number,
parse_markers,
placeholder,
resolve_marker,
resolve_support,
)
from app.engine.expressions import cell_value, column_kind
from app.engine.issuance import _close, _tolerance, extract_numbers
MASKED_WORD = "已遮罩"
_CELL_REF_RE = re.compile(
r"^(?P<alias>Q\d+)\.(?:r(?P<row>\d+)|(?P<bare_row>\d+))\.(?P<col>.+)$",
re.I,
)
_CELL_NOROW_RE = re.compile(r"^(?P<alias>Q\d+)\.(?P<col>[^.]+)$", re.I)
_UNSAFE_COL_RE = re.compile(r"[\[\]|\n\r]")
_FENCE_LINE_RE = re.compile(r"^\s*(`{3,}|~{3,})")
_INLINE_CODE_RE = re.compile(r"`+[^`\n]*`+")
_MD_LINK_URL_RE = re.compile(r"\]\([^\)\n]*\)")
_TABLE_SEP_LINE_RE = re.compile(r"^\s*\|[\s:\-|]+\|\s*$")
_ZERO_ROW_COUNT_RE = re.compile(r"^\s*(?:行|条记录|条数据)")
_NUMERIC_CODE_TEXT_RE = re.compile(r"^[+-]?\d+(?:\.\d+)?$")
_LEADING_ZERO_INT_RE = re.compile(r"^[+-]?0\d+$")
_ID_LIKE_COL_SUFFIXES = ("_id", "_code", "_no", "_key", "_type", "_group", "id", "code", "type", "group")
_CN_MONTH_DAY_RE = re.compile(r"(?:\d{4}\s*年\s*)?\d{1,2}\s*月\s*\d{1,2}\s*[日号]?")
_FORMULA_MUL_DIV_PREFIX_RE = re.compile(r"[×*÷/]\s*$")
@dataclass(frozen=True)
class _CellCandidate:
alias: str
query_order: int
row: int
col: str
col_idx: int
dec_value: Decimal
row_keys: tuple[str, ...]
raw_text: str = ""
is_text_code: bool = False
@dataclass
class _QueryInfo:
alias: str
artifact: str
columns: list[str]
col_by_lower: dict[str, str]
rows: list[list[Any]]
kinds: list[str | None]
masked_lower: set[str]
truncated: bool = False
def _to_decimal(val: Any) -> Decimal | None:
if val is None or isinstance(val, bool):
return None
if isinstance(val, Decimal):
return val if val.is_finite() else None
if isinstance(val, int):
return Decimal(val)
if isinstance(val, float):
if val != val or val in (float("inf"), float("-inf")):
return None
return Decimal(str(val))
if isinstance(val, str):
s = val.strip()
if not s or s == MASKED_WORD:
return None
try:
d = Decimal(s)
return d if d.is_finite() else None
except InvalidOperation:
return None
return None
def _load_query_infos(
catalog: dict[str, Any],
masked_columns: dict[str, Any] | None = None,
loader: Any = None,
) -> dict[str, _QueryInfo]:
read = loader or load_artifact
infos: dict[str, _QueryInfo] = {}
for alias, entry in catalog.items():
if not isinstance(entry, dict) or entry.get("kind") != "query":
continue
artifact = str(entry.get("artifact") or "")
if not artifact:
continue
try:
snap = read(artifact)
except Exception: # noqa: BLE001
snap = None
if not isinstance(snap, dict):
continue
cols = [str(c) for c in (snap.get("columns") or [])]
raw_rows = snap.get("rows") or []
rows: list[list[Any]] = []
for r in raw_rows:
if isinstance(r, list):
rows.append(r)
elif isinstance(r, tuple):
rows.append(list(r))
elif isinstance(r, dict):
rows.append([r.get(c) for c in cols])
kinds = [column_kind(snap, c) for c in cols]
masked_set: set[str] = set()
for mc in snap.get("mask_columns") or []:
if mc:
masked_set.add(str(mc).lower())
if masked_columns and artifact in masked_columns:
extra = masked_columns[artifact]
for mc in (extra.keys() if isinstance(extra, dict) else extra):
if mc:
masked_set.add(str(mc).lower())
# 若单元格值在工具边界已被替换为「已遮罩」,整列按遮罩列保护
for j, c in enumerate(cols):
if any(j < len(r) and r[j] == MASKED_WORD for r in rows):
masked_set.add(c.lower())
infos[alias] = _QueryInfo(
alias=alias,
artifact=artifact,
columns=cols,
col_by_lower={c.lower(): c for c in cols},
rows=rows,
kinds=kinds,
masked_lower=masked_set,
truncated=bool(snap.get("truncated") or entry.get("truncated")),
)
return infos
def _fenced_line_ranges(text: str) -> list[tuple[int, int]]:
"""返回围栏代码块(```...``` / ~~~...~~~)覆盖的字符区间,代码块内不做标记改写与自动回绑。"""
ranges: list[tuple[int, int]] = []
in_fence = False
fence_char = ""
fence_len = 0
fence_start = 0
pos = 0
for line in text.splitlines(keepends=True):
m = _FENCE_LINE_RE.match(line)
if m:
run = m.group(1)
if not in_fence:
in_fence = True
fence_char = run[0]
fence_len = len(run)
fence_start = pos
elif run[0] == fence_char and len(run) >= fence_len and line[m.end():].strip() == "":
in_fence = False
ranges.append((fence_start, pos + len(line)))
pos += len(line)
if in_fence:
ranges.append((fence_start, len(text)))
return ranges
def _in_ranges(idx: int, end: int, ranges: list[tuple[int, int]]) -> bool:
return any(a <= idx and end <= b or (a < end and idx < b) for a, b in ranges)
def _normalize_v_marker(
marker: dict[str, Any],
catalog: dict[str, Any],
qinfos: dict[str, _QueryInfo],
loader: Any = None,
) -> str:
ref = str(marker.get("ref") or "").strip()
conv = marker.get("conv")
conv_suffix = f"|{conv}" if conv else ""
alias = ""
row_idx: int | None = None
col_raw = ""
m_full = _CELL_REF_RE.match(ref)
if m_full:
q_upper = m_full.group("alias").upper()
alias = q_upper if q_upper in qinfos else m_full.group("alias")
row_idx = int(m_full.group("row") if m_full.group("row") is not None else m_full.group("bare_row"))
col_raw = m_full.group("col").strip()
else:
m_short = _CELL_NOROW_RE.match(ref)
if m_short:
q_upper = m_short.group("alias").upper()
qinfo = qinfos.get(q_upper)
if qinfo and len(qinfo.rows) == 1:
alias = q_upper
row_idx = 0
col_raw = m_short.group("col").strip()
if alias and alias in qinfos and row_idx is not None:
qinfo = qinfos[alias]
exact_col = qinfo.col_by_lower.get(col_raw.lower(), col_raw)
if exact_col.lower() in qinfo.masked_lower:
return MASKED_WORD
candidate_raw = f"[[v:{alias}.r{row_idx}.{exact_col}{conv_suffix}]]"
if len(candidate_raw) <= MARKER_MAX:
parsed = parse_markers(candidate_raw)
if parsed:
cite = resolve_marker(parsed[0], catalog, cells_allowed=True, loader=loader)
if cite.get("status") == "resolved":
return candidate_raw
# 若只是换算不兼容(例如文本列写了 |int),退回无换算再试一次
if conv:
plain_raw = f"[[v:{alias}.r{row_idx}.{exact_col}]]"
parsed_plain = parse_markers(plain_raw)
if parsed_plain:
cite_plain = resolve_marker(parsed_plain[0], catalog, cells_allowed=True, loader=loader)
if cite_plain.get("status") == "resolved":
return plain_raw
# 原样试试(例如引用的是分析结论 A1.r0.col)
cite_orig = resolve_marker(marker, catalog, cells_allowed=True, loader=loader)
if cite_orig.get("status") == "resolved":
return str(marker["raw"])
rendered = str(cite_orig.get("rendered") or "")
if rendered and rendered != placeholder(marker["kind"], marker["body"]):
return rendered
return "—"
def _normalize_table_marker(
marker: dict[str, Any],
catalog: dict[str, Any],
qinfos: dict[str, _QueryInfo],
loader: Any = None,
) -> str:
ref = str(marker.get("ref") or "").strip()
q_upper = ref.upper()
alias = q_upper if q_upper in qinfos else ref
options = dict(marker.get("options") or {})
if alias in qinfos:
qinfo = qinfos[alias]
if not qinfo.rows:
return "(未返回记录)"
if "cols" in options:
wanted_raw = [c.strip() for c in options["cols"].split(",") if c.strip()]
valid_cols = [
qinfo.col_by_lower[c.lower()]
for c in wanted_raw
if c.lower() in qinfo.col_by_lower and c.lower() not in qinfo.masked_lower
]
if valid_cols:
options["cols"] = ",".join(valid_cols[:TABLE_MAX_COLS])
else:
options.pop("cols", None)
else:
unmasked_cols = [c for c in qinfo.columns if c.lower() not in qinfo.masked_lower]
if not unmasked_cols:
return MASKED_WORD
if len(unmasked_cols) < len(qinfo.columns) or len(unmasked_cols) > TABLE_MAX_COLS:
options["cols"] = ",".join(unmasked_cols[:TABLE_MAX_COLS])
opt_str = " ".join(f"{k}={v}" for k, v in options.items())
body = f"{alias} {opt_str}".strip()
candidate_raw = f"[[table:{body}]]"
if len(candidate_raw) <= MARKER_MAX:
parsed = parse_markers(candidate_raw)
if parsed:
cite = resolve_marker(parsed[0], catalog, cells_allowed=True, loader=loader)
if cite.get("status") == "resolved":
return candidate_raw
# 回退到仅保留安全列的基础整表标记
safe_cols = [
c for c in qinfo.columns
if c.lower() not in qinfo.masked_lower and c.strip() == c and not _UNSAFE_COL_RE.search(c)
][:TABLE_MAX_COLS]
if safe_cols:
fallback_raw = (
f"[[table:{alias}]]"
if len(safe_cols) == len(qinfo.columns)
else f"[[table:{alias} cols={','.join(safe_cols)}]]"
)
parsed_fb = parse_markers(fallback_raw)
if parsed_fb and resolve_marker(parsed_fb[0], catalog, cells_allowed=True, loader=loader).get("status") == "resolved":
return fallback_raw
cite_orig = resolve_marker(marker, catalog, cells_allowed=True, loader=loader)
if cite_orig.get("status") == "resolved":
return str(marker["raw"])
return ""
def _sanitize_existing_markers(
text: str,
catalog: dict[str, Any],
qinfos: dict[str, _QueryInfo],
loader: Any = None,
) -> str:
"""第一遍:修复与清理模型显式写出的 [[...]] 标记,杜绝未解析占位符和句中 [[see:]] 断句。"""
# 先移除会造成断句的句中 [[see:...]](例如「根据 [[see:Q1]],」->「」)
fences = _fenced_line_ranges(text)
markers = parse_markers(text)
if not markers:
return text
out: list[str] = []
last = 0
for m in markers:
start, end = int(m["start"]), int(m["end"])
if _in_ranges(start, end, fences):
continue
out.append(text[last:start])
kind = m["kind"]
if kind == "v":
out.append(_normalize_v_marker(m, catalog, qinfos, loader=loader))
elif kind == "table":
out.append(_normalize_table_marker(m, catalog, qinfos, loader=loader))
elif kind == "see":
valid_refs = [
r for r in (m.get("refs") or [])
if resolve_support(r, catalog, loader=loader).get("status") == "resolved"
]
if valid_refs:
# 检查周围是否构成句中 _INLINE_SEE 违规,若是则丢弃此标记避免留下「以 为依据」断句
window = text[max(0, start - 6):min(len(text), end + 6)]
if _INLINE_SEE.search(window):
out.append("")
else:
out.append(f"[[see:{','.join(valid_refs)}]]")
else:
out.append("")
else:
cite = resolve_marker(m, catalog, cells_allowed=True, loader=loader)
if cite.get("status") == "resolved":
out.append(str(m["raw"]))
else:
rendered = str(cite.get("rendered") or "")
if rendered and rendered != placeholder(kind, m["body"]):
out.append(rendered)
elif kind in ("t", "c") and m.get("ref"):
out.append(f"`{m['ref']}`")
else:
out.append("")
last = end
out.append(text[last:])
return "".join(out)
def _alias_order(alias: str) -> int:
m = re.match(r"^Q(\d+)$", alias.strip(), re.I)
return int(m.group(1)) if m else 0
def _build_cell_candidates(qinfos: dict[str, _QueryInfo]) -> list[_CellCandidate]:
candidates: list[_CellCandidate] = []
for q_pos, (alias, qinfo) in enumerate(qinfos.items()):
q_order = _alias_order(alias) or (q_pos + 1)
for r_idx, row in enumerate(qinfo.rows):
row_keys: list[str] = []
for j, col in enumerate(qinfo.columns):
if j >= len(row) or col.lower() in qinfo.masked_lower:
continue
val = row[j]
if val is None or isinstance(val, bool):
continue
s = str(val).strip()
col_low = col.lower()
is_id_col = j == 0 or col_low.endswith(_ID_LIKE_COL_SUFFIXES)
if s and s != MASKED_WORD and (_to_decimal(val) is None or qinfo.kinds[j] == "text" or is_id_col):
row_keys.append(s)
if isinstance(val, str) and re.match(r"^\d{4}-\d{2}(?:-\d{2})?", s):
row_keys.append(s[:10])
keys_tuple = tuple(dict.fromkeys(k for k in row_keys if len(k) >= 2))
for j, col in enumerate(qinfo.columns):
if j >= len(row):
continue
if col.lower() in qinfo.masked_lower:
continue
if not col.strip() or col != col.strip() or _UNSAFE_COL_RE.search(col):
continue
kind = qinfo.kinds[j]
raw_val = row[j]
if kind == "text":
s_val = str(raw_val).strip() if raw_val is not None else ""
if not s_val or s_val == MASKED_WORD or not _NUMERIC_CODE_TEXT_RE.match(s_val):
continue
dec_code = _to_decimal(s_val)
if dec_code is None:
continue
candidates.append(
_CellCandidate(
alias=alias,
query_order=q_order,
row=r_idx,
col=col,
col_idx=j,
dec_value=dec_code,
row_keys=keys_tuple,
raw_text=s_val,
is_text_code=True,
)
)
continue
parsed_val = cell_value(raw_val, kind)
dec = _to_decimal(parsed_val)
if dec is None:
continue
candidates.append(
_CellCandidate(
alias=alias,
query_order=q_order,
row=r_idx,
col=col,
col_idx=j,
dec_value=dec,
row_keys=keys_tuple,
raw_text=str(raw_val).strip() if raw_val is not None else "",
is_text_code=False,
)
)
return candidates
def _table_headers_by_line(lines: list[str]) -> dict[int, list[str]]:
"""识别手写 Markdown 表格每行对应的表头列名,辅助同名数值按列对齐消歧。"""
mapping: dict[int, list[str]] = {}
i = 0
n = len(lines)
while i + 1 < n:
head = lines[i].strip()
sep = lines[i + 1].strip()
if head.startswith("|") and head.endswith("|") and _TABLE_SEP_LINE_RE.match(sep):
headers = [c.strip().strip("`*") for c in head.strip("|").split("|")]
k = i + 2
while k < n and lines[k].lstrip().startswith("|") and not _TABLE_SEP_LINE_RE.match(lines[k]):
mapping[k] = headers
k += 1
i = k
continue
i += 1
return mapping
def _auto_bind_bare_numbers(
text: str,
catalog: dict[str, Any],
qinfos: dict[str, _QueryInfo],
loader: Any = None,
) -> str:
"""第二遍:对正文中未写标记、但能唯一无歧义匹配未遮罩查询结果单元格的裸数字,自动绑定 [[v:Qn.rR.col]]。"""
cells = _build_cell_candidates(qinfos)
if not cells:
return text
cell_order = {id(c): idx for idx, c in enumerate(cells)}
text_code_by_raw: dict[str, list[_CellCandidate]] = {}
numeric_cells: list[_CellCandidate] = []
numeric_by_exact: dict[Decimal, list[_CellCandidate]] = {}
for c in cells:
if c.is_text_code:
text_code_by_raw.setdefault(c.raw_text, []).append(c)
else:
numeric_cells.append(c)
numeric_by_exact.setdefault(c.dec_value, []).append(c)
quant_cache: dict[tuple[str, int], dict[Decimal, list[_CellCandidate]]] = {}
def _quant_buckets(scale_kind: str, places: int) -> dict[Decimal, list[_CellCandidate]]:
key = (scale_kind, places)
cached = quant_cache.get(key)
if cached is not None:
return cached
buckets: dict[Decimal, list[_CellCandidate]] = {}
quant = Decimal(1).scaleb(-places)
if scale_kind == "pct":
for c in numeric_cells:
q_val = (c.dec_value * Decimal(100)).quantize(quant, rounding=ROUND_HALF_UP)
buckets.setdefault(q_val, []).append(c)
else:
divisor = Decimal(10000) if scale_kind == "万" else Decimal(100_000_000)
for c in numeric_cells:
q_val = (c.dec_value / divisor).quantize(quant, rounding=ROUND_HALF_UP)
buckets.setdefault(q_val, []).append(c)
quant_cache[key] = buckets
return buckets
fences = _fenced_line_ranges(text)
lines = text.splitlines(keepends=True)
table_headers = _table_headers_by_line(lines)
line_offsets: list[int] = []
acc = 0
for ln in lines:
line_offsets.append(acc)
acc += len(ln)
out_lines: list[str] = []
for line_idx, line in enumerate(lines):
line_start = line_offsets[line_idx]
line_end = line_start + len(line)
if _in_ranges(line_start, line_end, fences) or _TABLE_SEP_LINE_RE.match(line):
out_lines.append(line)
continue
# 表头行本身不自动替换列名里的数字
if (line_idx + 1) in table_headers and line.strip().startswith("|"):
out_lines.append(line)
continue
protected: list[tuple[int, int]] = []
for m in MARKER_RE.finditer(line):
protected.append((m.start(), m.end()))
for m in _INLINE_CODE_RE.finditer(line):
protected.append((m.start(), m.end()))
for m in _MD_LINK_URL_RE.finditer(line):
protected.append((m.start(), m.end()))
masked_chars = list(line)
for a, b in protected:
for k in range(max(0, a), min(len(masked_chars), b)):
if masked_chars[k] != "\n":
masked_chars[k] = " "
masked_line = "".join(masked_chars)
tokens = [
t for t in extract_numbers(masked_line)
if not _structural_number(masked_line, t)
]
if not tokens:
out_lines.append(line)
continue
headers_here = table_headers.get(line_idx)
# 若该行是 Markdown 表格数据行,计算各竖线位置以便定位当前 token 在第几列
pipe_positions = [idx for idx, ch in enumerate(line) if ch == "|"] if headers_here else []
replacements: list[tuple[int, int, str]] = []
for tok in tokens:
if _in_ranges(tok.start, tok.end, protected):
continue
# 只自动回绑阿拉伯数字(跳过「两天」「三成」等中文数字词)
if not any(ch.isascii() and ch.isdigit() for ch in tok.raw):
continue
# 「0 行」「0 条记录」不绑定到业务数值 0
if tok.value == 0 and _ZERO_ROW_COUNT_RE.match(line[tok.end:tok.end + 6]):
continue
raw_clean = tok.raw.strip()
unit_conv: str | None = None
unit_scale: Decimal | None = None
if raw_clean.endswith(("万", "亿")) and not raw_clean.endswith("万亿"):
unit_conv = raw_clean[-1]
unit_scale = Decimal(10000) if unit_conv == "万" else Decimal(100_000_000)
num_str = raw_clean[:-1].replace(",", "").strip()
else:
num_str = raw_clean.rstrip("%%").replace(",", "").strip()
target_dec = _to_decimal(num_str)
if target_dec is None:
continue
has_leading_zero = bool(_LEADING_ZERO_INT_RE.match(raw_clean))
places = max(int(tok.decimals or 0), 0)
matched: list[tuple[_CellCandidate, str | None]] = []
if has_leading_zero:
matched = [(c, None) for c in text_code_by_raw.get(raw_clean, ())]
elif unit_conv and unit_scale is not None:
matched = [(c, unit_conv) for c in _quant_buckets(unit_conv, places).get(target_dec, ())]
elif not tok.is_percent:
tc_hits = [(c, None) for c in text_code_by_raw.get(raw_clean, ())]
num_hits = [(c, None) for c in numeric_by_exact.get(target_dec, ())]
if tc_hits and num_hits:
matched = sorted(tc_hits + num_hits, key=lambda pair: cell_order[id(pair[0])])
else:
matched = tc_hits or num_hits
else:
# 单元格存 12.6(直接相等)或存 0.1258(正文写 12.6% 或 12.58%)
exact_hits = [(c, None) for c in numeric_by_exact.get(target_dec, ())]
pct_hits = [
(c, "pct")
for c in _quant_buckets("pct", places).get(target_dec, ())
if c.dec_value != target_dec
]
if exact_hits and pct_hits:
matched = sorted(exact_hits + pct_hits, key=lambda pair: cell_order[id(pair[0])])
else:
matched = exact_hits or pct_hits
if not matched:
continue
# 多候选消歧 1:按同行出现的行键命中数(如同时命中编码、名称、日期的明细行优先于仅命中编码的汇总行)收窄
if len(matched) > 1:
scored_rk = [
(sum(1 for rk in pair[0].row_keys if rk in line), pair)
for pair in matched
]
max_rk_hits = max((cnt for cnt, _ in scored_rk), default=0)
if max_rk_hits > 0:
matched = [pair for cnt, pair in scored_rk if cnt == max_rk_hits]
# 多候选消歧 2:Markdown 表格行内先按表头列名精确对齐,表头改写为中文时再按同列数查询的列序号对齐
if len(matched) > 1 and headers_here and len(pipe_positions) >= 2:
col_pos = sum(1 for p in pipe_positions if p < tok.start) - 1
if 0 <= col_pos < len(headers_here):
hdr = headers_here[col_pos].lower()
by_hdr = [pair for pair in matched if pair[0].col.lower() == hdr]
if by_hdr:
matched = by_hdr
if len(matched) > 1:
by_col_pos = [
pair
for pair in matched
if pair[0].col_idx == col_pos
and len(qinfos[pair[0].alias].columns) == len(headers_here)
]
if by_col_pos:
matched = by_col_pos
# 多候选消歧 3:跨汇总/明细查询的同名列同值,或同编码字符串在多行重复出现时,确定性取最新查询首行
unique_locs = {(c.alias, c.row, c.col) for c, _ in matched}
if len(unique_locs) > 1:
convs = {conv for _, conv in matched}
if len(convs) == 1:
col_names = {c.col.lower() for c, _ in matched}
all_same_text_code = all(c.is_text_code for c, _ in matched) and len({c.raw_text for c, _ in matched}) == 1
if len(col_names) == 1 or all_same_text_code:
matched = [
max(
matched,
key=lambda pair: (pair[0].query_order, -pair[0].row, -pair[0].col_idx),
)
]
unique_locs = {(c.alias, c.row, c.col) for c, _ in matched}
if len(unique_locs) != 1:
continue
chosen, conv = matched[0]
conv_part = f"|{conv}" if conv else ""
marker_str = f"[[v:{chosen.alias}.r{chosen.row}.{chosen.col}{conv_part}]]"
if len(marker_str) > MARKER_MAX:
continue
parsed = parse_markers(marker_str)
if not parsed:
continue
cite = resolve_marker(parsed[0], catalog, cells_allowed=True, loader=loader)
if cite.get("status") != "resolved":
continue
rendered = str(cite.get("rendered") or "")
rep_start, rep_end = tok.start, tok.end
# 若原文带 %(如 40%),但无 pct 换算的单元格本身存的是 40、渲染为 "40",保留原文末尾的 %
if tok.is_percent and not rendered.endswith("%") and line[rep_end - 1:rep_end] in ("%", "%"):
rep_end -= 1
replacements.append((rep_start, rep_end, marker_str))
if not replacements:
out_lines.append(line)
continue
rebuilt: list[str] = []
cursor = 0
for a, b, rep in replacements:
if a < cursor:
continue
rebuilt.append(line[cursor:a])
rebuilt.append(rep)
cursor = b
rebuilt.append(line[cursor:])
out_lines.append("".join(rebuilt))
return "".join(out_lines)
_TRAILING_V_CLUSTER_RE = re.compile(
r"(?P<punct>[。!?;.!?;])[ \t]*(?P<tail>(?:\[\[v:[^\]]+\]\][ \t,,、;;。.]*)+)(?P<nl>\r?\n?)$"
)
_ADJACENT_DUP_RE = re.compile(
r"(?<![A-Za-z0-9_])(?P<num>[+-]?\d[\d,]*(?:\.\d+)?[%%万亿]?)(?:[ \t]*(?P<marker>\[\[v:[^\]]+\]\])|[ \t]*[((][ \t]*(?P<pmarker>\[\[v:[^\]]+\]\])[ \t]*[))])"
)
def _comparable_decimal(raw_num: str) -> Decimal | None:
s = raw_num.strip().lstrip("+")
if not s:
return None
if s.endswith(("万", "亿")) and not s.endswith("万亿"):
s = s[:-1].strip()
else:
s = s.rstrip("%%").strip()
return _to_decimal(s.replace(",", ""))
def _dedupe_marker_echoes(
text: str,
catalog: dict[str, Any],
loader: Any = None,
) -> str:
"""消除模型误把 [[v:...]] 当作脚注追加在裸数字后或句末造成的重复渲染(如 `69,133 [[v:...]]` 或句末堆叠 `。[[v:...]]`)。"""
if "[[v:" not in text:
return text
fences = _fenced_line_ranges(text)
lines = text.splitlines(keepends=True)
acc = 0
out_lines: list[str] = []
for line in lines:
l_start, l_end = acc, acc + len(line)
acc = l_end
if _in_ranges(l_start, l_end, fences) or "[[v:" not in line:
out_lines.append(line)
continue
# 1) 紧邻重复:`69,133 [[v:Q1.r0.total_sys]]` 或 `69,133([[v:Q1.r0.total_sys]])`
def _collapse_adj(match: re.Match[str]) -> str:
num_txt = match.group("num")
m_txt = match.group("marker") or match.group("pmarker") or ""
d_num = _comparable_decimal(num_txt)
parsed = parse_markers(m_txt)
if d_num is not None and parsed:
cite = resolve_marker(parsed[0], catalog, cells_allowed=True, loader=loader)
if cite.get("status") == "resolved":
d_cite = _comparable_decimal(str(cite.get("rendered") or ""))
if d_cite is not None and d_cite == d_num:
return m_txt
return match.group(0)
cur = _ADJACENT_DUP_RE.sub(_collapse_adj, line)
# 2) 句末脚注式堆叠:`... 69,133 ... 50,617。[[v:Q1.r0.a]] [[v:Q1.r0.b]]`
m_tail = _TRAILING_V_CLUSTER_RE.search(cur)
if m_tail:
prefix = cur[: m_tail.start("tail")]
tail_str = m_tail.group("tail")
tail_markers = parse_markers(tail_str)
if tail_markers:
prefix_decs: set[Decimal] = set()
for tok in extract_numbers(prefix):
if any(ch.isascii() and ch.isdigit() for ch in tok.raw):
d = _comparable_decimal(tok.raw)
if d is not None:
prefix_decs.add(d)
for pm in parse_markers(prefix):
c = resolve_marker(pm, catalog, cells_allowed=True, loader=loader)
if c.get("status") == "resolved":
d = _comparable_decimal(str(c.get("rendered") or ""))
if d is not None:
prefix_decs.add(d)
all_dup = True
for tm in tail_markers:
c = resolve_marker(tm, catalog, cells_allowed=True, loader=loader)
d = _comparable_decimal(str(c.get("rendered") or "")) if c.get("status") == "resolved" else None
if d is None or d not in prefix_decs:
all_dup = False
break
if all_dup:
cur = prefix + m_tail.group("nl")
out_lines.append(cur)
return "".join(out_lines)
def collect_fast_allow_numbers(
prepared_text: str,
catalog: dict[str, Any],
*,
masked_columns: Mapping[str, Any] | None = None,
question: str = "",
loader: Any = None,
) -> list[str]:
"""为快速直查(numbers=off)提取有事实依据的上下文豁免字面量,消除标题复述、公式常数与行数统计的误报,同时保留对真编造数字的拦截。"""
if not prepared_text or not prepared_text.strip():
return []
qinfos = _load_query_infos(catalog, masked_columns=dict(masked_columns) if masked_columns else None, loader=loader)
fences = _fenced_line_ranges(prepared_text)
inline_code_spans = [
m.span()
for m in _INLINE_CODE_RE.finditer(prepared_text)
if not _in_ranges(m.start(), m.end(), fences)
]
# 构造与 compose_doc._bare_numbers 对齐的裸数字扫描面(抹去代码块与已绑定 [[...]] 标记)
chars = list(prepared_text)
for a, b in fences:
for k in range(max(0, a), min(len(chars), b)):
if chars[k] != "\n":
chars[k] = " "
for m in MARKER_RE.finditer(prepared_text):
for k in range(max(0, m.start()), min(len(chars), m.end())):
if chars[k] != "\n":
chars[k] = " "
for idx, ch in enumerate(chars):
if ch in _SYNTAX_BLANK:
chars[idx] = _SYNTAX_BLANK[ch]
masked_doc = "".join(chars)
bare_tokens = [t for t in extract_numbers(masked_doc) if not _structural_number(masked_doc, t)]
if not bare_tokens:
return []
verified_floats: set[float] = set()
verified_pct_floats: set[float] = set()
# 1) 用户原问题中显式给出的数字与编码(先剥离「11月11日」类中文月日,避免粘连编码如「395011月11」误拆成 395011)
if question and question.strip():
q_clean = _CN_MONTH_DAY_RE.sub(" ", question)
for qt in extract_numbers(q_clean):
verified_floats.add(float(qt.value))
for dm in re.finditer(r"(?<![A-Za-z0-9_.])\d+(?:\.\d+)?(?![A-Za-z0-9_])", q_clean):
try:
verified_floats.add(float(dm.group(0)))
except ValueError:
pass
# 2) 未遮罩查询快照中真实存在的单元格值(多列同值未能唯一绑定单个单元格时,数值本身仍有快照依据)与未截断结果集的精确计数
for cand in _build_cell_candidates(qinfos):
fv = float(cand.dec_value)
verified_floats.add(fv)
if not cand.is_text_code and "." in cand.raw_text and Decimal(0) < cand.dec_value < Decimal(1):
verified_pct_floats.add(float(cand.dec_value * Decimal(100)))
for qinfo in qinfos.values():
if qinfo.truncated or not qinfo.rows:
continue
verified_floats.add(float(len(qinfo.rows)))
for j, col in enumerate(qinfo.columns):
if col.lower() in qinfo.masked_lower:
continue
non_null_strs: list[str] = []
zero_cnt = 0
pos_cnt = 0
kind = qinfo.kinds[j]
for r in qinfo.rows:
if j >= len(r) or r[j] is None or r[j] == MASKED_WORD:
continue
s = str(r[j]).strip()
if s:
non_null_strs.append(s)
if kind != "text":
dv = _to_decimal(cell_value(r[j], kind))
if dv is not None:
if dv == 0:
zero_cnt += 1
elif dv > 0:
pos_cnt += 1
if non_null_strs:
verified_floats.add(float(len(set(non_null_strs))))
if kind == "text" or j == 0 or col.lower().endswith(_ID_LIKE_COL_SUFFIXES):
for g_cnt in Counter(non_null_strs).values():
verified_floats.add(float(g_cnt))
if zero_cnt > 0:
verified_floats.add(float(zero_cnt))
if pos_cnt > 0:
verified_floats.add(float(pos_cnt))
allow_out: list[str] = []
seen_allow: set[str] = set()
def _add_allow(token_str: str) -> None:
s = token_str.strip()
if s and s not in seen_allow:
seen_allow.add(s)
allow_out.append(s)
arabic_verified: list[tuple[Any, str]] = []
arabic_contextual: list[tuple[Any, str]] = []
arabic_unverified: list[Any] = []
for tok in bare_tokens:
is_arabic = any(ch.isascii() and ch.isdigit() for ch in tok.raw)
if not is_arabic:
# 中文数量词(如「两个班次」中的「两」):仅当对应数值在查询事实中存在时放行原词字面量,不污染阿拉伯数字容差集
if any(_close(tok.value, vf, _tolerance(tok)) for vf in verified_floats):
_add_allow(tok.raw)
continue
norm_raw = tok.raw.rstrip("%%").strip()
matched_verified = any(_close(tok.value, vf, _tolerance(tok)) for vf in verified_floats) or (
tok.is_percent and any(_close(tok.value, pf, _tolerance(tok)) for pf in verified_pct_floats)
)
if matched_verified:
arabic_verified.append((tok, norm_raw))
continue
in_code = _in_ranges(tok.start, tok.end, inline_code_spans)
before_slice = prepared_text[max(0, tok.start - 6):tok.start]
is_pct_formula_const = abs(tok.value - 100.0) < 1e-9 and bool(_FORMULA_MUL_DIV_PREFIX_RE.search(before_slice))
if in_code or is_pct_formula_const:
arabic_contextual.append((tok, norm_raw))
else:
arabic_unverified.append(tok)
for tok, norm_raw in arabic_verified:
if not any(_close(u.value, tok.value, _tolerance(u)) for u in arabic_unverified):
_add_allow(norm_raw)
for tok, norm_raw in arabic_contextual:
# 上下文豁免(行内反引号字面量、乘除 100% 公式换算常数)绝不能连带放过正文里同值的无出处编造数字
if not any(_close(u.value, tok.value, _tolerance(u)) for u in arabic_unverified):
_add_allow(norm_raw)
return allow_out
def normalize_and_bind_fast_citations(
raw: str,
catalog: dict[str, Any],
*,
masked_columns: dict[str, Any] | None = None,
loader: Any = None,
) -> str:
"""将快速直查 Agent 的输出归一为干净、无未解析标记且自动补齐唯一单元格出处的报告源文本。"""
if not raw or not raw.strip():
return raw or ""
qinfos = _load_query_infos(catalog, masked_columns=masked_columns, loader=loader)
step1 = _sanitize_existing_markers(raw, catalog, qinfos, loader=loader)
step2 = _dedupe_marker_echoes(step1, catalog, loader=loader)
if not qinfos:
return step2
return _auto_bind_bare_numbers(step2, catalog, qinfos, loader=loader)