diff --git a/backend/app/chat/direct.py b/backend/app/chat/direct.py index 52a4aa11..9187900d 100644 --- a/backend/app/chat/direct.py +++ b/backend/app/chat/direct.py @@ -49,19 +49,22 @@ DIRECT_SYSTEM_PROMPT = """你是 AgentLab 的「快速直查数据分析助手 【核心取数与防错纪律】 1. **{time_banner}** -2. **优先走标准语义指标**：若下方「已命中的标准语义指标卡片」直接对应用户所问指标，优先调用一次 `metric_query` 获取标准口径结果（支持 `date_from`、`date_to`、`group_by_date`、`group_by`、`filters`）。 +2. **严控轮次预算，杜绝多轮逆向摸底（最多 1～2 轮工具调用）**： + - 快速直查的目标是秒级直取答案，**严禁**展开多轮跨表试探与字段考古。 + - 若下方「已命中的标准语义指标卡片」直接对应用户所问指标且口径完整，第 1 轮直接调用 `metric_query`（支持 `date_from`、`date_to`、`group_by_date`、`group_by`、`filters`）拿到数据后立即作答。 + - 若第 1 轮返回空集、全空分组或口径明显异常（如底表混存多种记录类型、汇总层级或分类口径而缺少筛选条件），**最多再做 1 轮并行 `db_query__` 补查**（在同一轮内同时核对关联维表或配置表映射并查询过滤后的目标汇总 SQL），拿到结果立即收口作答；若仍存在跨表复杂口径分歧，直接呈现已核实的事实并建议用户点击右上角「切换深度分析」。 3. **合并查询，杜绝碎片化往返**： - 手写 `db_query__<源名>` 时，尽量在 **1 条（最多不超过 2 条）SQL** 内一次性查全所需指标与近期对比窗口（例如查「今天怎么样」时，可直接查近 7 天按日汇总 `WHERE stat_date >= '<7日前>' ORDER BY stat_date DESC`，一条 SQL 同时看清今天是否有数、最新到哪一天、以及相比前几天的走势！）。 - - 已开启并行工具调用（`parallel_tools=True`）：确实需要查两张互不依赖的表时，在同一轮一次性并行发出工具调用。 + - 已开启并行工具调用（`parallel_tools=True`）：确实需要同时看汇总与下钻明细、或查两张互不依赖的表时，必须在**同一轮**一次性并行发出工具调用，切勿拆成多轮串行调用。 4. **严守标准业务实体与数据目录护栏**： - 若下方给出了「标准业务实体与物理编码映射」，直接按映射出的列名与底层编码写过滤条件，不要发 `SELECT DISTINCT` 盲猜编码。 - - 严格遵守「数据目录强制防错护栏」中的业务时间列（`business_date`）、有效记录过滤条件（`valid_filter`）以及存量列（`stock`）不可跨期直接求和的约束。 + - 严格遵守「数据目录强制防错护栏」中的业务时间列（`business_date`）、有效记录过滤条件（`valid_filter`）、表粒度与去重约束（禁将不同汇总层级或不同业务分类行直接混算）以及存量列（`stock`）不可跨期直接求和的约束。 - 下方「预命中物理表结构」已包含完整字段与目录的表，直接写 SQL，不要重复调 `db_schema__` 查看；只有需要查找未列出的其他表时才用 `db_schema__`（支持传 `keyword` 检索或一次传多个表名）。 5. **诚实简洁作答与可点击数字证据**： - 每个数字都必须真实出自本轮工具返回结果，严禁编造或用数据剖析里的估算行数当查询答案。 - - 工具返回头部标有 `【证据 Q1】`、`【证据 Q2】` 等编号：凡是取自查询结果的数值，请直接用单元格引用标记 `[[v:Q1.r0.列名]]` **替代数字本身**（行号从 `r0` 起算；支持 `|万`、`|亿`、`|pct`、`|int`、`|.1` 等换算）。注意：`[[v:...]]` 会直接渲染成数值本身，正确写法为 `总客流 [[v:Q1.r0.total_sys]] 人次`；**严禁**先写裸数字再在后面或句末重复附带 `[[v:...]]`！ - - 多行明细优先直接写整表标记 `[[table:Q1]]`（或 `[[table:Q1 cols=列1,列2 rows=0-9]]`），整表内的数字会自动生成可点击证据；真实物理表名用反引号包裹（如 `` `footfall_daily_stat` ``）；日期统一写完整 `YYYY-MM-DD`（如 `2026-11-10`）。 - - 需要算合计、差值、占比或环比涨跌幅时，尽量在同一条 SQL 的 `SELECT` 中用 `ROUND(..., 1)` 直接算出结果列并用 `[[v:Q1.r0.列名]]` 引用（不要心算裸写百分比），让派生指标同样具备 SQL 级可点击证据。 + - 工具返回头部标有 `【证据 Q1】`、`【证据 Q2】` 等编号：凡是取自查询结果的数值或维度编码，请直接用单元格引用标记 `[[v:Q1.r0.列名]]` **替代该值本身**（行号从 `r0` 起算；支持 `|万`、`|亿`、`|pct`、`|int`、`|.1` 等换算）。注意：`[[v:Q1.r0.列名]]` 会直接渲染成数值本身，正确写法为 `总客流 [[v:Q1.r0.total_sys]] 人次`；**严禁**先写裸数字再在后面或句末重复附带引用标记！ + - 多行明细优先直接写整表标记 `[[table:Q1]]`（或 `[[table:Q1 cols=列1,列2 rows=0-9]]`），整表内的数字与编码会自动生成可点击证据；真实物理表名与字段名用反引号包裹（如 `` `footfall_daily_stat` ``）；日期统一写完整 `YYYY-MM-DD`（如 `2026-11-10`）。 + - 需要算合计、差值、占比、零值条数或环比涨跌幅时，尽量在同一条 SQL 的 `SELECT` 中直接算出结果列并用 `[[v:Q1.r0.列名]]` 引用（不要心算裸写统计个数或百分比），让所有派生数字同样具备 SQL 级可点击证据。 - 若当日数据尚未入库（查询未返回记录），明确写「暂无入库记录」或「未返回记录」（不要手写「0 行」），并主动给出已查到的最新可用日期数据供参考。""" @@ -118,7 +121,9 @@ def build_direct_graph( "prompt": user_prompt, "tools": tools, "parallel_tools": True, - "max_steps": 8, + "thinking": "off", + "effort": "low", + "max_steps": 4, "ask_data_optimization": 2, "emit_message": False, }, diff --git a/backend/app/chat/direct_evidence.py b/backend/app/chat/direct_evidence.py index 666422d5..41b3dd3d 100644 --- a/backend/app/chat/direct_evidence.py +++ b/backend/app/chat/direct_evidence.py @@ -9,6 +9,8 @@ """ from __future__ import annotations +from collections import Counter +from collections.abc import Mapping from dataclasses import dataclass from decimal import Decimal, InvalidOperation, ROUND_HALF_UP import re @@ -20,6 +22,7 @@ from app.engine.evidence import ( MARKER_RE, TABLE_MAX_COLS, _INLINE_SEE, + _SYNTAX_BLANK, _structural_number, parse_markers, placeholder, @@ -27,7 +30,7 @@ from app.engine.evidence import ( resolve_support, ) from app.engine.expressions import cell_value, column_kind -from app.engine.issuance import extract_numbers +from app.engine.issuance import _close, _tolerance, extract_numbers MASKED_WORD = "已遮罩" @@ -42,15 +45,24 @@ _INLINE_CODE_RE = re.compile(r"`+[^`\n]*`+") _MD_LINK_URL_RE = re.compile(r"\]\([^\)\n]*\)") _TABLE_SEP_LINE_RE = re.compile(r"^\s*\|[\s:\-|]+\|\s*$") _ZERO_ROW_COUNT_RE = re.compile(r"^\s*(?:行|条记录|条数据)") +_NUMERIC_CODE_TEXT_RE = re.compile(r"^[+-]?\d+(?:\.\d+)?$") +_LEADING_ZERO_INT_RE = re.compile(r"^[+-]?0\d+$") +_ID_LIKE_COL_SUFFIXES = ("_id", "_code", "_no", "_key", "_type", "_group", "id", "code", "type", "group") +_CN_MONTH_DAY_RE = re.compile(r"(?:\d{4}\s*年\s*)?\d{1,2}\s*月\s*\d{1,2}\s*[日号]?") +_FORMULA_MUL_DIV_PREFIX_RE = re.compile(r"[×*÷/]\s*$") @dataclass(frozen=True) class _CellCandidate: alias: str + query_order: int row: int col: str + col_idx: int dec_value: Decimal row_keys: tuple[str, ...] + raw_text: str = "" + is_text_code: bool = False @dataclass @@ -62,6 +74,7 @@ class _QueryInfo: rows: list[list[Any]] kinds: list[str | None] masked_lower: set[str] + truncated: bool = False def _to_decimal(val: Any) -> Decimal | None: @@ -138,6 +151,7 @@ def _load_query_infos( rows=rows, kinds=kinds, masked_lower=masked_set, + truncated=bool(snap.get("truncated") or entry.get("truncated")), ) return infos @@ -352,9 +366,15 @@ def _sanitize_existing_markers( return "".join(out) +def _alias_order(alias: str) -> int: + m = re.match(r"^Q(\d+)$", alias.strip(), re.I) + return int(m.group(1)) if m else 0 + + def _build_cell_candidates(qinfos: dict[str, _QueryInfo]) -> list[_CellCandidate]: candidates: list[_CellCandidate] = [] - for alias, qinfo in qinfos.items(): + for q_pos, (alias, qinfo) in enumerate(qinfos.items()): + q_order = _alias_order(alias) or (q_pos + 1) for r_idx, row in enumerate(qinfo.rows): row_keys: list[str] = [] for j, col in enumerate(qinfo.columns): @@ -364,7 +384,9 @@ def _build_cell_candidates(qinfos: dict[str, _QueryInfo]) -> list[_CellCandidate if val is None or isinstance(val, bool): continue s = str(val).strip() - if s and s != MASKED_WORD and (_to_decimal(val) is None or qinfo.kinds[j] == "text"): + col_low = col.lower() + is_id_col = j == 0 or col_low.endswith(_ID_LIKE_COL_SUFFIXES) + if s and s != MASKED_WORD and (_to_decimal(val) is None or qinfo.kinds[j] == "text" or is_id_col): row_keys.append(s) if isinstance(val, str) and re.match(r"^\d{4}-\d{2}(?:-\d{2})?", s): row_keys.append(s[:10]) @@ -378,9 +400,28 @@ def _build_cell_candidates(qinfos: dict[str, _QueryInfo]) -> list[_CellCandidate if not col.strip() or col != col.strip() or _UNSAFE_COL_RE.search(col): continue kind = qinfo.kinds[j] - if kind == "text": - continue raw_val = row[j] + if kind == "text": + s_val = str(raw_val).strip() if raw_val is not None else "" + if not s_val or s_val == MASKED_WORD or not _NUMERIC_CODE_TEXT_RE.match(s_val): + continue + dec_code = _to_decimal(s_val) + if dec_code is None: + continue + candidates.append( + _CellCandidate( + alias=alias, + query_order=q_order, + row=r_idx, + col=col, + col_idx=j, + dec_value=dec_code, + row_keys=keys_tuple, + raw_text=s_val, + is_text_code=True, + ) + ) + continue parsed_val = cell_value(raw_val, kind) dec = _to_decimal(parsed_val) if dec is None: @@ -388,10 +429,14 @@ def _build_cell_candidates(qinfos: dict[str, _QueryInfo]) -> list[_CellCandidate candidates.append( _CellCandidate( alias=alias, + query_order=q_order, row=r_idx, col=col, + col_idx=j, dec_value=dec, row_keys=keys_tuple, + raw_text=str(raw_val).strip() if raw_val is not None else "", + is_text_code=False, ) ) return candidates @@ -500,8 +545,17 @@ def _auto_bind_bare_numbers( if target_dec is None: continue + has_leading_zero = bool(_LEADING_ZERO_INT_RE.match(raw_clean)) matched: list[tuple[_CellCandidate, str | None]] = [] for cand in cells: + if has_leading_zero: + if cand.is_text_code and cand.raw_text == raw_clean: + matched.append((cand, None)) + continue + if cand.is_text_code: + if not tok.is_percent and not unit_conv and cand.raw_text == raw_clean: + matched.append((cand, None)) + continue if unit_conv and unit_scale is not None: scaled_u = cand.dec_value / unit_scale places = max(int(tok.decimals or 0), 0) @@ -521,28 +575,50 @@ def _auto_bind_bare_numbers( if not matched: continue - # 多候选消歧：先按同行出现的行键（如日期 2026-11-10、区域名「西区」）收窄 + # 多候选消歧 1：按同行出现的行键命中数（如同时命中编码、名称、日期的明细行优先于仅命中编码的汇总行）收窄 if len(matched) > 1: - by_row_key = [ - pair for pair in matched - if pair[0].row_keys and any(rk in line for rk in pair[0].row_keys) + scored_rk = [ + (sum(1 for rk in pair[0].row_keys if rk in line), pair) + for pair in matched ] - if len(by_row_key) == 1: - matched = by_row_key - elif len(by_row_key) > 1: - matched = by_row_key + max_rk_hits = max((cnt for cnt, _ in scored_rk), default=0) + if max_rk_hits > 0: + matched = [pair for cnt, pair in scored_rk if cnt == max_rk_hits] - # 若仍在 Markdown 表格行内且多列同值，按表头列名精确对齐收窄 + # 多候选消歧 2：Markdown 表格行内先按表头列名精确对齐，表头改写为中文时再按同列数查询的列序号对齐 if len(matched) > 1 and headers_here and len(pipe_positions) >= 2: col_pos = sum(1 for p in pipe_positions if p < tok.start) - 1 if 0 <= col_pos < len(headers_here): hdr = headers_here[col_pos].lower() by_hdr = [pair for pair in matched if pair[0].col.lower() == hdr] - if len(by_hdr) == 1: + if by_hdr: matched = by_hdr + if len(matched) > 1: + by_col_pos = [ + pair + for pair in matched + if pair[0].col_idx == col_pos + and len(qinfos[pair[0].alias].columns) == len(headers_here) + ] + if by_col_pos: + matched = by_col_pos - # 只在唯一无歧义命中某一个 (alias, row, col) 时才自动绑定，绝不瞎猜 + # 多候选消歧 3：跨汇总/明细查询的同名列同值，或同编码字符串在多行重复出现时，确定性取最新查询首行 unique_locs = {(c.alias, c.row, c.col) for c, _ in matched} + if len(unique_locs) > 1: + convs = {conv for _, conv in matched} + if len(convs) == 1: + col_names = {c.col.lower() for c, _ in matched} + all_same_text_code = all(c.is_text_code for c, _ in matched) and len({c.raw_text for c, _ in matched}) == 1 + if len(col_names) == 1 or all_same_text_code: + matched = [ + max( + matched, + key=lambda pair: (pair[0].query_order, -pair[0].row, -pair[0].col_idx), + ) + ] + unique_locs = {(c.alias, c.row, c.col) for c, _ in matched} + if len(unique_locs) != 1: continue @@ -673,6 +749,149 @@ def _dedupe_marker_echoes( return "".join(out_lines) +def collect_fast_allow_numbers( + prepared_text: str, + catalog: dict[str, Any], + *, + masked_columns: Mapping[str, Any] | None = None, + question: str = "", + loader: Any = None, +) -> list[str]: + """为快速直查（numbers=off）提取有事实依据的上下文豁免字面量，消除标题复述、公式常数与行数统计的误报，同时保留对真编造数字的拦截。""" + if not prepared_text or not prepared_text.strip(): + return [] + + qinfos = _load_query_infos(catalog, masked_columns=dict(masked_columns) if masked_columns else None, loader=loader) + fences = _fenced_line_ranges(prepared_text) + inline_code_spans = [ + m.span() + for m in _INLINE_CODE_RE.finditer(prepared_text) + if not _in_ranges(m.start(), m.end(), fences) + ] + + # 构造与 compose_doc._bare_numbers 对齐的裸数字扫描面（抹去代码块与已绑定 [[...]] 标记） + chars = list(prepared_text) + for a, b in fences: + for k in range(max(0, a), min(len(chars), b)): + if chars[k] != "\n": + chars[k] = " " + for m in MARKER_RE.finditer(prepared_text): + for k in range(max(0, m.start()), min(len(chars), m.end())): + if chars[k] != "\n": + chars[k] = " " + for idx, ch in enumerate(chars): + if ch in _SYNTAX_BLANK: + chars[idx] = _SYNTAX_BLANK[ch] + masked_doc = "".join(chars) + + bare_tokens = [t for t in extract_numbers(masked_doc) if not _structural_number(masked_doc, t)] + if not bare_tokens: + return [] + + verified_floats: set[float] = set() + verified_pct_floats: set[float] = set() + + # 1) 用户原问题中显式给出的数字与编码（先剥离「11月11日」类中文月日，避免粘连编码如「395011月11」误拆成 395011） + if question and question.strip(): + q_clean = _CN_MONTH_DAY_RE.sub(" ", question) + for qt in extract_numbers(q_clean): + verified_floats.add(float(qt.value)) + for dm in re.finditer(r"(?= len(r) or r[j] is None or r[j] == MASKED_WORD: + continue + s = str(r[j]).strip() + if s: + non_null_strs.append(s) + if kind != "text": + dv = _to_decimal(cell_value(r[j], kind)) + if dv is not None: + if dv == 0: + zero_cnt += 1 + elif dv > 0: + pos_cnt += 1 + if non_null_strs: + verified_floats.add(float(len(set(non_null_strs)))) + if kind == "text" or j == 0 or col.lower().endswith(_ID_LIKE_COL_SUFFIXES): + for g_cnt in Counter(non_null_strs).values(): + verified_floats.add(float(g_cnt)) + if zero_cnt > 0: + verified_floats.add(float(zero_cnt)) + if pos_cnt > 0: + verified_floats.add(float(pos_cnt)) + + allow_out: list[str] = [] + seen_allow: set[str] = set() + + def _add_allow(token_str: str) -> None: + s = token_str.strip() + if s and s not in seen_allow: + seen_allow.add(s) + allow_out.append(s) + + arabic_verified: list[tuple[Any, str]] = [] + arabic_contextual: list[tuple[Any, str]] = [] + arabic_unverified: list[Any] = [] + + for tok in bare_tokens: + is_arabic = any(ch.isascii() and ch.isdigit() for ch in tok.raw) + if not is_arabic: + # 中文数量词（如「两个班次」中的「两」）：仅当对应数值在查询事实中存在时放行原词字面量，不污染阿拉伯数字容差集 + if any(_close(tok.value, vf, _tolerance(tok)) for vf in verified_floats): + _add_allow(tok.raw) + continue + + norm_raw = tok.raw.rstrip("%％").strip() + matched_verified = any(_close(tok.value, vf, _tolerance(tok)) for vf in verified_floats) or ( + tok.is_percent and any(_close(tok.value, pf, _tolerance(tok)) for pf in verified_pct_floats) + ) + if matched_verified: + arabic_verified.append((tok, norm_raw)) + continue + + in_code = _in_ranges(tok.start, tok.end, inline_code_spans) + before_slice = prepared_text[max(0, tok.start - 6):tok.start] + is_pct_formula_const = abs(tok.value - 100.0) < 1e-9 and bool(_FORMULA_MUL_DIV_PREFIX_RE.search(before_slice)) + if in_code or is_pct_formula_const: + arabic_contextual.append((tok, norm_raw)) + else: + arabic_unverified.append(tok) + + for tok, norm_raw in arabic_verified: + if not any(_close(u.value, tok.value, _tolerance(u)) for u in arabic_unverified): + _add_allow(norm_raw) + + for tok, norm_raw in arabic_contextual: + # 上下文豁免（行内反引号字面量、乘除 100% 公式换算常数）绝不能连带放过正文里同值的无出处编造数字 + if not any(_close(u.value, tok.value, _tolerance(u)) for u in arabic_unverified): + _add_allow(norm_raw) + + return allow_out + + def normalize_and_bind_fast_citations( raw: str, catalog: dict[str, Any], diff --git a/backend/app/chat/grounding.py b/backend/app/chat/grounding.py index b8eff3d1..e34f8ad5 100644 --- a/backend/app/chat/grounding.py +++ b/backend/app/chat/grounding.py @@ -28,7 +28,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.data import catalog, catalog_profile_jobs, schema_view from app.data.entities import confirmed_bindings, list_entities, translate_entity_value from app.data.names import name_key -from app.data.semantic_metrics import list_metrics +from app.data.semantic_metrics import find_companion_lookup_tables, list_metrics from app.db.models import DataSource from app.memory.embeddings import _tokens @@ -284,6 +284,20 @@ def _extract_table_guardrails( } ) + dedup_note = visible_notes.get("dedup") + if isinstance(dedup_note, Mapping) and dedup_note.get("value"): + dedup_val = str(dedup_note["value"]).strip() + if dedup_val: + rule_str = f"去重与分层防混算：{dedup_val}" + out.append( + { + "kind": "dedup", + "table": display_tbl, + "rule": rule_str, + "note": rule_str, + } + ) + cols = visible_notes.get("columns") if isinstance(cols, Mapping): stock_cols: list[str] = [] @@ -421,6 +435,34 @@ async def ground_question( picked_for_ds.append(resolved) reasons_for_table[resolved] = f"实体「{me['label']}」绑定表" + masked = schema_view.masked_set(ds) + try: + rel_graph = schema_view.relation_graph(ds, raw_notes) + except Exception: # noqa: BLE001 + rel_graph = {} + + def _include_companion_lookup(limit_cap: int) -> list[dict[str, Any]]: + if not picked_for_ds: + return [] + comps = find_companion_lookup_tables( + tables_map, + picked_for_ds, + visible_by_table=visible_by_table, + rel_graph=rel_graph, + masked_set=masked, + ) + for comp in comps: + if comp["col_count"] > 12: + continue + ct_name = comp["table_name"] + if ct_name not in picked_for_ds and len(picked_for_ds) + len(matched_tables) < limit_cap: + picked_for_ds.append(ct_name) + reasons_for_table[ct_name] = "关联维表与字典" + break + return comps + + _include_companion_lookup(MAX_GROUNDED_TABLES) + # (c) BM25 数据目录检索直选（0 LLM 调用） remaining_slots = MAX_GROUNDED_TABLES - len(matched_tables) - len(picked_for_ds) if remaining_slots > 0: @@ -436,18 +478,17 @@ async def ground_question( picked_for_ds.append(tname) reasons_for_table[tname] = "指定数据源核心表" + companion_links = _include_companion_lookup(MAX_GROUNDED_TABLES + 1) + if not picked_for_ds: continue # 补一跳连接中间表（若有） try: - rel_graph = schema_view.relation_graph(ds, raw_notes) - completed_tables, _ = schema_view.complete(picked_for_ds, rel_graph, room=1) + completed_tables, _ = schema_view.complete(picked_for_ds, rel_graph, room=1) if rel_graph else (picked_for_ds, []) except Exception: # noqa: BLE001 - rel_graph = {} completed_tables = picked_for_ds - masked = schema_view.masked_set(ds) ds_sections: list[str] = [f"### 数据源 `{ds.name}`（{ds.kind}）预命中表与目录"] for tname in completed_tables: if len(matched_tables) >= MAX_GROUNDED_TABLES + 1: @@ -485,13 +526,24 @@ async def ground_question( else: ds_sections.append(schema_view.table_brief(ds, tname)) - # 若该源有多张表入选，附上已确认/推断的 JOIN 条件 - if len(completed_tables) >= 2 and rel_graph: + # 若该源有多张表入选，附上已确认/推断的 JOIN 条件与共享键关联提示 + if len(completed_tables) >= 2: qual = schema_view.qualifier(ds) - edges = schema_view._edges_within(rel_graph, completed_tables) - if edges: - join_lines = [f" - {schema_view._edge_line(e, qual)}" for e in edges[:10]] - ds_sections.append("表间标准连接条件：\n" + "\n".join(join_lines)) + join_lines: list[str] = [] + if rel_graph: + edges = schema_view._edges_within(rel_graph, completed_tables) + join_lines.extend(f" - {schema_view._edge_line(e, qual)}" for e in edges[:10]) + completed_set = set(completed_tables) + for comp in companion_links: + if comp["linked_to"] in completed_set and comp["table_name"] in completed_set: + cond = " AND ".join( + f"{qual(comp['linked_to'])}.{sc} = {qual(comp['table_name'])}.{lc}" + for sc, lc in zip(comp["source_columns"], comp["lookup_columns"]) + ) + if cond and not any(cond in jl for jl in join_lines): + join_lines.append(f" - {cond}（共享编码列关联）") + if join_lines: + ds_sections.append("表间标准连接条件：\n" + "\n".join(join_lines[:12])) table_blocks.append("\n".join(ds_sections)) if len(matched_tables) >= MAX_GROUNDED_TABLES: diff --git a/backend/app/data/semantic_decompose.py b/backend/app/data/semantic_decompose.py index ef4175d5..952afae2 100644 --- a/backend/app/data/semantic_decompose.py +++ b/backend/app/data/semantic_decompose.py @@ -39,6 +39,7 @@ from app.data.semantic_metrics import ( create_metric, list_metrics, metric_to_dict, + probe_schema_lookup_and_discriminators, update_metric, ) from app.db.base import SessionLocal, utcnow @@ -101,16 +102,18 @@ CATALOG_DECOMPOSE_SYSTEM_PROMPT = """你是 AgentLab 的「数据目录与字段 3. changes[].path 仅支持以下规范路径： - 表级： · "label"：表中文名称（纯名词短语，勿写公式） - · "grain"：一行代表什么业务粒度（如 "每条产线每个班次每天一行"） + · "description"：表业务用途与多口径/多层级存储说明 + · "grain"：一行代表什么业务粒度（如 "每个门店每天一行" 或 "混存多种汇总层级与分类记录"） · "business_date"：{"column": "物理日期列名", "rule": "统计归属说明（可空串）"} - · "valid_filter"：默认有效记录 SQL 过滤片段（如 "is_deleted = 0"，不含 WHERE） - - 列级（把 替换为真实物理列名，如 ATTRIBUTE01）： - · "columns..label"：字段中文名或符号含义（纯名称，如 "计划开班时长"，严禁包含 =、/、*、SUM( 等计算公式符号！） - · "columns..unit"：单位（如 "秒"、"%"、"件"、"人次"） - · "columns..measure"：仅限 "flow"（流量，可跨日加总，如时长、产量）、"stock"（存量，不可跨期直接求和）、"ratio"（比率，如良品率）、"identifier"（编码标识）、"status"（状态码）、"attribute"（属性） - · "columns..meaning"：关键业务含义与存储说明（如 "底表直接存百分数如 99.65，引用时须除以 100，切勿重复乘 100"；注意文字中不要写等号公式，用自然中文表述） - · "columns..codes"：枚举码值字典，如 {"1": "正常", "0": "停用"} -4. 特别留意工业宽表中的盲列（如 ATTRIBUTE01～ATTRIBUTE55）：凡文档给出了字段对照或符号说明的，务必逐列提取 label、unit、measure 与 meaning！ + · "valid_filter"：默认有效记录 SQL 过滤片段（如 "is_deleted = 0"，不含 WHERE；仅填全表通用的有效条件，若不同指标查不同分类切勿把某一分类锁死在表级 valid_filter） + · "dedup"：去重与防重算说明（如 "同表混存汇总行与明细行，聚合前必须按层级或类型列过滤"） + - 列级（把 替换为真实物理列名）： + · "columns..label"：字段中文名或符号含义（纯名称，严禁包含 =、/、*、SUM( 等计算公式符号！） + · "columns..unit"：单位（如 "元"、"秒"、"%"、"件"、"人次"） + · "columns..measure"：仅限 "flow"（流量，可跨日加总）、"stock"（存量，不可跨期直接求和）、"ratio"（比率）、"identifier"（编码标识）、"status"（状态/分类码）、"attribute"（属性/分组码） + · "columns..meaning"：关键业务含义与存储说明（如百分数存储方式、跨分类字段复用含义等；注意文字中不要写等号公式，用自然中文表述） + · "columns..codes"：枚举码值字典，如 {"1": "正常", "0": "停用"}（务必使用主表实探出的精确码值字符串作为 key） +4. 同表异构、多粒度或字典驱动字段拆解准则：凡文档或【关联字典配置表与低基数分类列实探】给出了编码对照、层级说明或通用列映射的，务必完整提取分类列的 `codes`、度量列的 `label`/`unit`/`measure`/`meaning` 以及表级 `dedup` 或 `guardrail_notes` 防混算红线！ """ ENTITY_DECOMPOSE_SYSTEM_PROMPT = """你是 AgentLab 的「标准业务实体拆解专家」。请结合数据源物理表结构（含最新字段说明）与业务口径文档，识别多张表或业务提问中共用的核心维度实体（如产线、车间、机台、产品型号、不良原因、区域等），输出增量操作 JSON。 @@ -416,6 +419,7 @@ def _build_schema_prompt_block( masked_set: set[str], col_entity: Mapping[tuple[str, str], str], overlay_notes: Mapping[str, Mapping[str, Any]] | None = None, + probe_hints: str = "", ) -> str: lines: list[str] = [f"数据源：{ds.name}（方言：{ds.kind}）"] if (ds.kind or "").strip().lower() == "oracle": @@ -466,7 +470,7 @@ def _build_schema_prompt_block( col_notes = notes.get("columns") if isinstance(notes.get("columns"), Mapping) else {} t_short = name_key(tname.rsplit(".", 1)[-1]) - for col in (tinfo.get("columns") or [])[:75]: + for col in (tinfo.get("columns") or [])[:90]: if not isinstance(col, Mapping) or not col.get("name"): continue cname = str(col["name"]) @@ -474,6 +478,7 @@ def _build_schema_prompt_block( if name_key(cname) in masked_set: lines.append(f" · {cname} ({ctype}) [遮罩列]") continue + c_comment = str(col.get("comment") or "").strip() cn = col_notes.get(cname) if isinstance(col_notes, Mapping) else None c_label = (cn.get("label") or {}).get("value") if isinstance(cn, Mapping) and isinstance(cn.get("label"), Mapping) else "" c_unit = (cn.get("unit") or {}).get("value") if isinstance(cn, Mapping) and isinstance(cn.get("unit"), Mapping) else "" @@ -483,20 +488,28 @@ def _build_schema_prompt_block( if isinstance(cn, Mapping) and isinstance(cn.get("meaning"), Mapping) else ((cn.get("note") or {}).get("value") if isinstance(cn, Mapping) and isinstance(cn.get("note"), Mapping) else "") ) + c_codes_raw = (cn.get("codes") or {}).get("value") if isinstance(cn, Mapping) and isinstance(cn.get("codes"), Mapping) else None + c_codes_str = "" + if isinstance(c_codes_raw, Mapping) and c_codes_raw: + code_pairs = [f"{k}={v}" if v else str(k) for k, v in list(c_codes_raw.items())[:16]] + c_codes_str = f"码值：{'/'.join(code_pairs)}" ent_key = col_entity.get((t_short, name_key(cname)), "") extra_bits = [ b for b in ( - c_label, + c_label or c_comment, f"单位：{c_unit}" if c_unit else "", f"度量：{c_measure}" if c_measure else "", f"含义：{c_meaning}" if c_meaning else "", + c_codes_str, f"已绑定标准实体：{ent_key}" if ent_key else "", ) if b ] suffix = f" — {'，'.join(extra_bits)}" if extra_bits else "" lines.append(f" · {cname} ({ctype}){suffix}") + if probe_hints: + lines.append(probe_hints) return "\n".join(lines) @@ -690,7 +703,22 @@ async def _execute_job(job_id: str) -> None: doc_budget -= len(snippet) doc_sections.append(f"--- 文档《{bundle.title}》 ---\n{snippet}") + raw_notes_map = {t: e.notes for t, e in entries.items()} + visible_by_table = {t: catalog.visible_notes(e.notes) for t, e in entries.items()} + rel_graph = catalog.relation_graph(raw_notes_map, schema_cache=cache) + probe_hints = await probe_schema_lookup_and_discriminators( + ds, + tables, + selected_tables, + visible_by_table=visible_by_table, + rel_graph=rel_graph, + masked_set=masked_set, + bound_entity_cols=set(col_entity.keys()), + ) + prep_detail = f"已扫描 {len(selected_tables)} 张物理表" + if probe_hints: + prep_detail += "，实探关联配置维表与分类取值" if doc_sections: prep_detail += f"，载入 {len(doc_sections)} 份口径文档" if any(d.get("created_now") for d in kb_documents): @@ -722,7 +750,7 @@ async def _execute_job(job_id: str) -> None: if "catalog" in scopes: t_cat = time.perf_counter() - if not doc_sections and not (job.note or "").strip(): + if not doc_sections and not (job.note or "").strip() and not probe_hints: steps = _update_step_list( steps, "catalog", @@ -747,7 +775,7 @@ async def _execute_job(job_id: str) -> None: steps=steps, ) schema_block = _build_schema_prompt_block( - ds, tables, selected_tables, entries, masked_set, col_entity + ds, tables, selected_tables, entries, masked_set, col_entity, probe_hints=probe_hints ) cat_prompt_parts = [schema_block] if doc_sections: @@ -884,6 +912,7 @@ async def _execute_job(job_id: str) -> None: masked_set, col_entity, overlay_notes=overlay_notes, + probe_hints=probe_hints, ) met_lines = [enriched_schema_block] if existing_metric_keys: @@ -970,6 +999,7 @@ async def _execute_job(job_id: str) -> None: masked_set, col_entity, overlay_notes=overlay_notes, + probe_hints=probe_hints, ) ent_lines = [enriched_schema_block] if entities_list: diff --git a/backend/app/data/semantic_metrics.py b/backend/app/data/semantic_metrics.py index 2948ad32..4e65bad2 100644 --- a/backend/app/data/semantic_metrics.py +++ b/backend/app/data/semantic_metrics.py @@ -1313,6 +1313,7 @@ def compile_metric_sql( safe_limit = max(1, min(int(limit or 200), 1000)) metric_col_alias = _quote_ident(str(m.get("key") or "metric_value"), kind, is_alias=True) + desc_clause = _desc_order_clause(kind) if calc_mode == "sql_template" and sql_template: slots_used = {tok.group(1).strip().lower() for tok in _SLOT_TOKEN_RE.finditer(sql_template)} @@ -1333,14 +1334,14 @@ def compile_metric_sql( if group_exprs: sql += " GROUP BY " + ", ".join(group_exprs) if not order_parts: - order_parts.append(f"{len(select_parts)} DESC") + order_parts.append(f"{len(select_parts)} {desc_clause}") if order_parts: sql += " ORDER BY " + ", ".join(order_parts) sql = _apply_row_limit(sql, safe_limit, kind, force_if_missing=bool(group_exprs)) return sql, alignments if group_exprs and not tpl_order_parts: - tpl_order_parts.append(f"{len(dim_selects) + 1} DESC") + tpl_order_parts.append(f"{len(dim_selects) + 1} {desc_clause}") period_days_sql = _compute_period_days_sql( source=source, @@ -1392,13 +1393,21 @@ def compile_metric_sql( if group_exprs: sql += " GROUP BY " + ", ".join(group_exprs) if not order_parts: - order_parts.append(f"{len(select_parts)} DESC") + order_parts.append(f"{len(select_parts)} {desc_clause}") if order_parts: sql += " ORDER BY " + ", ".join(order_parts) sql = _apply_row_limit(sql, safe_limit, kind, force_if_missing=bool(group_exprs)) return sql, alignments +def _desc_order_clause(kind: str | None) -> str: + """Oracle / PostgreSQL 默认 DESC 将 NULL 排在最前（NULLS FIRST），外层套 ROWNUM 或 LIMIT 时会把有效数值行截掉。""" + k = (kind or "").strip().lower() + if k in ("oracle", "postgres", "postgresql", "sqlite"): + return "DESC NULLS LAST" + return "DESC" + + async def probe_latest_metric_date( source: DataSource, *, @@ -1496,6 +1505,392 @@ async def preview_metric( } +# -------------------------------------------------------------------------- +# 通用关联维表 / 字典配置表发现与低基数分类列实探（零行业/列名硬编码） +# -------------------------------------------------------------------------- + +_GENERIC_AUDIT_COL_KEYS = frozenset( + { + "id", + "rowid", + "createdat", + "updatedat", + "createtime", + "updatetime", + "createdby", + "updatedby", + "creator", + "updater", + "isdeleted", + "deleted", + "delflag", + "version", + "remark", + "remarks", + "note", + "notes", + "memo", + "description", + "sort", + "sortorder", + "seq", + } +) + + +def _sql_type_family(ctype: str) -> str: + t = (ctype or "").strip().lower() + if not t: + return "other" + if any(k in t for k in ("date", "time", "timestamp")): + return "temporal" + if any(k in t for k in ("char", "text", "string", "clob", "enum", "json")): + return "string" + if any(k in t for k in ("float", "double", "real")): + return "float" + if any(k in t for k in ("int", "serial", "bit")): + return "int" + if any(k in t for k in ("num", "dec", "money")): + return "int" if re.search(r"\(\s*\d+\s*,\s*0\s*\)", t) else "numeric" + return "other" + + +def _is_compact_lookup_table( + tinfo: Mapping[str, Any] | None, + *, + visible_notes: Mapping[str, Any] | None = None, + masked_set: set[str] | None = None, +) -> bool: + """判断物理表是否为紧凑的维表、码表、枚举字典或元数据配置表（纯结构与目录角色判定）。""" + if not isinstance(tinfo, Mapping): + return False + v_notes = visible_notes or {} + t_kind = str((v_notes.get("kind") or {}).get("value") or "").strip().lower() if isinstance(v_notes.get("kind"), Mapping) else "" + if t_kind in ("fact", "log", "snapshot"): + return False + masked = masked_set or set() + unmasked_cols = [ + c + for c in (tinfo.get("columns") or []) + if isinstance(c, Mapping) and c.get("name") and name_key(str(c["name"])) not in masked + ] + if t_kind in ("dimension", "config") and 2 <= len(unmasked_cols) <= 24: + return True + if not (2 <= len(unmasked_cols) <= 16): + return False + families = [_sql_type_family(str(c.get("type") or "")) for c in unmasked_cols] + str_cnt = sum(1 for f in families if f == "string") + num_measure_cnt = sum(1 for f in families if f in ("float", "numeric")) + temporal_cnt = sum(1 for f in families if f == "temporal") + return str_cnt >= 2 and str_cnt * 2 >= len(unmasked_cols) and num_measure_cnt <= 3 and temporal_cnt <= 2 + + +def find_companion_lookup_tables( + tables: Mapping[str, Any], + selected_tables: Sequence[str], + *, + visible_by_table: Mapping[str, Mapping[str, Any]] | None = None, + rel_graph: Mapping[str, Sequence[Any]] | None = None, + masked_set: set[str] | None = None, +) -> list[dict[str, Any]]: + """基于关系图（FK / 目录关系）与高区分度共享键（IDF），通用发现候选事实表关联的紧凑维表或字典配置表。""" + if not isinstance(tables, Mapping) or not tables or not selected_tables: + return [] + masked = masked_set or set() + vis_map = visible_by_table or {} + + col_table_freq: dict[str, int] = {} + table_cols_by_key: dict[str, dict[str, tuple[str, str]]] = {} + table_col_counts: dict[str, int] = {} + for tname, tinfo in tables.items(): + if not isinstance(tinfo, Mapping): + continue + raw_cols = [c for c in (tinfo.get("columns") or []) if isinstance(c, Mapping) and c.get("name")] + table_col_counts[str(tname)] = len(raw_cols) + ck_map: dict[str, tuple[str, str]] = {} + for c in raw_cols: + cname = str(c["name"]) + ck = name_key(cname) + if not ck or ck in masked or ck in _GENERIC_AUDIT_COL_KEYS: + continue + fam = _sql_type_family(str(c.get("type") or "")) + if fam in ("temporal", "float"): + continue + ck_map[ck] = (cname, fam) + table_cols_by_key[str(tname)] = ck_map + for ck in ck_map: + col_table_freq[ck] = col_table_freq.get(ck, 0) + 1 + + max_shared_freq = max(3, int(len(tables) * 0.45)) + lookup_candidates = { + str(tname) + for tname, tinfo in tables.items() + if _is_compact_lookup_table(tinfo, visible_notes=vis_map.get(str(tname)), masked_set=masked) + } + if not lookup_candidates: + return [] + + found_by_name: dict[str, dict[str, Any]] = {} + for src_tbl in selected_tables: + if src_tbl not in table_cols_by_key: + continue + src_col_cnt = table_col_counts.get(src_tbl, 0) + src_keys = table_cols_by_key[src_tbl] + + # 1) 显式关系图（外键、目录确认关系、命名推断关系）连接到的紧凑维表/配置表 + for edge in (rel_graph or {}).get(src_tbl, ()): + target = str(getattr(edge, "to_table", "") or "") + if not target or target == src_tbl or target not in lookup_candidates: + continue + from_cols = [ + str(c) + for c in (getattr(edge, "from_columns", None) or ()) + if name_key(str(c)) not in masked + ] + to_cols = [ + str(c) + for c in (getattr(edge, "to_columns", None) or ()) + if name_key(str(c)) not in masked + ] + if not from_cols or not to_cols: + continue + entry = found_by_name.get(target) + t_col_cnt = table_col_counts.get(target, 99) + score = (0 if getattr(edge, "status", "") in ("verified", "confirmed") else 1, t_col_cnt, target) + if entry is None or score < entry["_score"]: + found_by_name[target] = { + "table_name": target, + "linked_to": src_tbl, + "source_columns": from_cols, + "lookup_columns": to_cols, + "col_count": t_col_cnt, + "_score": score, + } + + # 2) 无外键视图 / 宽表与紧凑字典表之间的高区分度同名键匹配 + for target in lookup_candidates: + if target == src_tbl: + continue + t_col_cnt = table_col_counts.get(target, 99) + t_kind = ( + str(((vis_map.get(target) or {}).get("kind") or {}).get("value") or "").strip().lower() + if isinstance((vis_map.get(target) or {}).get("kind"), Mapping) + else "" + ) + if t_col_cnt >= src_col_cnt and t_kind not in ("dimension", "config"): + continue + tgt_keys = table_cols_by_key.get(target) or {} + shared_cks = [ + ck + for ck in src_keys + if ck in tgt_keys and col_table_freq.get(ck, 999) <= max_shared_freq + ] + if not shared_cks: + continue + shared_cks.sort(key=lambda ck: col_table_freq.get(ck, 999)) + best_freq = col_table_freq.get(shared_cks[0], 99) + score = (2, best_freq, t_col_cnt, target) + entry = found_by_name.get(target) + if entry is None or score < entry["_score"]: + found_by_name[target] = { + "table_name": target, + "linked_to": src_tbl, + "source_columns": [src_keys[ck][0] for ck in shared_cks[:2]], + "lookup_columns": [tgt_keys[ck][0] for ck in shared_cks[:2]], + "col_count": t_col_cnt, + "_score": score, + } + + results = sorted(found_by_name.values(), key=lambda x: x["_score"]) + for item in results: + item.pop("_score", None) + return results + + +async def probe_schema_lookup_and_discriminators( + ds: DataSource, + tables: Mapping[str, Any], + selected_tables: Sequence[str], + *, + visible_by_table: Mapping[str, Mapping[str, Any]] | None = None, + rel_graph: Mapping[str, Sequence[Any]] | None = None, + masked_set: set[str] | None = None, + bound_entity_cols: set[tuple[str, str]] | None = None, +) -> str: + """在提取口径/拆解目录时，通用实探候选表的低基数分类/粒度列取值与紧凑字典配置表内容，杜绝跨类混算与编码盲猜。""" + if not isinstance(tables, Mapping) or not tables or not selected_tables: + return "" + masked = masked_set or set() + vis_map = visible_by_table or {} + ent_cols = bound_entity_cols or set() + + companions = find_companion_lookup_tables( + tables, + selected_tables, + visible_by_table=vis_map, + rel_graph=rel_graph, + masked_set=masked, + ) + + # 收集候选低基数分类/粒度/类型鉴别列：(table_name, col_name, priority) + disc_candidates: list[tuple[int, str, str]] = [] + seen_disc: set[tuple[str, str]] = set() + + def _add_disc(prio: int, tbl: str, col_name: str) -> None: + key = (tbl, name_key(col_name)) + if not col_name or key[1] in masked or key[1] in _GENERIC_AUDIT_COL_KEYS or key in seen_disc: + return + seen_disc.add(key) + disc_candidates.append((prio, tbl, col_name)) + + for comp in companions: + if comp["col_count"] <= 10: + for sc in comp["source_columns"]: + _add_disc(0, comp["linked_to"], sc) + + for tname in selected_tables: + tinfo = tables.get(tname) + if not isinstance(tinfo, Mapping): + continue + if _is_compact_lookup_table(tinfo, visible_notes=vis_map.get(tname), masked_set=masked): + continue + v_notes = vis_map.get(tname) or {} + col_notes = v_notes.get("columns") if isinstance(v_notes.get("columns"), Mapping) else {} + raw_cols = [c for c in (tinfo.get("columns") or []) if isinstance(c, Mapping) and c.get("name")] + fams = [(str(c["name"]), _sql_type_family(str(c.get("type") or ""))) for c in raw_cols] + num_cnt = sum(1 for _, f in fams if f in ("float", "numeric", "int")) + str_cols = [n for n, f in fams if f == "string" and name_key(n) not in masked and name_key(n) not in _GENERIC_AUDIT_COL_KEYS] + + for cname, _ in fams: + cn = col_notes.get(cname) if isinstance(col_notes, Mapping) else None + c_meas = ( + str((cn.get("measure") or {}).get("value") or "").strip().lower() + if isinstance(cn, Mapping) and isinstance(cn.get("measure"), Mapping) + else "" + ) + if c_meas in ("status", "attribute"): + _add_disc(1, tname, cname) + + # 对典型异构表（键值窄表 1～2 个数值列，或多指标宽表 >=8 个数值列），抽样未绑定实体的字符分类列验证是否为低基数分区列 + if (1 <= num_cnt <= 2 and len(str_cols) >= 2) or (num_cnt >= 8 and str_cols): + t_short = name_key(tname.rsplit(".", 1)[-1]) + for sc in str_cols[:4]: + if (t_short, name_key(sc)) not in ent_cols: + _add_disc(2, tname, sc) + + disc_candidates.sort(key=lambda x: x[0]) + + sampled_disc_lines: list[str] = [] + for _, tname, cname in disc_candidates[:5]: + q_col = _format_col_ref(ds, tname, cname) + q_tbl = _format_table_ref(ds, tname) + sql = _apply_row_limit( + f"SELECT DISTINCT {q_col} FROM {q_tbl} WHERE {q_col} IS NOT NULL ORDER BY 1", + 21, + ds.kind, + force_if_missing=True, + ) + try: + res = await run_query(ds, sql, limits=QueryLimits(timeout_seconds=3, max_rows=21)) + except Exception: # noqa: BLE001 + continue + vals = [str(r[0]) for r in (res.rows or []) if r and r[0] is not None and str(r[0]).strip()] + # 仅保留 1～20 个不同值的真实低基数分类/粒度/状态列；高基数业务主键自动排除 + if 1 <= len(vals) <= 20 and not res.truncated: + lits = "、".join(_sql_literal(v) for v in vals) + sampled_disc_lines.append(f"- 表 `{tname}` 低基数分类/粒度列 `{cname}` 实探全部取值（共 {len(vals)} 种）：{lits}") + + lookup_lines: list[str] = [] + char_budget = 3400 + compact_dicts = [c for c in companions if c["col_count"] <= 10] + for comp in compact_dicts[:3]: + if char_budget <= 0: + break + lt_name = comp["table_name"] + lt_info = tables.get(lt_name) or {} + unmasked_cols = [ + str(c["name"]) + for c in (lt_info.get("columns") or []) + if isinstance(c, Mapping) and c.get("name") and name_key(str(c["name"])) not in masked + ][:10] + if len(unmasked_cols) < 2: + continue + sel_exprs = [_format_col_ref(ds, lt_name, c) for c in unmasked_cols] + q_lt = _format_table_ref(ds, lt_name) + sql = _apply_row_limit( + f"SELECT {', '.join(sel_exprs)} FROM {q_lt} ORDER BY 1, 2", + 160, + ds.kind, + force_if_missing=True, + ) + try: + lres = await run_query(ds, sql, limits=QueryLimits(timeout_seconds=3, max_rows=160)) + except Exception: # noqa: BLE001 + continue + if not lres.rows or len(lres.rows) > 150: + continue + + join_pair = " = ".join( + f"{comp['linked_to']}.{sc} ↔ {lt_name}.{lc}" + for sc, lc in zip(comp["source_columns"], comp["lookup_columns"]) + ) + lookup_lines.append( + f"- 关联字典/配置维表 `{lt_name}`（关联键：{join_pair}；字段：{', '.join(unmasked_cols)}）：" + ) + key_col_name = comp["lookup_columns"][0] if comp["lookup_columns"] else unmasked_cols[0] + key_idx = next((i for i, c in enumerate(unmasked_cols) if name_key(c) == name_key(key_col_name)), 0) + + if len(lres.rows) <= 12: + for r in lres.rows: + kv = "，".join( + f"{unmasked_cols[i]}={_sql_literal(r[i])}" + for i in range(min(len(unmasked_cols), len(r))) + if r[i] is not None and str(r[i]).strip() != "" + ) + if kv: + row_line = f" · {kv}" + lookup_lines.append(row_line) + char_budget -= len(row_line) + else: + grouped: dict[str, list[str]] = {} + other_idxs = [i for i in range(len(unmasked_cols)) if i != key_idx] + for r in lres.rows: + if key_idx >= len(r) or r[key_idx] is None: + continue + k_val = str(r[key_idx]).strip() + rest_vals = [ + str(r[i]).strip() + for i in other_idxs + if i < len(r) and r[i] is not None and str(r[i]).strip() != "" + ] + if not k_val or not rest_vals: + continue + item_s = " / ".join(rest_vals) + bucket = grouped.setdefault(k_val, []) + if item_s not in bucket: + bucket.append(item_s) + for k_val, items in grouped.items(): + line = f" · {unmasked_cols[key_idx]}={_sql_literal(k_val)}：{'；'.join(items[:32])}" + if len(line) > char_budget: + line = line[:char_budget] + "…" + lookup_lines.append(line) + char_budget -= len(line) + if char_budget <= 0: + break + + if not sampled_disc_lines and not lookup_lines: + return "" + + out = [ + "\n【关联字典配置表与低基数分类列实探】", + "若同一物理表存在多种记录类型、汇总层级或由字典表定义的字段分组，不同分类下度量列含义或粒度可能不同，严禁不加筛选直接全表混算：", + *sampled_disc_lines, + *lookup_lines, + "建模注意：若指标仅适用于特定业务分类、层级或配置组，必须在 `filter_sql`（单表模式）或 `WHERE`（模板模式）中使用主事实表实探出的精确字面值限定过滤条件（注意保留前导零或大小写，切勿臆造码值），并将核心业务层级列放入 `dimensions`。", + ] + return "\n".join(out) + + # -------------------------------------------------------------------------- # AI 辅助从数据目录与业务文档批量提取指标卡片 # -------------------------------------------------------------------------- @@ -1512,10 +1907,11 @@ SUGGEST_SYSTEM_PROMPT = """你是 AgentLab 的「语义指标建模专家」。 · sql_template 写单条只读 SELECT 或 WITH ... SELECT 语句，不含分号与 SQL 注释； · 仅允许使用以下安全槽位：{{select_dims}}（放在 SELECT 开头用于动态展开分组列）、{{and_where}}（放在 WHERE 1=1 之后承接日期与维度筛选）、{{where}}、{{group_by}}、{{order_by}}、{{period_days}}、{{date_from}}、{{date_to}}、{{limit}}； · 多表模板下，date_column 与 dimensions[].column 必须带主表别名前缀（如 "s.stat_date"、"s.line_code"），table_name 填主事实表名；若表旁标注了「SQL 全名：」，在 sql_template 的 FROM / JOIN 中务必使用该带 schema 前缀的全名； - · 严格遵循目标数据源方言语法：例如当方言为 oracle 时，严禁写 LIMIT（行数限制由系统自动套外层 ROWNUM）、严禁写 CAST(... AS TEXT)（转字符用 TO_CHAR、转数值用 TO_NUMBER）、表或子查询别名不要写 AS（写 FROM tbl t）。 -5. 若数据目录注明某度量列为「存量（stock，不可跨期直接求和）」，优先使用 AVG(...) 或在 description 中明确标注「存量指标，多日查询请开启按日分组查看趋势」。 -6. 若物理列旁标注了「已绑定标准实体：」，在对应 dimension 的 entity_key 字段填入该实体键。 -7. 避免与「已存在的指标标识」重复；提取文档或目录中最核心的 4～16 个业务指标（单表与多表复杂指标均可覆盖）。 + · 严格遵循目标数据源方言语法：例如当方言为 oracle 时，严禁写 LIMIT（行数限制由系统自动套外层 ROWNUM）、严禁写 CAST(... AS TEXT)（转字符用 TO_CHAR、转数值用 TO_NUMBER）、表或子查询别名不要写 AS（写 FROM tbl t），排序统一交由 {{order_by}} 槽位展开（自动带 DESC NULLS LAST）。 +5. 同表异构记录、多粒度与字典驱动表隔离准则：若主事实表混存多种业务类型、汇总层级、科目/指标代码或由关联配置字典表区分字段含义的分组记录，严禁不加过滤直接跨类混算！必须在 filter_sql（单表模式）或模板 WHERE 中使用主表真实存储的精确码值限定目标分类或层级，并将核心下钻维度列完整放入 dimensions。 +6. 若数据目录注明某度量列为「存量（stock，不可跨期直接求和）」，优先使用 AVG(...) 或在 description 中明确标注「存量指标，多日查询请开启按日分组查看趋势」。 +7. 若物理列旁标注了「已绑定标准实体：」，在对应 dimension 的 entity_key 字段填入该实体键。 +8. 避免与「已存在的指标标识」重复；提取文档或目录中最核心的 4～16 个业务指标（单表与多表复杂指标均可覆盖）。 单表指标示例： { @@ -1620,11 +2016,18 @@ async def suggest_metrics_from_source( if existing_keys: lines.append(f"已存在的指标标识（请勿重复）：{', '.join(sorted(existing_keys))}") lines.append("\n【物理表与数据目录】") + raw_notes_map = {t: e.notes for t, e in entries.items()} + visible_by_table = {t: catalog.visible_notes(e.notes) for t, e in entries.items()} + rel_graph = catalog.relation_graph(raw_notes_map, schema_cache=cache) + for tname in selected_tables: tinfo = tables.get(tname) or {} - notes = catalog.visible_notes(entries[tname].notes) if tname in entries else {} + notes = visible_by_table.get(tname) or {} t_label = (notes.get("label") or {}).get("value") if isinstance(notes.get("label"), Mapping) else "" + t_comment = str(tinfo.get("comment") or "").strip() + t_desc = (notes.get("description") or {}).get("value") if isinstance(notes.get("description"), Mapping) else "" t_grain = (notes.get("grain") or {}).get("value") if isinstance(notes.get("grain"), Mapping) else "" + t_dedup = (notes.get("dedup") or {}).get("value") if isinstance(notes.get("dedup"), Mapping) else "" b_date_raw = (notes.get("business_date") or {}).get("value") if isinstance(notes.get("business_date"), Mapping) else "" if isinstance(b_date_raw, Mapping): b_col = str(b_date_raw.get("column") or "").strip() @@ -1637,19 +2040,23 @@ async def suggest_metrics_from_source( qualified_t = str(tinfo.get("qualified") or "").strip() if qualified_t and qualified_t != tname: head_bits.append(f"SQL 全名：{qualified_t}") - if t_label: - head_bits.append(f"中文名：{t_label}") + if t_label or t_comment: + head_bits.append(f"中文名：{t_label or t_comment}") if t_grain: head_bits.append(f"粒度：{t_grain}") if b_date: head_bits.append(f"业务日期列：{b_date}") if v_filt: head_bits.append(f"有效过滤：{v_filt}") + if t_dedup: + head_bits.append(f"去重约束：{t_dedup}") + if t_desc: + head_bits.append(f"说明：{str(t_desc)[:120]}") lines.append("- " + " ｜ ".join(head_bits)) col_notes = notes.get("columns") if isinstance(notes.get("columns"), Mapping) else {} t_short = name_key(tname.rsplit(".", 1)[-1]) col_descs: list[str] = [] - for col in (tinfo.get("columns") or [])[:45]: + for col in (tinfo.get("columns") or [])[:90]: if not isinstance(col, Mapping) or not col.get("name"): continue cname = str(col["name"]) @@ -1657,21 +2064,42 @@ async def suggest_metrics_from_source( if name_key(cname) in masked_set: col_descs.append(f" · {cname} ({ctype}) [遮罩列]") continue + c_comment = str(col.get("comment") or "").strip() cn = col_notes.get(cname) if isinstance(col_notes, Mapping) else None c_label = (cn.get("label") or {}).get("value") if isinstance(cn, Mapping) and isinstance(cn.get("label"), Mapping) else "" c_unit = (cn.get("unit") or {}).get("value") if isinstance(cn, Mapping) and isinstance(cn.get("unit"), Mapping) else "" c_measure = (cn.get("measure") or {}).get("value") if isinstance(cn, Mapping) and isinstance(cn.get("measure"), Mapping) else "" + c_meaning = (cn.get("meaning") or {}).get("value") if isinstance(cn, Mapping) and isinstance(cn.get("meaning"), Mapping) else "" + c_codes_raw = (cn.get("codes") or {}).get("value") if isinstance(cn, Mapping) and isinstance(cn.get("codes"), Mapping) else None + c_codes_str = "" + if isinstance(c_codes_raw, Mapping) and c_codes_raw: + code_pairs = [f"{k}={v}" if v else str(k) for k, v in list(c_codes_raw.items())[:16]] + c_codes_str = f"码值：{'/'.join(code_pairs)}" ent_key = col_entity.get((t_short, name_key(cname)), "") extra_bits = [b for b in ( - c_label, + c_label or c_comment, f"单位：{c_unit}" if c_unit else "", f"度量：{c_measure}" if c_measure else "", + f"含义：{str(c_meaning)[:100]}" if c_meaning else "", + c_codes_str, f"已绑定标准实体：{ent_key}" if ent_key else "", ) if b] suffix = f" — {'，'.join(extra_bits)}" if extra_bits else "" col_descs.append(f" · {cname} ({ctype}){suffix}") lines.extend(col_descs) + probe_hints = await probe_schema_lookup_and_discriminators( + ds, + tables, + selected_tables, + visible_by_table=visible_by_table, + rel_graph=rel_graph, + masked_set=masked_set, + bound_entity_cols=set(col_entity.keys()), + ) + if probe_hints: + lines.append(probe_hints) + doc_titles: list[str] = [] if document_ids: docs = list( diff --git a/backend/app/engine/nodes/report.py b/backend/app/engine/nodes/report.py index 56cd14bc..ca4e1d85 100644 --- a/backend/app/engine/nodes/report.py +++ b/backend/app/engine/nodes/report.py @@ -803,7 +803,7 @@ async def _store(doc: dict[str, Any], ctx: NodeContext) -> str | None: async def _run_fast_inline_report(state: GraphState, ctx: NodeContext, on_violation: str) -> dict[str, Any]: """快速直查专用零模型直通证据封装：0 次 LLM 调用、< 10ms 纯内存组装 report_doc 与封存事件。""" from app.api.evidence import withheld_metrics - from app.chat.direct_evidence import normalize_and_bind_fast_citations + from app.chat.direct_evidence import collect_fast_allow_numbers, normalize_and_bind_fast_citations from app.engine.evidence import iter_segments started = time.perf_counter() @@ -855,6 +855,17 @@ async def _run_fast_inline_report(state: GraphState, ctx: NodeContext, on_violat prepared = normalize_and_bind_fast_citations(_unfenced(raw_answer), catalog, masked_columns=query_masks) entities = report_entities(spec, ctx.node) == "link" allow = report_allowance(spec, ctx.node) + if str(ctx.cfg("numbers", "off") or "off") == "off": + raw_input = state.get("input") if isinstance(state.get("input"), Mapping) else {} + question_str = str(raw_input.get("question") or "") + fast_allow = collect_fast_allow_numbers( + prepared, + catalog, + masked_columns=query_masks, + question=question_str, + ) + if fast_allow: + allow = list(dict.fromkeys([*(allow or []), *fast_allow])) doc = await masked( compose_doc( prepared, diff --git a/backend/app/providers/catalog.py b/backend/app/providers/catalog.py index a610c82d..b4f3fe40 100644 --- a/backend/app/providers/catalog.py +++ b/backend/app/providers/catalog.py @@ -201,6 +201,12 @@ def supports_adaptive_thinking(model: str) -> bool: return _norm(model).startswith(_ADAPTIVE_THINKING_PREFIXES) +def requires_thinking(model: str) -> bool: + """该模型是否强制始终开启思考（如 catalog 中标注 thinking='always' 的模型，传 disabled 会被服务方拒绝）。""" + target = _norm(model) + return any(_norm(entry["id"]) == target and entry.get("thinking") == "always" for entry in ANTHROPIC_MODELS) + + def supports_effort(model: str) -> bool: return _norm(model).startswith(_EFFORT_PREFIXES) diff --git a/backend/app/providers/factory.py b/backend/app/providers/factory.py index 4389225e..5ed3aa85 100644 --- a/backend/app/providers/factory.py +++ b/backend/app/providers/factory.py @@ -276,6 +276,7 @@ def build_chat_model(provider: Provider, spec: ModelSpec) -> BaseChatModel: model_kwargs: dict[str, Any] = {} thinking_on = False mode = spec.thinking or "summarized" + effort = spec.effort if catalog.supports_adaptive_thinking(model): if mode in ("adaptive", "summarized"): @@ -286,7 +287,13 @@ def build_chat_model(provider: Provider, spec: ModelSpec) -> BaseChatModel: } thinking_on = True elif mode == "off": - kwargs["thinking"] = {"type": "disabled"} + if catalog.requires_thinking(model): + # 强制思考型模型不接受 disabled：保留隐藏式 adaptive，同时把推理档位压到 low + kwargs["thinking"] = {"type": "adaptive", "display": "omitted"} + thinking_on = True + effort = effort or "low" + else: + kwargs["thinking"] = {"type": "disabled"} elif mode != "off" and catalog.needs_interleaved_beta(model): # 光挂 header 是不够的：这一代默认根本没开 thinking， # 没有思考可交错，header 就是一条什么都不做的空设置 @@ -306,8 +313,8 @@ def build_chat_model(provider: Provider, spec: ModelSpec) -> BaseChatModel: if thinking_on: kwargs.pop("temperature", None) - if spec.effort and catalog.supports_effort(model): - model_kwargs["output_config"] = {"effort": spec.effort} + if effort and catalog.supports_effort(model): + kwargs["output_config"] = {"effort": effort} if model_kwargs: kwargs["model_kwargs"] = model_kwargs @@ -339,11 +346,59 @@ def build_chat_model(provider: Provider, spec: ModelSpec) -> BaseChatModel: # 兼容接口都填了 base_url，于是一次运行上百次调用记成 0 token、0 美元——llm.end 读的 usage_metadata # 就来自这一块。不认 stream_options 的服务由 ReasoningAwareChatOpenAI 去掉它重发，不让调用失败 kwargs["stream_usage"] = True - return _ReasoningAwareChatOpenAI(**kwargs) + reasoning_hint = _openai_reasoning_hint(provider.kind, model, spec) + return _ReasoningAwareChatOpenAI(reasoning_hint=reasoning_hint, **kwargs) + + +def _openai_reasoning_hint(provider_kind: str, model: str, spec: ModelSpec) -> dict[str, Any]: + """按模型家族生成 OpenAI / OpenAI 兼容接口的思考开关或推理强度参数。 + + 仅当调用方显式指定了 spec.thinking == 'off' 或 spec.effort 时才生成提示参数； + 默认未指定时返回空字典，完全不改变常规节点调用体。若目标网关不认该参数（400/422）， + ReasoningAwareChatOpenAI 会在尚未收到首块响应时自动剥离重发一次并记忆该端点+模型。 + """ + if spec.thinking != "off" and not spec.effort: + return {} + norm = catalog._norm(model).lower() + if spec.thinking == "off": + if norm.startswith("qwen"): + return { + "extra_body": { + "enable_thinking": False, + "chat_template_kwargs": {"enable_thinking": False}, + } + } + if norm.startswith(("glm", "kimi", "deepseek", "doubao")): + return {"extra_body": {"thinking": {"type": "disabled"}}} + if provider_kind == "openai": + if norm.startswith(("gpt-5", "o1", "o3", "o4")): + return {"reasoning_effort": spec.effort or "low"} + return {} + return {"reasoning_effort": spec.effort or "low"} + + if spec.effort: + if provider_kind == "openai" and not norm.startswith(("gpt-5", "o1", "o3", "o4")): + return {} + return {"reasoning_effort": spec.effort} + return {} + + +def _merge_reasoning_hint(kwargs: dict[str, Any], hint: dict[str, Any]) -> dict[str, Any]: + out = dict(kwargs) + for k, v in hint.items(): + if k == "extra_body" and isinstance(v, dict): + existing = out.get("extra_body") + merged_eb = dict(v) + if isinstance(existing, dict): + merged_eb.update(existing) + out["extra_body"] = merged_eb + elif k not in out: + out[k] = v + return out def _reasoning_aware_openai() -> type: - """ChatOpenAI 的子类，保留供应商的思考内容。 + """ChatOpenAI 的子类，保留供应商的思考内容并支持安全降级思考控制参数。 langchain 的 ChatOpenAI 只认官方 OpenAI 规范，第三方加的 reasoning_content 是**明确丢弃**的（它的文档原话："Non-standard response fields added by @@ -360,8 +415,11 @@ def _reasoning_aware_openai() -> type: 只加字段，不改 langchain 对正文的任何处理。 """ from langchain_openai import ChatOpenAI + from pydantic import PrivateAttr class ReasoningAwareChatOpenAI(ChatOpenAI): # type: ignore[misc] + _reasoning_hint: dict[str, Any] = PrivateAttr(default_factory=dict) + def _convert_chunk_to_generation_chunk( self, chunk: dict, default_chunk_class: type, base_generation_info: dict | None ): @@ -378,13 +436,89 @@ def _reasoning_aware_openai() -> type: gen.message.additional_kwargs["reasoning_content"] = text return gen + async def _agenerate(self, messages: Any, stop: Any = None, run_manager: Any = None, **kwargs: Any): + hint_key = (_endpoint_key(self), str(getattr(self, "model_name", "") or "")) + use_hint = bool(self._reasoning_hint) and hint_key not in _NO_REASONING_HINT + if not use_hint: + return await super()._agenerate(messages, stop, run_manager, **kwargs) + try: + return await super()._agenerate( + messages, stop, run_manager, **_merge_reasoning_hint(kwargs, self._reasoning_hint) + ) + except Exception as e: # noqa: BLE001 + if not _rejects_reasoning_hint(e): + raise + first = e + try: + res = await super()._agenerate(messages, stop, run_manager, **kwargs) + _NO_REASONING_HINT.add(hint_key) + logger.warning("模型接入 %s/%s 不支持思考控制参数（%s），已自动剥离", hint_key[0], hint_key[1], str(first)[:200]) + return res + except Exception as e: + raise e from first + async def _astream(self, messages: Any, stop: Any = None, run_manager: Any = None, **kwargs: Any): - """流式调用；服务方不认 stream_options 时去掉它重发一次，之后这个接口不再带它。 + """流式调用；服务方不认思考控制参数或 stream_options 时去掉它重发一次，之后这个接口不再带它。 只在还没收到任何一块时重发：拒绝都发生在请求刚发出去的时候，重发不会重复输出。""" usage = not self._use_responses_api({**kwargs, **self.model_kwargs}) \ and self._should_stream_usage(kwargs.get("stream_usage"), **kwargs) key = _endpoint_key(self) + hint_key = (key, str(getattr(self, "model_name", "") or "")) + use_hint = bool(self._reasoning_hint) and hint_key not in _NO_REASONING_HINT + + if use_hint: + started = False + hint_kwargs = _merge_reasoning_hint(kwargs, self._reasoning_hint) + if not usage or key in _NO_STREAM_USAGE: + hint_kwargs = {k: v for k, v in hint_kwargs.items() if k != "stream_options"} | {"stream_usage": False} + try: + async for chunk in super()._astream(messages, stop, run_manager, **hint_kwargs): + started = True + yield chunk + return + except Exception as e: # noqa: BLE001 + if started: + raise + if _rejects_stream_options_specifically(e) and usage and key not in _NO_STREAM_USAGE: + # 明确点名拒绝 stream_options：剥离 stream_options 但保留思考控制参数再试 + no_usage_hint_kwargs = { + k: v for k, v in hint_kwargs.items() if k != "stream_options" + } | {"stream_usage": False} + try: + async for chunk in super()._astream(messages, stop, run_manager, **no_usage_hint_kwargs): + if key not in _NO_STREAM_USAGE: + _NO_STREAM_USAGE.add(key) + logger.warning( + "模型接入 %s 不支持流式用量回传（%s），已改为不请求用量", key, str(e)[:200] + ) + yield chunk + return + except Exception: # noqa: BLE001 + pass + if not _rejects_reasoning_hint(e): + raise + # 剥离思考控制参数后走下方常规流式逻辑；若成则记入 _NO_REASONING_HINT + yielded_after_strip = False + async for chunk in self._astream_without_hint(messages, stop, run_manager, usage=usage, key=key, **kwargs): + if not yielded_after_strip and hint_key not in _NO_REASONING_HINT: + _NO_REASONING_HINT.add(hint_key) + logger.warning( + "模型接入 %s/%s 不支持思考控制参数（%s），已自动剥离", + hint_key[0], + hint_key[1], + str(e)[:200], + ) + yielded_after_strip = True + yield chunk + return + + async for chunk in self._astream_without_hint(messages, stop, run_manager, usage=usage, key=key, **kwargs): + yield chunk + + async def _astream_without_hint( + self, messages: Any, stop: Any = None, run_manager: Any = None, *, usage: bool, key: str, **kwargs: Any + ): first: Exception | None = None if usage and key not in _NO_STREAM_USAGE: started = False @@ -416,12 +550,32 @@ def _reasoning_aware_openai() -> type: #: 认定不支持 stream_options 的接口（base_url）。进程内有效：重启后再试一次，服务方可能已经升级 _NO_STREAM_USAGE: set[str] = set() +#: 认定不支持思考控制参数（reasoning_effort / enable_thinking / thinking）的 (base_url, model) +_NO_REASONING_HINT: set[tuple[str, str]] = set() def _endpoint_key(model: Any) -> str: return str(getattr(model, "openai_api_base", None) or "https://api.openai.com/v1") +def _rejects_stream_options_specifically(exc: Exception) -> bool: + text = str(exc).lower() + return "stream_options" in text or "include_usage" in text + + +def _rejects_reasoning_hint(exc: Exception) -> bool: + """报错是否是服务方拒绝了思考开关或 reasoning_effort 参数（排除上下文超长与 max_tokens 额度拒绝）。""" + from langchain_core.exceptions import ContextOverflowError + from app.providers.budget import budget_refused + + if isinstance(exc, ContextOverflowError) or budget_refused(exc): + return False + text = str(exc).lower() + if any(w in text for w in ("reasoning_effort", "enable_thinking", "chat_template_kwargs", "thinking", "unsupportedparams")): + return True + return getattr(exc, "status_code", None) in (400, 422) + + def _rejects_stream_options(exc: Exception) -> bool: """这个报错是不是服务方不认 stream_options。 @@ -441,8 +595,11 @@ def _rejects_stream_options(exc: Exception) -> bool: class _LazyReasoningChatOpenAI: """延迟导入 langchain_openai：它加载得慢，而 mock/anthropic 路径用不上。""" - def __call__(self, **kwargs: Any) -> Any: - return _reasoning_aware_openai()(**kwargs) + def __call__(self, *, reasoning_hint: dict[str, Any] | None = None, **kwargs: Any) -> Any: + inst = _reasoning_aware_openai()(**kwargs) + if reasoning_hint: + inst._reasoning_hint = dict(reasoning_hint) + return inst _ReasoningAwareChatOpenAI = _LazyReasoningChatOpenAI() diff --git a/backend/app/tools/metrics.py b/backend/app/tools/metrics.py index 2e5f0710..395353b6 100644 --- a/backend/app/tools/metrics.py +++ b/backend/app/tools/metrics.py @@ -12,16 +12,21 @@ """ from __future__ import annotations +from collections.abc import Mapping import json from typing import Any from pydantic import BaseModel, Field, model_validator +from app.data import catalog +from app.data.engine import masked_columns from app.data.entities import list_entities +from app.data.names import name_key from app.data.semantic_metrics import ( MetricInvalid, MetricNotFound, compile_metric_sql, + find_companion_lookup_tables, get_metric, is_empty_metric_result, probe_latest_metric_date, @@ -159,7 +164,52 @@ async def run_metric_query( has_groups = bool(group_by_date or group_by) rows_data = payload.get("rows") if isinstance(payload.get("rows"), list) else [] - if is_empty_metric_result(rows_data, has_groups=has_groups) and row.date_column: + all_metric_null = bool( + has_groups + and rows_data + and all(isinstance(r, (list, tuple)) and len(r) >= 2 and r[-1] is None for r in rows_data) + ) + if all_metric_null: + payload["all_metric_values_null"] = True + payload["null_group_row_count"] = len(rows_data) + if len(rows_data) > 5: + payload["rows"] = rows_data[:5] + + is_empty_or_null = is_empty_metric_result(rows_data, has_groups=has_groups) or all_metric_null + cache = ds.schema_cache if isinstance(ds.schema_cache, Mapping) else {} + tables_map = (cache.get("tables") or {}) if isinstance(cache, Mapping) else {} + if is_empty_or_null and isinstance(tables_map, Mapping) and row.table_name: + resolved_tbl = catalog.resolve_table_name(cache, row.table_name) or row.table_name + masked_set = {name_key(c) for c in masked_columns(ds.options)} + entries = await catalog.read_catalog(session, ds.id) + vis_by_tbl = {t: catalog.visible_notes(e.notes) for t, e in entries.items()} + v_notes = vis_by_tbl.get(resolved_tbl) or {} + v_filt = str((v_notes.get("valid_filter") or {}).get("value") or "").strip() if isinstance(v_notes.get("valid_filter"), Mapping) else "" + v_dedup = str((v_notes.get("dedup") or {}).get("value") or "").strip() if isinstance(v_notes.get("dedup"), Mapping) else "" + companions = find_companion_lookup_tables( + tables_map, + [resolved_tbl], + visible_by_table=vis_by_tbl, + masked_set=masked_set, + ) + hint_bits: list[str] = [] + if v_filt and v_filt.lower() not in (row.filter_sql or "").lower(): + hint_bits.append(f"目录有效过滤 `{v_filt}`") + if v_dedup: + hint_bits.append(f"去重与分层提示「{v_dedup}」") + if companions: + comp_desc = "、".join( + f"`{c['table_name']}`（关联列 `{', '.join(c['source_columns'])}`）" + for c in companions[:2] + ) + hint_bits.append(f"关联维表或配置表 {comp_desc}") + if hint_bits: + payload["diagnostic_hint"] = ( + f"本次指标结果为空或全为 NULL；底表 `{resolved_tbl}` 存在 {'；'.join(hint_bits)}，" + "若需排查分类口径或维度映射，请在单轮内并行联查核实。" + ) + + if is_empty_or_null and row.date_column: latest_date = await probe_latest_metric_date( ds, table_name=row.table_name, diff --git a/backend/tests/test_fast_ask_general_optimizations.py b/backend/tests/test_fast_ask_general_optimizations.py new file mode 100644 index 00000000..44120df3 --- /dev/null +++ b/backend/tests/test_fast_ask_general_optimizations.py @@ -0,0 +1,509 @@ +"""快速直查通用性能优化、异构表/关联字典通用探查与零延迟数字证据防误报回归测试。""" +from __future__ import annotations + +import json +import sqlite3 +from pathlib import Path +from typing import Any +from unittest.mock import AsyncMock + +import httpx +import openai +import pytest +import sqlglot +from langchain_core.messages import AIMessageChunk, HumanMessage +from langchain_core.outputs import ChatGeneration, ChatGenerationChunk, ChatResult + +from app.chat.direct import build_direct_graph +from app.chat.direct_evidence import collect_fast_allow_numbers, normalize_and_bind_fast_citations +from app.chat.grounding import GroundingBundle, ground_question +from app.data import catalog +from app.data.guard import check as guard_check +from app.data.semantic_metrics import ( + compile_metric_sql, + create_metric, + find_companion_lookup_tables, + probe_schema_lookup_and_discriminators, +) +from app.core.crypto import encrypt +from app.db.base import SessionLocal +from app.db.models import DataSource, Provider +from app.engine.evidence import compose_doc +from app.providers.factory import ( + ModelSpec, + _NO_REASONING_HINT, + _ReasoningAwareChatOpenAI, + _openai_reasoning_hint, + build_chat_model, +) +from app.tools.metrics import run_metric_query +from app.tools.registry import ToolContext, Versions +from tests.fixtures.sources import drop_source, sqlite_source + + +# --------------------------------------------------------------------------- +# 1. 跨模型网关关闭深度思考（thinking="off" / effort="low"）与 400 自动回退 +# --------------------------------------------------------------------------- + + +def test_fast_direct_graph_disables_thinking_and_caps_steps() -> None: + graph = build_direct_graph(bundle=GroundingBundle(), has_metrics=True) + agent_node = next(n for n in graph["nodes"] if n["id"] == "agent1") + cfg = agent_node["data"]["config"] + assert cfg["thinking"] == "off" + assert cfg["effort"] == "low" + assert cfg["max_steps"] == 4 + assert cfg["parallel_tools"] is True + + +def test_provider_thinking_off_payloads_across_vendors() -> None: + anth_prov = Provider(name="anth-test", kind="anthropic", api_key=encrypt("sk-test"), enabled=True) + + # Anthropic 普通模型：显式 disabled + effort=low + anth_std = build_chat_model( + anth_prov, + ModelSpec(model="claude-sonnet-4-6", thinking="off", effort="low"), + ) + assert anth_std.thinking == {"type": "disabled"} + assert anth_std.output_config == {"effort": "low"} + + # Anthropic 强制思考模型（catalog 中标注 thinking='always'）：不发 disabled，仅设为隐藏式 adaptive + effort=low + anth_req = build_chat_model( + anth_prov, + ModelSpec(model="claude-fable-5-1", thinking="off", effort="low"), + ) + assert anth_req.thinking == {"type": "adaptive", "display": "omitted"} + assert anth_req.output_config == {"effort": "low"} + + # OpenAI 兼容网关各主流家族推理开关映射 + off_spec = ModelSpec(thinking="off", effort="low") + qwen_hint = _openai_reasoning_hint("openai_compatible", "qwen3-235b-a22b", off_spec) + assert qwen_hint["extra_body"]["enable_thinking"] is False + + ds_hint = _openai_reasoning_hint("openai_compatible", "deepseek-v3.2", off_spec) + assert ds_hint["extra_body"] == {"thinking": {"type": "disabled"}} + + gpt_hint = _openai_reasoning_hint("openai", "gpt-5", off_spec) + assert gpt_hint == {"reasoning_effort": "low"} + + +@pytest.mark.asyncio +async def test_openai_compatible_gateway_strips_rejected_reasoning_hint_and_caches(monkeypatch: pytest.MonkeyPatch) -> None: + _NO_REASONING_HINT.clear() + seen_kwargs: list[dict[str, Any]] = [] + + async def fake_agenerate(self: Any, messages: Any, stop: Any = None, run_manager: Any = None, **kwargs: Any) -> ChatResult: + seen_kwargs.append(dict(kwargs)) + if "reasoning_effort" in kwargs or "extra_body" in kwargs: + req = httpx.Request("POST", "https://strict-gateway.example/v1/chat/completions") + resp = httpx.Response( + 400, + request=req, + json={"error": {"message": "Unrecognized request argument supplied: reasoning_effort"}}, + ) + raise openai.BadRequestError("Unrecognized request argument supplied: reasoning_effort", response=resp, body=resp.json()) + from langchain_core.messages import AIMessage + + return ChatResult(generations=[ChatGeneration(message=AIMessage(content="ok"))]) + + monkeypatch.setattr("langchain_openai.ChatOpenAI._agenerate", fake_agenerate) + + llm = _ReasoningAwareChatOpenAI( + model="custom-corp-llm", + api_key="sk-test", + base_url="https://strict-gateway.example/v1", + reasoning_hint={"reasoning_effort": "low"}, + ) + res1 = await llm._agenerate([HumanMessage(content="hi")]) + assert res1.generations[0].message.content == "ok" + assert len(seen_kwargs) == 2 + assert seen_kwargs[0].get("reasoning_effort") == "low" + assert "reasoning_effort" not in seen_kwargs[1] + + # 第二次调用命中进程级缓存，不再触发 400 重试 + seen_kwargs.clear() + await llm._agenerate([HumanMessage(content="hi again")]) + assert len(seen_kwargs) == 1 + assert "reasoning_effort" not in seen_kwargs[0] + + +@pytest.mark.asyncio +async def test_openai_stream_strips_rejected_reasoning_hint(monkeypatch: pytest.MonkeyPatch) -> None: + _NO_REASONING_HINT.clear() + calls = 0 + + async def fake_astream(self: Any, messages: Any, stop: Any = None, run_manager: Any = None, **kwargs: Any): + nonlocal calls + calls += 1 + if "extra_body" in kwargs: + req = httpx.Request("POST", "https://strict-stream.example/v1/chat/completions") + resp = httpx.Response( + 422, + request=req, + json={"detail": "Extra inputs are not permitted: enable_thinking"}, + ) + raise openai.UnprocessableEntityError("Extra inputs are not permitted", response=resp, body=resp.json()) + yield ChatGenerationChunk(message=AIMessageChunk(content="fast-answer")) + + monkeypatch.setattr("langchain_openai.ChatOpenAI._astream", fake_astream) + + llm = _ReasoningAwareChatOpenAI( + model="qwen3-coder", + api_key="sk-test", + base_url="https://strict-stream.example/v1", + reasoning_hint={"extra_body": {"enable_thinking": False}}, + ) + chunks = [c async for c in llm._astream([HumanMessage(content="hello")])] + assert calls == 2 + assert "".join(str(c.message.content) for c in chunks) == "fast-answer" + + +# --------------------------------------------------------------------------- +# 2. Oracle / PostgreSQL 分组指标 DESC NULLS LAST 方言修复与稀疏聚合验证 +# --------------------------------------------------------------------------- + + +def test_compile_metric_sql_nulls_last_by_dialect() -> None: + metric_def = { + "key": "sparse_ratio", + "name": "稀疏达成率", + "calc_mode": "expression", + "table_name": "fact_perf", + "sql_expression": "ROUND(SUM(actual_val) * 100.0 / NULLIF(SUM(target_val), 0), 2)", + "date_column": "stat_date", + "filter_sql": "", + "dimensions": [{"key": "unit_id", "label": "单元", "column": "unit_id"}], + } + schema = { + "tables": { + "fact_perf": { + "qualified": "DW.fact_perf", + "columns": [ + {"name": "unit_id", "type": "VARCHAR2(32)"}, + {"name": "stat_date", "type": "DATE"}, + {"name": "actual_val", "type": "NUMBER"}, + {"name": "target_val", "type": "NUMBER"}, + ], + } + } + } + + ora_ds = DataSource(id="ora1", name="ora_dw", kind="oracle", schema_cache=schema) + ora_sql, _ = compile_metric_sql(metric_def, source=ora_ds, group_by=["unit_id"], limit=50) + assert "ORDER BY 2 DESC NULLS LAST" in ora_sql + assert "WHERE ROWNUM <= 50" in ora_sql + guard_check(ora_sql, readonly=True, dialect="oracle", source_name="ora_dw") + sqlglot.parse_one(ora_sql, read="oracle") + + pg_ds = DataSource(id="pg1", name="pg_dw", kind="postgresql", schema_cache=schema) + pg_sql, _ = compile_metric_sql(metric_def, source=pg_ds, group_by=["unit_id"], limit=50) + assert "ORDER BY 2 DESC NULLS LAST LIMIT 50" in pg_sql + sqlglot.parse_one(pg_sql, read="postgres") + + mysql_ds = DataSource(id="my1", name="my_dw", kind="mysql", schema_cache=schema) + my_sql, _ = compile_metric_sql(metric_def, source=mysql_ds, group_by=["unit_id"], limit=50) + assert "ORDER BY 2 DESC LIMIT 50" in my_sql + assert "NULLS LAST" not in my_sql + sqlglot.parse_one(my_sql, read="mysql") + + +# --------------------------------------------------------------------------- +# 3. 通用关联维表 / 字典配置表发现、低基数分类列实探与全空分组折叠诊断 +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_general_companion_lookup_and_discriminator_probing_across_domains(tmp_path: Path) -> None: + db_file = tmp_path / "general_erp.sqlite" + conn = sqlite3.connect(db_file) + conn.executescript( + """ + CREATE TABLE fact_ledger_wide ( + tx_id TEXT, + org_code TEXT, + book_date TEXT, + entry_category TEXT, + secret_tag TEXT, + val_01 REAL, + val_02 REAL, + val_03 REAL, + val_04 REAL, + val_05 REAL, + val_06 REAL, + val_07 REAL, + val_08 REAL + ); + CREATE TABLE meta_category_schema ( + entry_category TEXT, + category_title TEXT, + slot_col TEXT, + slot_label TEXT, + slot_unit TEXT + ); + CREATE TABLE dim_org_unit ( + org_code TEXT, + org_name TEXT, + region_name TEXT + ); + """ + ) + # 插入带前导零的分类码 '009' 与 '003'，以及 >20 种高基数流水号 tx_id（验证高基数列自动排除） + for i in range(25): + cat = "009" if i < 5 else "003" + v1 = 100.0 if cat == "009" else None + v2 = 12.0 if cat == "009" else None + conn.execute( + "INSERT INTO fact_ledger_wide VALUES (?, ?, ?, ?, ?, ?, ?, NULL, NULL, NULL, NULL, NULL, NULL)", + (f"TX-{i:04d}", "3950" if i % 2 == 0 else "3951", "2026-11-11", cat, f"SEC-{i}", v1, v2), + ) + conn.executemany( + "INSERT INTO meta_category_schema VALUES (?, ?, ?, ?, ?)", + [ + ("003", "能耗台账", "val_01", "总电量", "kWh"), + ("009", "运营统计", "val_01", "应出勤秒数", "秒"), + ("009", "运营统计", "val_02", "异常中断秒数", "秒"), + ], + ) + conn.executemany( + "INSERT INTO dim_org_unit VALUES (?, ?, ?)", + [("3950", "一分部", "华东"), ("3951", "二分部", "华东")], + ) + conn.commit() + conn.close() + + ds_id = await sqlite_source( + "erp_general_test", + db_file, + options={"mask_columns": ["secret_tag"]}, + schema=True, + ) + try: + async with SessionLocal() as session: + ds = await session.get(DataSource, ds_id) + assert ds is not None + schema_cache = ds.schema_cache + + # 1) 通用关联紧凑维表发现（无外键约束，纯靠高区分度共享键与结构识别） + tables_map = schema_cache["tables"] + masked = {"secret_tag"} + companions = find_companion_lookup_tables( + tables_map, + ["fact_ledger_wide"], + masked_set=masked, + ) + comp_names = [c["table_name"] for c in companions] + assert "meta_category_schema" in comp_names + assert "dim_org_unit" in comp_names + + # 2) 低基数分类列与紧凑字典实探：保留 '009' 前导零，高基数 tx_id 与遮罩列 secret_tag 绝不混入 + hints = await probe_schema_lookup_and_discriminators( + ds, + tables_map, + ["fact_ledger_wide"], + masked_set=masked, + ) + assert "'009'" in hints + assert "'003'" in hints + assert "异常中断秒数" in hints + assert "TX-0001" not in hints + assert "secret_tag" not in hints + assert "SEC-" not in hints + + # 3) 语义预命中自动捎带紧凑关联配置表，且普通使用者模式不泄露物理表名 + metric_row, _ = await create_metric( + session, + { + "key": "erp_interrupt_rate", + "name": "异常中断率", + "source_id": ds.id, + "calc_mode": "expression", + "table_name": "fact_ledger_wide", + "sql_expression": "ROUND(SUM(val_02) * 100.0 / NULLIF(SUM(val_01), 0), 2)", + "date_column": "book_date", + "filter_sql": "", + "dimensions": [{"key": "tx_id", "label": "流水", "column": "tx_id"}], + }, + ) + await catalog.write_entry( + session, + ds.id, + "fact_ledger_wide", + { + "dedup": {"value": "同表含多类台账记录，聚合前须限定 entry_category", "source": "human", "status": "confirmed"}, + }, + if_version=0, + actor="test", + ) + await session.commit() + + bundle = await ground_question( + session, + "查询 2026-11-11 的异常中断率", + datasource_ids=[ds.id], + restricted=False, + ) + grounded_tbls = [t["table_name"] for t in bundle.matched_tables] + assert "fact_ledger_wide" in grounded_tbls + assert any(t in grounded_tbls for t in ("meta_category_schema", "dim_org_unit")) + assert any(g["kind"] == "dedup" for g in bundle.guardrails) + + bundle_restricted = await ground_question( + session, + "查询 2026-11-11 的异常中断率", + datasource_ids=[ds.id], + restricted=True, + ) + for ut in bundle_restricted.ui_summary["tables"]: + assert "fact_ledger_wide" not in ut["table_name"] + assert "meta_category_schema" not in ut["reason"] + + # 4) 当分组查询返回全 NULL 时，自动折叠重复 NULL 行并输出通用目录诊断提示 + 最新可用日期 + raw_tool_out = await run_metric_query( + ToolContext(data_versions=Versions.CURRENT), + metric=metric_row.key, + date_from="2026-11-11", + date_to="2026-11-11", + group_by=["tx_id"], + filters={"tx_id": [f"TX-{i:04d}" for i in range(5, 18)]}, + limit=50, + ) + tool_payload = json.loads(raw_tool_out) + assert tool_payload["all_metric_values_null"] is True + assert tool_payload["null_group_row_count"] == 13 + assert len(tool_payload["rows"]) == 5 + assert tool_payload["latest_available_date"] == "2026-11-11" + assert "diagnostic_hint" in tool_payload + assert "meta_category_schema" in tool_payload["diagnostic_hint"] + + await session.delete(metric_row) + await session.commit() + finally: + await drop_source(ds_id) + + +# --------------------------------------------------------------------------- +# 4. 快速直查零延迟数字证据封装：字符串编码、多表消歧与零误报验证 +# --------------------------------------------------------------------------- + + +def test_fast_evidence_binds_string_codes_tables_and_preserves_hallucination_guard() -> None: + snapshots: dict[str, dict[str, Any]] = { + "art_dict": { + "columns": ["cat_code", "cat_name"], + "column_types": {"cat_code": "text", "cat_name": "text"}, + "rows": [ + ["003", "能耗数据"], + ["7", "效率报表"], + ["009", "班次统计"], + ], + "truncated": False, + }, + "art_sum": { + "columns": ["site_code", "unit_cnt", "stop_sec", "open_sec", "stop_rate", "unplanned_rate"], + "column_types": { + "site_code": "text", + "unit_cnt": "number", + "stop_sec": "number", + "open_sec": "number", + "stop_rate": "number", + "unplanned_rate": "number", + }, + "rows": [ + ["3950", 1, 8653, 82751, 10.46, 0.0], + ["3951", 7, 20571, 580986, 3.54, 0.42], + ], + "truncated": False, + }, + "art_dtl": { + "columns": ["site_code", "unit_id", "unit_name", "open_sec", "stop_sec", "stop_rate", "unplanned_rate"], + "column_types": { + "site_code": "text", + "unit_id": "text", + "unit_name": "text", + "open_sec": "number", + "stop_sec": "number", + "stop_rate": "number", + "unplanned_rate": "number", + }, + "rows": [ + ["3950", "U395001", "一号单元42", 82751, 8653, 10.46, 0.0], + ["3951", "U395102", "二号单元28", 75264, 10471, 13.91, 2.2], + ["3951", "U395101", "三号单元", 73722, 10100, 13.7, 1.1], + ["3951", "U395103", "四号单元", 86400, 0, 0.0, 0.0], + ["3951", "U395104", "五号单元", 86400, 0, 0.0, 0.0], + ["3951", "U395105", "六号单元", 86400, 0, 0.0, 0.0], + ["3951", "U395106", "七号单元", 86400, 0, 0.0, 0.0], + ["3951", "U395107", "八号单元", 86400, 0, 0.0, 0.0], + ], + "truncated": False, + }, + } + cat = { + "Q1": {"kind": "query", "artifact": "art_dict", "columns": snapshots["art_dict"]["columns"]}, + "Q2": {"kind": "query", "artifact": "art_sum", "columns": snapshots["art_sum"]["columns"]}, + "Q3": {"kind": "query", "artifact": "art_dtl", "columns": snapshots["art_dtl"]["columns"]}, + } + loader = lambda aid: snapshots[aid] + question = "3951和395011月11 的异常率" + + raw_answer = """### 2026-11-11 异常率（站点 3950 / 3951） + +按「班次统计」（`cat_code='009'`）口径汇总 —— 异常率 = 异常时长 ÷ 应开时长 × 100.0%： + +| 站点 | 单元数 | 异常时长(秒) | 应开时长(秒) | 异常率 | 计划外率 | +| --- | --- | --- | --- | --- | --- | +| 3950 | 1 | 8,653 | 82,751 | 10.46% | 0% | +| 3951 | 7 | 20,571 | 580,986 | 3.54% | 0.42% | + +一句话结论：3950 当日异常率 10.46%（唯一单元一号单元42，异常 8,653 秒）；3951 异常率 3.54%，且 7 个单元中有 5 个为 0。 + +### 3951 单元级明细 + +| site_code | unit_id | unit_name | open_sec | stop_sec | stop_rate | unplanned_rate | +| --- | --- | --- | --- | --- | --- | --- | +| 3950 | U395001 | 一号单元42 | 82,751 | 8,653 | 10.46 | 0 | +| 3951 | U395102 | 二号单元28 | 75,264 | 10,471 | 13.91 | 2.2 | +| 3951 | U395101 | 三号单元 | 73,722 | 10,100 | 13.7 | 1.1 | +| 3951 | U395103 | 四号单元 | 86,400 | 0 | 0 | 0 | + +说明：2026-11-11 两站点均已含两个班次记录；默认卡片混算了 003 与 7 分组，本次已按 009 分组计算。""" + + prepared = normalize_and_bind_fast_citations(raw_answer, cat, loader=loader) + # 验证前导零字符串编码 '009' 与 '003' 被正确绑定且渲染保留前导零 + assert "[[v:Q1.r2.cat_code]]" in prepared + assert "[[v:Q1.r0.cat_code]]" in prepared + + allow = collect_fast_allow_numbers(prepared, cat, question=question, loader=loader) + doc = compose_doc( + prepared, + cat, + node_id="report1", + run_id="run_test_ok", + allow_numbers=allow, + cells_allowed=True, + entities=False, + loader=loader, + ) + assert "009" in doc["markdown"] + assert "003" in doc["markdown"] + assert doc["stats"]["numbers_cited"] >= 15 + assert doc["stats"]["uncited_numbers"] == 0 + assert doc["violations"] == [] + + # 反例：若模型在回答中凭空编造了一个快照与提问中都不存在的数字（如 99.87% 或额外编造的 100 次），必须照常标出 uncited_number！ + hallucinated_raw = raw_answer + "\n\n此外昨日异常率为 99.87%，累计发生 100 次故障。" + prep_bad = normalize_and_bind_fast_citations(hallucinated_raw, cat, loader=loader) + allow_bad = collect_fast_allow_numbers(prep_bad, cat, question=question, loader=loader) + doc_bad = compose_doc( + prep_bad, + cat, + node_id="report1", + run_id="run_test_bad", + allow_numbers=allow_bad, + cells_allowed=True, + entities=False, + loader=loader, + ) + uncited_msgs = [v["message"] for v in doc_bad["violations"] if v["code"] == "uncited_number"] + assert any("99.87%" in m for m in uncited_msgs) + assert any("100" in m for m in uncited_msgs)