AgentLab/backend/app/look/tables.py

"""查看的对象之一：数据表（look 文档 §2.4 D2、D3、D6）。

- overview：每个库一行（data/schema_view.overview_line），外加「现有工作流用到的表」只列表名；
- resolve：点名直取，认全名、只写表名、大小写不同、「库名.表名」（catalog.resolve_table_name）；只有一个候选库时可以不写库名；
- pick：候选库的表加起来不超过 PICK_LIMIT 张就全给，否则用挑选模型按需求从表目录（表名｜中文名｜粒度）里挑；候选超过
  PREFILTER_LIMIT 张（几千张表的库）时先按需求检索一遍（schema_view.search_tables，BM25），挑选模型只看前 PREFILTER_LIMIT
  个命中、工作流已用到的表、命中表的一跳关联表（next-round 03 A2）；
- 按需加入：点名或挑中探查范围外的表（对象索引里有、schema_cache 里没有），当场只读反射这一张、落盘、留痕
  （source_objects.add_to_scope：谁、哪一轮、哪一次查看、需求原文），每次查看最多 ON_DEMAND_PER_LOOK 张，超了照实说；
  look 记录里记 added_to_scope。读到的结构这一轮接着用（换成带新缓存的源），下一轮、运行时、SQL 检查、目录都看得到；
- detail：沿关系图补上两跳以内的中间表（合计不超过 CONTEXT_TABLE_LIMIT），每张表给全部字段和数据目录（遮罩列只给
  列名和类型）、剖析事实（行数、日期范围，附「只供写查询参考」），再给表之间的连接条件和连接路径；超过字数上限的表只给
  字段名。

表结构、目录、剖析事实、范围外的对象清单在开流之前读好（load）：用单独的会话读，读不到就当没有，不挡生成。
"""
from __future__ import annotations

import logging
from collections.abc import Mapping
from dataclasses import dataclass, field
from typing import Any

from app.data import catalog, introspect, schema_view
from app.data.catalog import INFERRED_MARK, JoinEdge
from app.data.names import name_key
from app.look.base import (
    CONTEXT_TABLE_LIMIT, NEIGHBOR_LIMIT, ON_DEMAND_PER_LOOK, PICK_LIMIT, PREFILTER_LIMIT, LookTarget,
)
from app.look.pick import Picker, PickFailed

logger = logging.getLogger(__name__)

# ==========================================================================
# 写给模型的文字
# ==========================================================================

OVERVIEW_HEAD_PROMPT = "已接入的数据源（要看表结构时用 look，kind 写 table）："
OVERVIEW_SEEDS_PROMPT = "现有工作流用到的表：{tables}（来自现有工作流，字段要用时 look）"
SOURCE_HEAD_PROMPT = "{source}（{kind}）：{lead}"
LEAD_NAMED_PROMPT = "点名查看 {n} 张"
LEAD_PICKED_PROMPT = "按需求从 {total} 张表中挑出 {n} 张"
LEAD_ALL_PROMPT = "候选只有 {n} 张表，全部给出"
LEAD_JOINED_PROMPT = "另为连接补上 {n} 张"
JOINED_MARK_PROMPT = "（为连接补上）"
CAPPED_PROMPT = "一次最多查看 {limit} 张表，点名的只取了前 {limit} 张"
BRIEF_NOTE_PROMPT = "（这一次查看的字数超过上限，后面的表只给了字段名）"
MISSING_PROMPT = "没找到：{items}（表结构里没有，可能没探查到；别拿名字相近的表顶替）"
MISSING_SOURCE_PROMPT = "没找到数据源「{source}」：不在这一轮可用的数据源里"
UNEXPLORED_PROMPT = ("（{source} 共 {total} 张表，只探查了 {explored} 张；用户说的表在这里找不到时，先用 db_schema__{source} "
                     "查，也查不到就查数据字典（information_schema.columns，SQLite 用 sqlite_master），别猜）")
#: 有对象索引的源（探查范围以外的表有名字、可以按需加入）
OUT_OF_SCOPE_PROMPT = ("（{source} 共 {total} 张表，探查范围内 {explored} 张；用户说的表在这里找不到时，点名 look 它，或者用 "
                       "db_schema__{source} 按 keyword 查：范围外的表会按需加入，别猜）")
ADDED_PROMPT = "按需加入探查范围：{tables}（在探查范围外，这次当场读了结构）"
ADD_CAPPED_PROMPT = "一次查看最多按需加入 {limit} 张探查范围外的表，{tables} 这次没有加入：下一次 look 点名再看"
ADD_FAILED_PROMPT = "{table} 在探查范围外，按需加入时没能读到结构：{why}"
PREFILTERED_PROMPT = "数据源「{source}」（共 {total} 张表，按需求预筛出 {n} 张，探查范围外的标出来了）："
OUTSIDE_MARK_PROMPT = "（探查范围外，选中后按需加入）"
INDEX_MORE_PROMPT = "…另有 {n} 张没有列出：点名查看，或者换个说法写 need 再 look"
INFERRED_PROMPT = ("标「推断，未确认」的说明和连接条件是按列名、数据库注释或模型推断出来的，没有人确认过：照抄前对照字段"
                   "核实")
INDEX_HEAD_PROMPT = "数据源「{source}」（共 {total} 张表）："
INDEX_PARTIAL_PROMPT = "数据源「{source}」（共 {total} 张表，只探查了 {explored} 张，下面只列探查到的）："
INDEX_SCOPED_PROMPT = ("数据源「{source}」（共 {total} 张表，探查范围内 {explored} 张，下面只列范围内的；范围外的点名查看时"
                       "按需加入）：")
PICK_WHAT_PROMPT = "数据表"
PICK_LINE_PROMPT = "每行一张表：表名｜中文名｜粒度"


@dataclass
class SourcePlan:
    """一个候选库：表结构、原样的目录（含驳回项：关系图要靠驳回项压住同一条关系的现推结果）、现有工作流用到的表。"""

    source: Any
    entries: dict[str, catalog.CatalogEntry] = field(default_factory=dict)
    #: 现有工作流 SQL 里查过的表（schema_cache 的键）
    seeds: list[str] = field(default_factory=list)
    #: 剖析事实 {表名: facts}（catalog_profile_jobs.last_facts）：表详情里附一节
    facts: dict[str, dict[str, Any]] = field(default_factory=dict)
    #: 对象索引里探查范围外的对象（source_objects.outside_objects）：检索、点名时用，点到了按需加入
    outside: list[Any] = field(default_factory=list)
    _graph: dict[str, list[JoinEdge]] | None = None
    _search: Any = None
    _outside_keys: dict[str, Any] | None = None

    @property
    def name(self) -> str:
        return self.source.name

    @property
    def tables(self) -> dict[str, Any]:
        return (getattr(self.source, "schema_cache", None) or {}).get("tables") or {}

    @property
    def explored(self) -> int:
        return len(self.tables)

    @property
    def total(self) -> int:
        """这个源一共几张表（含视图）：探查时数据库报的总数。老缓存没记总数时按探查到的算。"""
        raw = (getattr(self.source, "schema_cache", None) or {}).get("total")
        return max(raw if isinstance(raw, int) and not isinstance(raw, bool) else 0, self.explored)

    @property
    def unexplored(self) -> int:
        """探查截断了的话还有几张没探查到；没截断为 0（反射失败跳过的表是探查了、读不到，不算）。"""
        truncated = (getattr(self.source, "schema_cache", None) or {}).get("truncated")
        return max(self.total - self.explored, 0) if truncated else 0

    @property
    def scoped(self) -> bool:
        """有对象索引（按探查范围物化的 schema_cache）：范围外的表有名字、可以按需加入。"""
        return isinstance((getattr(self.source, "schema_cache", None) or {}).get("scope"), dict)

    def outside_of(self, name: str) -> Any | None:
        """对象索引里范围外的这个对象（按 name_key 比，写了 schema 前缀的只看表名）。"""
        if self._outside_keys is None:
            self._outside_keys = {name_key(o.name): o for o in self.outside}
        return self._outside_keys.get(name_key(name.rsplit(".", 1)[-1]))

    def search(self) -> Any:
        """这个源的检索（第一次要预筛时才建）。"""
        if self._search is None:
            self._search = schema_view.TableSearch(self.source, self.notes, self.outside)
        return self._search

    def replace_cache(self, cache: dict[str, Any], added: list[str]) -> None:
        """按需加入以后：换成带新 schema_cache 的源（不改调用方会话里的数据源对象），关系图、检索重建，加进来的不再算范围外。"""
        base = self.source._base if isinstance(self.source, _WithCache) else self.source
        self.source = _WithCache(base, cache)
        keys = {name_key(n) for n in added}
        self.outside = [o for o in self.outside if name_key(o.name) not in keys]
        self._graph = self._search = self._outside_keys = None

    def outside_qualified(self, name: str) -> str:
        schema = (getattr(self.source, "schema_cache", None) or {}).get("schema")
        return f"{schema}.{name}" if schema else name

    @property
    def raw_notes(self) -> dict[str, dict[str, Any]]:
        return {t: e.notes for t, e in self.entries.items()}

    @property
    def notes(self) -> dict[str, dict[str, Any]]:
        """给模型看的目录：去掉驳回项。"""
        return {t: catalog.visible_notes(e.notes) for t, e in self.entries.items()}

    @property
    def catalog_tables(self) -> int:
        """有数据目录的表数：去掉驳回项之后还有内容的。"""
        return sum(1 for e in self.entries.values() if catalog.visible_notes(e.notes))

    def graph(self) -> dict[str, list[JoinEdge]]:
        """关系图，第一次要用时才推。推不出来就当没有：少了连接条件只是少一个提示。"""
        if self._graph is None:
            try:
                self._graph = schema_view.relation_graph(self.source, self.raw_notes)
            except Exception:  # noqa: BLE001
                logger.exception("数据源 %s 的关系图推不出来，这次查看不给连接条件", self.name)
                self._graph = {}
        return self._graph

    def qualified(self, key: str) -> str:
        return schema_view.qualifier(self.source)(key)


class _WithCache:
    """数据源换上一份新的 schema_cache（按需加入以后）：别的属性照原来的。不是 ORM 对象，改它不会被调用方的会话写回库。"""

    def __init__(self, base: Any, cache: dict[str, Any]) -> None:
        self._base = base
        self.schema_cache = cache

    def __getattr__(self, name: str) -> Any:
        return getattr(self._base, name)


@dataclass(frozen=True)
class TableRef:
    """查看结果里的一张表：第几个候选库、schema_cache 的键、怎么来的（named 点名 / picked 按需求挑出）。"""

    plan: int
    key: str
    how: str = "named"


def graph_tables(graph: Mapping[str, Any] | None, sources: list[Any]) -> dict[str, list[str]]:
    """现有工作流里查库节点的 SQL 用到的表：{源名: [schema_cache 的键]}，按出现顺序。

    只认调用工具节点上写死的 SQL（db_query__<源>）；Agent 运行时自己写的 SQL 图里看不到。哪些节点算写死的查询
    以 sqlcheck.tool_query 为准（catalog_impact.tool_queries，和发布记录、SQL 检查同一份）：换成别的类型的节点
    配置里残留的 tool、args 不算。节点认两种写法：画布的 {data: {config}} 和 GraphSpec 的 {config}。SQL 解析
    不了的跳过。发布前修复、升级证据由系统代为查看的就是这些表（look 文档 D5）。
    """
    from app.data.catalog_impact import tool_queries
    from app.engine.evidence import sql_tables

    by_name = {getattr(s, "name", None): s for s in sources}
    out: dict[str, list[str]] = {}
    nodes = (graph or {}).get("nodes") or []
    for _, source_name, sql in tool_queries(n for n in nodes if isinstance(n, dict)):
        source = by_name.get(source_name)
        if source is None:
            continue
        try:
            names = sql_tables(sql)
        except Exception:  # noqa: BLE001 - 认不出表就当没用到
            continue
        found = out.setdefault(source.name, [])
        for table in names:
            key = catalog.resolve_table_name(source.schema_cache, table)
            if key and key not in found:
                found.append(key)
    return {k: v for k, v in out.items() if v}


class TableKind:
    kind = "table"
    label = "表"

    def __init__(self, plans: list[SourcePlan], *, picker: Picker | None = None,
                 trail: Mapping[str, Any] | None = None) -> None:
        self.plans = plans
        self.picker = picker
        #: 按需加入的留痕：{actor, conversation_id}（开流前定下）；这一次查看的需求原文、编号由 begin 记
        self.trail = dict(trail or {})
        self._need = ""
        self._ref = ""
        #: 这一次查看里已经按需加入了几张、加了哪些（{第几个库: [表名]}）、没加成的（{(库, 点名): 说法}）
        self._added_now = 0
        self._added: dict[int, list[str]] = {}
        self._notes: list[str] = []
        self._prefiltered: dict[int, int] = {}
        self._last_index: dict[tuple[int, ...], str] = {}

    @classmethod
    async def load(cls, sources: list[Any], *, graph: Mapping[str, Any] | None = None,
                   picker: Picker | None = None, catalog_view: Any = None,
                   trail: Mapping[str, Any] | None = None) -> TableKind:
        """读各源的数据目录、剖析事实、探查范围外的对象清单。graph 是现有工作流（改图时的原图、追问时上一轮的图），概览里
        列出它查过的表。trail 是按需加入的留痕（谁、哪一轮对话）：不给时谁取这次请求的身份。

        读哪一份目录由 catalog_view 定（data/catalog_view.py：当前目录、目录快照、关；只用已核实的项），不给是当前目录。
        查看结果里的目录和运行时 db_schema 给模型看的、自查用的是同一份——评测的「目录对照」量的才是整份贡献。"""
        from app.data.catalog_view import CatalogView, read

        view = CatalogView.parse(catalog_view)
        seeds = graph_tables(graph, sources)
        plans = [SourcePlan(source=s, entries=await read(view, s) if introspect.table_names(s) else {},
                            seeds=seeds.get(s.name, [])) for s in sources]
        await _load_extras(plans)
        trail = dict(trail or {})
        if "actor" not in trail:
            from app.api.deps import current_principal

            trail["actor"] = current_principal().username
        return cls(plans, picker=picker, trail=trail)

    def begin(self, *, need: str, ref: str) -> None:
        """一次查看开始（LookService 调）：按需加入的上限按这一次算，留痕记这次的需求和编号。"""
        self._need, self._ref = need, ref
        self._added_now = 0
        self._added = {}
        self._notes = []
        self._prefiltered = {}
        # 交回给模型点名的那份预筛目录只对这一次的需求有效
        self._last_index = {}

    # ---------------------------------------------------------------- 概览、目录

    async def overview(self) -> list[str]:
        if not self.plans:
            return []
        lines = [OVERVIEW_HEAD_PROMPT]
        lines += [schema_view.overview_line(p.source, catalog_tables=p.catalog_tables) for p in self.plans]
        seeds = [f"{p.name} 的 " + "、".join(p.qualified(t) for t in p.seeds) for p in self.plans if p.seeds]
        if seeds:
            lines.append(OVERVIEW_SEEDS_PROMPT.format(tables="；".join(seeds)))
        return lines

    def _candidates(self, sources: list[str] | None) -> tuple[list[int], list[str]]:
        """候选库（在这一轮可用的范围里）和认不出的库名。sources 不写时为全部。"""
        if not sources:
            return [i for i, p in enumerate(self.plans) if p.tables], []
        by_name = {p.name: i for i, p in enumerate(self.plans)}
        found = [by_name[s] for s in sources if s in by_name]
        return list(dict.fromkeys(found)), [s for s in sources if s not in by_name]

    async def index(self, sources: list[str] | None) -> str:
        """交回给模型点名的表目录。这一次查看预筛过的，交回预筛的那一份（几千张表的整份目录塞不进来）；没预筛过、表又太多
        的只列前 PREFILTER_LIMIT 张，写明另有几张。"""
        candidates = self._candidates(sources)[0]
        if (cached := self._last_index.get(tuple(candidates))) is not None:
            return cached
        blocks = []
        for i in candidates:
            p = self.plans[i]
            head = (INDEX_HEAD_PROMPT if not p.unexplored else INDEX_SCOPED_PROMPT if p.scoped
                    else INDEX_PARTIAL_PROMPT).format(source=p.name, total=p.total, explored=p.explored)
            names = list(p.tables)
            body = schema_view.table_index(p.source, p.notes, tables=names[:PREFILTER_LIMIT])
            if len(names) > PREFILTER_LIMIT:
                body += "\n" + INDEX_MORE_PROMPT.format(n=len(names) - PREFILTER_LIMIT)
            blocks.append(f"{head}\n{body}")
        return "\n\n".join(blocks)

    # ---------------------------------------------------------------- 点名、挑选

    def _prefixed(self, name: str) -> tuple[int, str] | None:
        """「库名.表名」这种写法：(第几个库, 表名)。"""
        return next(((i, name[len(p.name) + 1:]) for i, p in enumerate(self.plans)
                     if name.startswith(p.name) and name[len(p.name):len(p.name) + 1] in (".", ":", "：")), None)

    def _find(self, source: str | None, name: str) -> TableRef | None:
        """一个点名 → 表。写了库名的只在那个库里找；表名前面带着库名的也认；都没写的按顺序在各库里找。"""
        if source is not None:
            candidates = [i for i, p in enumerate(self.plans) if p.name == source]
        elif (hit := self._prefixed(name)) is not None:
            candidates, name = [hit[0]], hit[1]
        else:
            candidates = [i for i, p in enumerate(self.plans) if p.tables]
        for i in candidates:
            key = catalog.resolve_table_name(self.plans[i].source.schema_cache, name)
            if key:
                return TableRef(i, key)
        return None

    def _find_outside(self, source: str | None, name: str) -> tuple[int, Any] | None:
        """点名的表在哪个候选库的探查范围外（对象索引里有）：(第几个库, 对象)。规则同 _find。"""
        if source is not None:
            candidates = [i for i, p in enumerate(self.plans) if p.name == source]
        elif (hit := self._prefixed(name)) is not None:
            candidates, name = [hit[0]], hit[1]
        else:
            candidates = [i for i, p in enumerate(self.plans) if p.tables or p.outside]
        for i in candidates:
            obj = self.plans[i].outside_of(name)
            if obj is not None:
                return i, obj
        return None

    async def _add(self, wanted: dict[int, list[str]]) -> dict[tuple[int, str], TableRef]:
        """按需加入（每次查看最多 ON_DEMAND_PER_LOOK 张，超了的照实说）：{(第几个库, 对象名): 加进来以后的 TableRef}。读不到
        结构、加不进去的记进这一次的说明。"""
        from app.data import source_objects as so

        out: dict[tuple[int, str], TableRef] = {}
        capped: list[str] = []
        for i, names in wanted.items():
            p = self.plans[i]
            room = max(ON_DEMAND_PER_LOOK - self._added_now, 0)
            take, rest = names[:room], names[room:]
            capped += [f"{p.name} 的 {n}" for n in rest]
            if not take:
                continue
            source_id = getattr(p.source, "id", None)
            if not source_id:
                continue
            try:
                got = await so.add_to_scope(source_id, take, trail=so.Trail(
                    via=so.VIA_LOOK, actor=self.trail.get("actor"), conversation_id=self.trail.get("conversation_id"),
                    ref=self._ref, need=self._need))
            except Exception as e:  # noqa: BLE001 - 按需加入是辅助步骤，失败只是少一张表
                from app.core.errors import explain

                logger.exception("按需加入失败（数据源 %s）", p.name)
                self._notes += [ADD_FAILED_PROMPT.format(table=n, why=explain(e)[0]) for n in take]
                continue
            self._added_now += len(got.added)
            for name, why in {**got.failed, **got.refused}.items():
                self._notes.append(ADD_FAILED_PROMPT.format(table=name, why=why))
            if got.cache is not None and got.added:
                p.replace_cache(got.cache, got.added)
                self._added.setdefault(i, []).extend(got.added)
            for name in got.added + got.already:
                key = catalog.resolve_table_name(p.source.schema_cache, name)
                if key:
                    out[(i, name_key(name))] = TableRef(i, key)
        if capped:
            self._notes.append(ADD_CAPPED_PROMPT.format(limit=ON_DEMAND_PER_LOOK, tables="、".join(capped)))
        return out

    async def resolve(self, targets: list[LookTarget]) -> tuple[list[TableRef], list[LookTarget]]:
        found: list[TableRef] = []
        missing: list[LookTarget] = []
        single = len([p for p in self.plans if p.tables or p.outside]) == 1
        outside: list[tuple[LookTarget, int, Any]] = []
        for t in targets:
            source = t.get("source") or None
            if source is not None and source not in {p.name for p in self.plans}:
                missing.append(t)
                continue
            where = None if single and source is None else source
            ref = self._find(where, str(t.get("name") or ""))
            if ref is None:
                hit = self._find_outside(where, str(t.get("name") or ""))
                if hit is not None:
                    outside.append((t, hit[0], hit[1]))
                else:
                    missing.append(t)
            elif ref not in found:
                found.append(ref)
        if outside:
            wanted: dict[int, list[str]] = {}
            for _t, i, obj in outside:
                if obj.name not in wanted.setdefault(i, []):
                    wanted[i].append(obj.name)
            added = await self._add(wanted)
            for t, i, obj in outside:
                ref = added.get((i, name_key(obj.name)))
                if ref is None:
                    missing.append(t)
                elif ref not in found:
                    found.append(ref)
        return found, missing

    async def pick(self, need: str, sources: list[str] | None) -> tuple[list[TableRef], str | None]:
        candidates, _ = self._candidates(sources)
        if not any(self.plans[i].tables for i in candidates):
            # 候选库一张表都没探查到：不是「按需求没挑出来」，照实说为什么是空的（和概览行同一句）
            bare = [self.plans[i] for i in candidates] if sources else self.plans
            if bare:
                return [], "；".join(f"数据源「{p.name}」{introspect.why_empty(p.source.schema_cache or {})}"
                                    for p in bare)
        pool = [TableRef(i, key, "picked") for i in candidates for key in self.plans[i].tables]
        outside = sum(len(self.plans[i].outside) for i in candidates)
        if len(pool) + outside <= PICK_LIMIT and not outside:
            return pool, None
        if self.picker is None:
            return [], "没有可用的挑选模型"
        if len(pool) + outside > PREFILTER_LIMIT:
            index = self._prefiltered_index(need, candidates)
        else:
            index = "\n\n".join(self._full_index(self.plans[i]) for i in candidates)
        self._last_index[tuple(candidates)] = index
        used = "；".join(f"数据源「{p.name}」的 " + "、".join(p.qualified(t) for t in p.seeds)
                        for p in (self.plans[i] for i in candidates) if p.seeds)
        try:
            picks = await self.picker.pick(need=need, what=PICK_WHAT_PROMPT, line=PICK_LINE_PROMPT, index=index,
                                           used=used)
        except PickFailed as e:
            return [], str(e)
        names = {self.plans[i].name for i in candidates}
        out: list[TableRef] = []
        wanted: dict[int, list[str]] = {}
        order: list[tuple[str, Any]] = []
        for source, name in picks:
            if len(candidates) == 1:
                # 只有一个库要挑时不看模型写的库名（写错了也不要紧）
                where: str | None = self.plans[candidates[0]].name
            elif source in names or source is None:
                where = source
            elif self._prefixed(name):
                where = None
            else:
                # 写了一个不在候选里的库名：表名前带着库名的照样认，否则丢掉
                continue
            ref = self._find(where, name)
            if ref is not None and ref.plan in candidates:
                if not any(r.plan == ref.plan and r.key == ref.key for r in out):
                    out.append(TableRef(ref.plan, ref.key, "picked"))
                    order.append(("in", out[-1]))
                continue
            hit = self._find_outside(where, name)
            if hit is not None and hit[0] in candidates and hit[1].name not in wanted.setdefault(hit[0], []):
                wanted[hit[0]].append(hit[1].name)
                order.append(("out", (hit[0], name_key(hit[1].name))))
        if any(wanted.values()):
            added = await self._add({i: n for i, n in wanted.items() if n})
            ranked: list[TableRef] = []
            for kind, item in order:
                ref = item if kind == "in" else added.get(item)
                if ref is not None and not any(r.plan == ref.plan and r.key == ref.key for r in ranked):
                    ranked.append(TableRef(ref.plan, ref.key, "picked"))
            out = ranked
        if picks and not out:
            return [], "挑出的表在表结构中都不存在"
        return out[:PICK_LIMIT], None

    def _full_index(self, p: SourcePlan) -> str:
        """整份表目录（候选不多时交给挑选模型的那一份）：范围内的每张一行，范围外的另起一行、标出来。"""
        head = (INDEX_PARTIAL_PROMPT if p.unexplored and not p.outside else INDEX_HEAD_PROMPT).format(
            source=p.name, total=p.total, explored=p.explored)
        lines = [schema_view.table_index(p.source, p.notes)]
        lines += [self._outside_line(p, o) for o in p.outside]
        return f"{head}\n" + "\n".join(x for x in lines if x)

    def _outside_line(self, p: SourcePlan, obj: Any) -> str:
        comment = f"｜{obj.comment}" if getattr(obj, "comment", None) else ""
        return f"{p.outside_qualified(obj.name)}{comment}{OUTSIDE_MARK_PROMPT}"

    def _prefiltered_index(self, need: str, candidates: list[int]) -> str:
        """几千张表的库：按需求检索一遍，挑选模型只看前 PREFILTER_LIMIT 个命中（各库合起来按得分排）、工作流已用到的表、
        命中表的一跳关联表（最多 NEIGHBOR_LIMIT 张）。检索什么都没命中时退回各库表结构里的前几张。"""
        hits: list[tuple[float, int, Any]] = []
        for i in candidates:
            for hit in self.plans[i].search().search(need, limit=PREFILTER_LIMIT):
                hits.append((hit.score, i, hit))
        hits.sort(key=lambda h: (-h[0], h[1], h[2].name))
        top = hits[:PREFILTER_LIMIT]
        if not top:
            per = max(PREFILTER_LIMIT // max(len(candidates), 1), 1)
            top = [(0.0, i, schema_view.SearchHit(n, 0.0, True)) for i in candidates
                   for n in list(self.plans[i].tables)[:per]]
        blocks = []
        for i in candidates:
            p = self.plans[i]
            mine = [h for _s, j, h in top if j == i]
            inside = [h.name for h in mine if h.in_scope]
            picked = list(dict.fromkeys([*p.seeds, *inside]))
            neighbors: list[str] = []
            graph = p.graph() if inside else {}
            for name in inside:
                for edge in graph.get(name, ()):
                    if edge.to_table in p.tables and edge.to_table not in picked and edge.to_table not in neighbors:
                        neighbors.append(edge.to_table)
                if len(neighbors) >= NEIGHBOR_LIMIT:
                    break
            shown = picked + neighbors[:NEIGHBOR_LIMIT]
            outside = [h.name for h in mine if not h.in_scope]
            if not shown and not outside:
                continue
            self._prefiltered[i] = len(shown) + len(outside)
            lines = [schema_view.table_index(p.source, p.notes, tables=shown)] if shown else []
            for name in outside:
                obj = p.outside_of(name)
                if obj is not None:
                    lines.append(self._outside_line(p, obj))
            head = PREFILTERED_PROMPT.format(source=p.name, total=p.total, n=len(shown) + len(outside))
            blocks.append(head + "\n" + "\n".join(x for x in lines if x))
        return "\n\n".join(blocks)

    def key(self, obj: TableRef) -> str:
        return f"table:{self.plans[obj.plan].name}:{obj.key}"

    # ---------------------------------------------------------------- 详情

    async def detail(self, objects: list[TableRef], budget: int, *,
                     missing: list[LookTarget] = ()) -> tuple[str, list[dict[str, Any]]]:
        capped = len(objects) > CONTEXT_TABLE_LIMIT
        objects = objects[:CONTEXT_TABLE_LIMIT]
        by_plan: dict[int, list[TableRef]] = {}
        for ref in objects:
            by_plan.setdefault(ref.plan, []).append(ref)
        room = CONTEXT_TABLE_LIMIT - len(objects)
        lines: list[str] = []
        blocks: list[dict[str, Any]] = []
        used = 0
        briefed = False
        for i, refs in by_plan.items():
            p = self.plans[i]
            first = [r.key for r in refs]
            tables, routes = list(first), []
            if len(first) > 1 and room > 0:
                tables, routes = schema_view.complete(first, p.graph(), room)
                room -= len(tables) - len(first)
            joined = tables[len(first):]
            named = [r.key for r in refs if r.how == "named"]
            picked = [r.key for r in refs if r.how == "picked"]
            lead = []
            if named:
                lead.append(LEAD_NAMED_PROMPT.format(n=len(named)))
            if picked:
                # 候选不超过 PICK_LIMIT 张时没有挑，全部给出：照实说
                every = len(picked) == len(p.tables) and len(p.tables) <= PICK_LIMIT and not p.unexplored
                lead.append(LEAD_ALL_PROMPT.format(n=len(picked)) if every
                            else LEAD_PICKED_PROMPT.format(total=p.total, n=len(picked)))
            if joined:
                lead.append(LEAD_JOINED_PROMPT.format(n=len(joined)))
            lines.append(SOURCE_HEAD_PROMPT.format(source=p.name, kind=p.source.kind, lead="，".join(lead)))
            notes = p.raw_notes
            for key in tables:
                full = schema_view.table_detail(p.source, key, catalog_notes=notes, profile=p.facts)
                block = full.split("\n")
                if used + len(full) > budget:
                    block = [schema_view.table_brief(p.source, key)]
                    briefed = True
                used += sum(len(x) + 1 for x in block)
                mark = JOINED_MARK_PROMPT if key in joined else ""
                lines.append(f"  · {block[0]}{mark}")
                lines += [f"    {x}" for x in block[1:]]
            lines += schema_view.join_lines(p.source, p.graph(), tables, indent="  ")
            lines += schema_view.route_lines(p.source, routes, indent="  ")
            if self._added.get(i):
                lines.append("  " + ADDED_PROMPT.format(tables="、".join(p.qualified(t) for t in self._added[i])))
            if p.unexplored:
                prompt = OUT_OF_SCOPE_PROMPT if p.scoped else UNEXPLORED_PROMPT
                lines.append("  " + prompt.format(source=p.name, total=p.total, explored=p.explored))
            blocks.append(self._record(p, named=named, picked=picked, joined=joined,
                                       missing=[str(t.get("name") or "") for t in missing
                                                if self._missing_in(t) == i]))
        if briefed:
            lines.append(BRIEF_NOTE_PROMPT)
        if capped:
            lines.append(CAPPED_PROMPT.format(limit=CONTEXT_TABLE_LIMIT))
        lines += self._notes
        lines += self._missing_lines(missing, blocks, set(by_plan))
        text = "\n".join(lines)
        if INFERRED_MARK in text:
            text += "\n" + INFERRED_PROMPT
        return text, blocks

    def _missing_in(self, target: LookTarget) -> int | None:
        """没找到的点名落在哪个候选库（记录按库分块）：写了库名就是那个库，只有一个库时就是它。"""
        source = target.get("source")
        names = [i for i, p in enumerate(self.plans) if p.name == source] if source else \
            [i for i, p in enumerate(self.plans) if p.tables]
        return names[0] if len(names) == 1 else None

    def _missing_lines(self, missing: list[LookTarget], blocks: list[dict[str, Any]], shown: set[int]) -> list[str]:
        """没找到的点名：库认不出的说库，表认不出的说表；落在没给详情的库里的，另补一块记录。"""
        lines: list[str] = []
        tables: list[str] = []
        known = {p.name for p in self.plans}
        for t in missing:
            source = t.get("source")
            if source and source not in known:
                lines.append(MISSING_SOURCE_PROMPT.format(source=source))
                blocks.append({"kind": "table", "source": source, "source_id": None, "named": [], "picked": [],
                               "joined": [], "missing": [str(t["name"])] if t.get("name") else []})
                continue
            where = self._missing_in(t)
            label = f"{self.plans[where].name} 的 {t.get('name')}" if where is not None else str(t.get("name"))
            tables.append(label)
            if where is not None and where not in shown:
                shown.add(where)
                blocks.append(self._record(self.plans[where], named=[], picked=[], joined=[],
                                           missing=[str(m.get("name") or "") for m in missing
                                                    if self._missing_in(m) == where]))
        if tables:
            lines.append(MISSING_PROMPT.format(items="、".join(tables)))
        return lines

    def _record(self, p: SourcePlan, *, named: list[str], picked: list[str], joined: list[str],
                missing: list[str]) -> dict[str, Any]:
        """look 事件里一个库的那一块（look 文档 §2.5.7）。表名写全名。"""
        given = [*named, *picked, *joined]
        synced = getattr(p.source, "schema_synced_at", None)
        out: dict[str, Any] = {
            "kind": "table", "source": p.name, "source_id": getattr(p.source, "id", None),
            "named": [p.qualified(t) for t in named], "picked": [p.qualified(t) for t in picked],
            "joined": [p.qualified(t) for t in joined], "missing": missing, "total": p.total,
            "catalog_tables": p.catalog_tables,
            "schema_synced_at": synced.isoformat() if synced is not None else None,
            "catalog_versions": {p.qualified(t): p.entries[t].version for t in given if t in p.entries},
        }
        if p.unexplored:
            out["explored"] = p.explored
        index = next(i for i, q in enumerate(self.plans) if q is p)
        if self._added.get(index):
            out["added_to_scope"] = [p.qualified(t) for t in self._added[index]]
        if index in self._prefiltered:
            out["prefiltered"] = self._prefiltered[index]
        return out


async def _load_extras(plans: list[SourcePlan]) -> None:
    """各源的剖析事实、探查范围外的对象清单：一个短会话读完。读不到就当没有（少一节剖析事实、少几个可按需加入的名字），
    不挡生成。上传表格、记录表没有对象索引；探查范围以内就是全部的源（没有范围外的对象）不读清单。"""
    from app.data import catalog_profile_jobs, source_objects
    from app.db.base import SessionLocal

    try:
        async with SessionLocal() as session:
            for p in plans:
                source_id = getattr(p.source, "id", None)
                if not source_id:
                    continue
                if p.tables:
                    p.facts = await catalog_profile_jobs.last_facts(session, source_id)
                scope = (getattr(p.source, "schema_cache", None) or {}).get("scope")
                if isinstance(scope, dict) and scope.get("out_of_scope"):
                    p.outside = await source_objects.outside_objects(session, p.source)
    except Exception:  # noqa: BLE001
        logger.exception("读剖析事实、探查范围外的对象清单失败，这一轮查看不给这两样")