Files
AgentLab/backend/app/look/tables.py
T

391 lines
20 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""查看的对象之一:数据表(look 文档 §2.4 D2、D3、D6)。
- overview:每个库一行(data/schema_view.overview_line),外加「现有工作流用到的表」只列表名;
- resolve:点名直取,认全名、只写表名、大小写不同、「库名.表名」(catalog.resolve_table_name);只有一个候选库时可以不写库名;
- pick:候选库的表加起来不超过 PICK_LIMIT 张就全给,否则用挑选模型按需求从表目录(表名|中文名|粒度)里挑;
- detail:沿关系图补上两跳以内的中间表(合计不超过 CONTEXT_TABLE_LIMIT),每张表给全部字段和数据目录(遮罩列只给
列名和类型),再给表之间的连接条件和连接路径;超过字数上限的表只给字段名。
表结构和目录在开流之前读好(load):目录用单独的会话读,读不到就当没有,不挡生成。
"""
from __future__ import annotations
import logging
from collections.abc import Mapping
from dataclasses import dataclass, field
from typing import Any
from app.data import catalog, introspect, schema_view
from app.data.catalog import INFERRED_MARK, JoinEdge
from app.look.base import CONTEXT_TABLE_LIMIT, PICK_LIMIT, LookTarget
from app.look.pick import Picker, PickFailed
logger = logging.getLogger(__name__)
# ==========================================================================
# 写给模型的文字
# ==========================================================================
OVERVIEW_HEAD_PROMPT = "已接入的数据源(要看表结构时用 look,kind 写 table):"
OVERVIEW_SEEDS_PROMPT = "现有工作流用到的表:{tables}(来自现有工作流,字段要用时 look)"
SOURCE_HEAD_PROMPT = "{source}({kind}):{lead}"
LEAD_NAMED_PROMPT = "点名查看 {n} 张"
LEAD_PICKED_PROMPT = "按需求从 {total} 张表中挑出 {n} 张"
LEAD_ALL_PROMPT = "候选只有 {n} 张表,全部给出"
LEAD_JOINED_PROMPT = "另为连接补上 {n} 张"
JOINED_MARK_PROMPT = "(为连接补上)"
CAPPED_PROMPT = "一次最多查看 {limit} 张表,点名的只取了前 {limit} 张"
BRIEF_NOTE_PROMPT = "(这一次查看的字数超过上限,后面的表只给了字段名)"
MISSING_PROMPT = "没找到:{items}(表结构里没有,可能没探查到;别拿名字相近的表顶替)"
MISSING_SOURCE_PROMPT = "没找到数据源「{source}」:不在这一轮可用的数据源里"
UNEXPLORED_PROMPT = ("({source} 共 {total} 张表,只探查了 {explored} 张;用户说的表在这里找不到时,先用 db_schema__{source} "
"查,也查不到就查数据字典(information_schema.columns,SQLite 用 sqlite_master),别猜)")
INFERRED_PROMPT = ("标「推断,未确认」的说明和连接条件是按列名、数据库注释或模型推断出来的,没有人确认过:照抄前对照字段"
"核实")
INDEX_HEAD_PROMPT = "数据源「{source}」(共 {total} 张表):"
INDEX_PARTIAL_PROMPT = "数据源「{source}」(共 {total} 张表,只探查了 {explored} 张,下面只列探查到的):"
PICK_WHAT_PROMPT = "数据表"
PICK_LINE_PROMPT = "每行一张表:表名|中文名|粒度"
@dataclass
class SourcePlan:
"""一个候选库:表结构、原样的目录(含驳回项:关系图要靠驳回项压住同一条关系的现推结果)、现有工作流用到的表。"""
source: Any
entries: dict[str, catalog.CatalogEntry] = field(default_factory=dict)
#: 现有工作流 SQL 里查过的表(schema_cache 的键)
seeds: list[str] = field(default_factory=list)
_graph: dict[str, list[JoinEdge]] | None = None
@property
def name(self) -> str:
return self.source.name
@property
def tables(self) -> dict[str, Any]:
return (getattr(self.source, "schema_cache", None) or {}).get("tables") or {}
@property
def explored(self) -> int:
return len(self.tables)
@property
def total(self) -> int:
"""这个源一共几张表(含视图):探查时数据库报的总数。老缓存没记总数时按探查到的算。"""
raw = (getattr(self.source, "schema_cache", None) or {}).get("total")
return max(raw if isinstance(raw, int) and not isinstance(raw, bool) else 0, self.explored)
@property
def unexplored(self) -> int:
"""探查截断了的话还有几张没探查到;没截断为 0(反射失败跳过的表是探查了、读不到,不算)。"""
truncated = (getattr(self.source, "schema_cache", None) or {}).get("truncated")
return max(self.total - self.explored, 0) if truncated else 0
@property
def raw_notes(self) -> dict[str, dict[str, Any]]:
return {t: e.notes for t, e in self.entries.items()}
@property
def notes(self) -> dict[str, dict[str, Any]]:
"""给模型看的目录:去掉驳回项。"""
return {t: catalog.visible_notes(e.notes) for t, e in self.entries.items()}
@property
def catalog_tables(self) -> int:
"""有数据目录的表数:去掉驳回项之后还有内容的。"""
return sum(1 for e in self.entries.values() if catalog.visible_notes(e.notes))
def graph(self) -> dict[str, list[JoinEdge]]:
"""关系图,第一次要用时才推。推不出来就当没有:少了连接条件只是少一个提示。"""
if self._graph is None:
try:
self._graph = schema_view.relation_graph(self.source, self.raw_notes)
except Exception: # noqa: BLE001
logger.exception("数据源 %s 的关系图推不出来,这次查看不给连接条件", self.name)
self._graph = {}
return self._graph
def qualified(self, key: str) -> str:
return schema_view.qualifier(self.source)(key)
@dataclass(frozen=True)
class TableRef:
"""查看结果里的一张表:第几个候选库、schema_cache 的键、怎么来的(named 点名 / picked 按需求挑出)。"""
plan: int
key: str
how: str = "named"
def graph_tables(graph: Mapping[str, Any] | None, sources: list[Any]) -> dict[str, list[str]]:
"""现有工作流里查库节点的 SQL 用到的表:{源名: [schema_cache 的键]},按出现顺序。
只认调用工具节点上写死的 SQL(db_query__<源>);Agent 运行时自己写的 SQL 图里看不到。哪些节点算写死的查询
以 sqlcheck.tool_query 为准(catalog_impact.tool_queries,和发布记录、SQL 检查同一份):换成别的类型的节点
配置里残留的 tool、args 不算。节点认两种写法:画布的 {data: {config}} 和 GraphSpec 的 {config}。SQL 解析
不了的跳过。发布前修复、升级证据由系统代为查看的就是这些表(look 文档 D5)。
"""
from app.data.catalog_impact import tool_queries
from app.engine.evidence import sql_tables
by_name = {getattr(s, "name", None): s for s in sources}
out: dict[str, list[str]] = {}
nodes = (graph or {}).get("nodes") or []
for _, source_name, sql in tool_queries(n for n in nodes if isinstance(n, dict)):
source = by_name.get(source_name)
if source is None:
continue
try:
names = sql_tables(sql)
except Exception: # noqa: BLE001 - 认不出表就当没用到
continue
found = out.setdefault(source.name, [])
for table in names:
key = catalog.resolve_table_name(source.schema_cache, table)
if key and key not in found:
found.append(key)
return {k: v for k, v in out.items() if v}
class TableKind:
kind = "table"
label = "表"
def __init__(self, plans: list[SourcePlan], *, picker: Picker | None = None) -> None:
self.plans = plans
self.picker = picker
@classmethod
async def load(cls, sources: list[Any], *, graph: Mapping[str, Any] | None = None,
picker: Picker | None = None) -> TableKind:
"""读各源的数据目录。graph 是现有工作流(改图时的原图、追问时上一轮的图),概览里列出它查过的表。"""
from app.tools.datasource import _catalog_of
seeds = graph_tables(graph, sources)
plans = [SourcePlan(source=s, entries=await _catalog_of(s) if introspect.table_names(s) else {},
seeds=seeds.get(s.name, [])) for s in sources]
return cls(plans, picker=picker)
# ---------------------------------------------------------------- 概览、目录
async def overview(self) -> list[str]:
if not self.plans:
return []
lines = [OVERVIEW_HEAD_PROMPT]
lines += [schema_view.overview_line(p.source, catalog_tables=p.catalog_tables) for p in self.plans]
seeds = [f"{p.name} 的 " + "、".join(p.qualified(t) for t in p.seeds) for p in self.plans if p.seeds]
if seeds:
lines.append(OVERVIEW_SEEDS_PROMPT.format(tables=";".join(seeds)))
return lines
def _candidates(self, sources: list[str] | None) -> tuple[list[int], list[str]]:
"""候选库(在这一轮可用的范围里)和认不出的库名。sources 不写时为全部。"""
if not sources:
return [i for i, p in enumerate(self.plans) if p.tables], []
by_name = {p.name: i for i, p in enumerate(self.plans)}
found = [by_name[s] for s in sources if s in by_name]
return list(dict.fromkeys(found)), [s for s in sources if s not in by_name]
async def index(self, sources: list[str] | None) -> str:
blocks = []
for i in self._candidates(sources)[0]:
p = self.plans[i]
head = (INDEX_PARTIAL_PROMPT if p.unexplored else INDEX_HEAD_PROMPT).format(
source=p.name, total=p.total, explored=p.explored)
blocks.append(f"{head}\n{schema_view.table_index(p.source, p.notes)}")
return "\n\n".join(blocks)
# ---------------------------------------------------------------- 点名、挑选
def _prefixed(self, name: str) -> tuple[int, str] | None:
"""「库名.表名」这种写法:(第几个库, 表名)。"""
return next(((i, name[len(p.name) + 1:]) for i, p in enumerate(self.plans)
if name.startswith(p.name) and name[len(p.name):len(p.name) + 1] in (".", ":", ":")), None)
def _find(self, source: str | None, name: str) -> TableRef | None:
"""一个点名 → 表。写了库名的只在那个库里找;表名前面带着库名的也认;都没写的按顺序在各库里找。"""
if source is not None:
candidates = [i for i, p in enumerate(self.plans) if p.name == source]
elif (hit := self._prefixed(name)) is not None:
candidates, name = [hit[0]], hit[1]
else:
candidates = [i for i, p in enumerate(self.plans) if p.tables]
for i in candidates:
key = catalog.resolve_table_name(self.plans[i].source.schema_cache, name)
if key:
return TableRef(i, key)
return None
async def resolve(self, targets: list[LookTarget]) -> tuple[list[TableRef], list[LookTarget]]:
found: list[TableRef] = []
missing: list[LookTarget] = []
single = len([p for p in self.plans if p.tables]) == 1
for t in targets:
source = t.get("source") or None
if source is not None and source not in {p.name for p in self.plans}:
missing.append(t)
continue
ref = self._find(None if single and source is None else source, str(t.get("name") or ""))
if ref is None:
missing.append(t)
elif ref not in found:
found.append(ref)
return found, missing
async def pick(self, need: str, sources: list[str] | None) -> tuple[list[TableRef], str | None]:
candidates, _ = self._candidates(sources)
if not any(self.plans[i].tables for i in candidates):
# 候选库一张表都没探查到:不是「按需求没挑出来」,照实说为什么是空的(和概览行同一句)
bare = [self.plans[i] for i in candidates] if sources else self.plans
if bare:
return [], ";".join(f"数据源「{p.name}」{introspect.why_empty(p.source.schema_cache or {})}"
for p in bare)
pool = [TableRef(i, key, "picked") for i in candidates for key in self.plans[i].tables]
if len(pool) <= PICK_LIMIT:
return pool, None
if self.picker is None:
return [], "没有可用的挑选模型"
index = "\n\n".join(
f"{(INDEX_PARTIAL_PROMPT if p.unexplored else INDEX_HEAD_PROMPT).format(source=p.name, total=p.total, explored=p.explored)}\n"
f"{schema_view.table_index(p.source, p.notes)}" for p in (self.plans[i] for i in candidates))
used = ";".join(f"数据源「{p.name}」的 " + "、".join(p.qualified(t) for t in p.seeds)
for p in (self.plans[i] for i in candidates) if p.seeds)
try:
picks = await self.picker.pick(need=need, what=PICK_WHAT_PROMPT, line=PICK_LINE_PROMPT, index=index,
used=used)
except PickFailed as e:
return [], str(e)
names = {self.plans[i].name for i in candidates}
out: list[TableRef] = []
for source, name in picks:
if len(candidates) == 1:
# 只有一个库要挑时不看模型写的库名(写错了也不要紧)
ref = self._find(self.plans[candidates[0]].name, name)
elif source in names or source is None:
ref = self._find(source, name)
else:
# 写了一个不在候选里的库名:表名前带着库名的照样认,否则丢掉
ref = self._find(None, name) if self._prefixed(name) else None
if ref is not None and ref.plan in candidates and not any(r.plan == ref.plan and r.key == ref.key
for r in out):
out.append(TableRef(ref.plan, ref.key, "picked"))
if picks and not out:
return [], "挑出的表在表结构中都不存在"
return out[:PICK_LIMIT], None
def key(self, obj: TableRef) -> str:
return f"table:{self.plans[obj.plan].name}:{obj.key}"
# ---------------------------------------------------------------- 详情
async def detail(self, objects: list[TableRef], budget: int, *,
missing: list[LookTarget] = ()) -> tuple[str, list[dict[str, Any]]]:
capped = len(objects) > CONTEXT_TABLE_LIMIT
objects = objects[:CONTEXT_TABLE_LIMIT]
by_plan: dict[int, list[TableRef]] = {}
for ref in objects:
by_plan.setdefault(ref.plan, []).append(ref)
room = CONTEXT_TABLE_LIMIT - len(objects)
lines: list[str] = []
blocks: list[dict[str, Any]] = []
used = 0
briefed = False
for i, refs in by_plan.items():
p = self.plans[i]
first = [r.key for r in refs]
tables, routes = list(first), []
if len(first) > 1 and room > 0:
tables, routes = schema_view.complete(first, p.graph(), room)
room -= len(tables) - len(first)
joined = tables[len(first):]
named = [r.key for r in refs if r.how == "named"]
picked = [r.key for r in refs if r.how == "picked"]
lead = []
if named:
lead.append(LEAD_NAMED_PROMPT.format(n=len(named)))
if picked:
# 候选不超过 PICK_LIMIT 张时没有挑,全部给出:照实说
every = len(picked) == len(p.tables) and len(p.tables) <= PICK_LIMIT and not p.unexplored
lead.append(LEAD_ALL_PROMPT.format(n=len(picked)) if every
else LEAD_PICKED_PROMPT.format(total=p.total, n=len(picked)))
if joined:
lead.append(LEAD_JOINED_PROMPT.format(n=len(joined)))
lines.append(SOURCE_HEAD_PROMPT.format(source=p.name, kind=p.source.kind, lead=",".join(lead)))
notes = p.raw_notes
for key in tables:
full = schema_view.table_detail(p.source, key, catalog_notes=notes)
block = full.split("\n")
if used + len(full) > budget:
block = [schema_view.table_brief(p.source, key)]
briefed = True
used += sum(len(x) + 1 for x in block)
mark = JOINED_MARK_PROMPT if key in joined else ""
lines.append(f" · {block[0]}{mark}")
lines += [f" {x}" for x in block[1:]]
lines += schema_view.join_lines(p.source, p.graph(), tables, indent=" ")
lines += schema_view.route_lines(p.source, routes, indent=" ")
if p.unexplored:
lines.append(" " + UNEXPLORED_PROMPT.format(source=p.name, total=p.total, explored=p.explored))
blocks.append(self._record(p, named=named, picked=picked, joined=joined,
missing=[str(t.get("name") or "") for t in missing
if self._missing_in(t) == i]))
if briefed:
lines.append(BRIEF_NOTE_PROMPT)
if capped:
lines.append(CAPPED_PROMPT.format(limit=CONTEXT_TABLE_LIMIT))
lines += self._missing_lines(missing, blocks, set(by_plan))
text = "\n".join(lines)
if INFERRED_MARK in text:
text += "\n" + INFERRED_PROMPT
return text, blocks
def _missing_in(self, target: LookTarget) -> int | None:
"""没找到的点名落在哪个候选库(记录按库分块):写了库名就是那个库,只有一个库时就是它。"""
source = target.get("source")
names = [i for i, p in enumerate(self.plans) if p.name == source] if source else \
[i for i, p in enumerate(self.plans) if p.tables]
return names[0] if len(names) == 1 else None
def _missing_lines(self, missing: list[LookTarget], blocks: list[dict[str, Any]], shown: set[int]) -> list[str]:
"""没找到的点名:库认不出的说库,表认不出的说表;落在没给详情的库里的,另补一块记录。"""
lines: list[str] = []
tables: list[str] = []
known = {p.name for p in self.plans}
for t in missing:
source = t.get("source")
if source and source not in known:
lines.append(MISSING_SOURCE_PROMPT.format(source=source))
blocks.append({"kind": "table", "source": source, "source_id": None, "named": [], "picked": [],
"joined": [], "missing": [str(t["name"])] if t.get("name") else []})
continue
where = self._missing_in(t)
label = f"{self.plans[where].name} 的 {t.get('name')}" if where is not None else str(t.get("name"))
tables.append(label)
if where is not None and where not in shown:
shown.add(where)
blocks.append(self._record(self.plans[where], named=[], picked=[], joined=[],
missing=[str(m.get("name") or "") for m in missing
if self._missing_in(m) == where]))
if tables:
lines.append(MISSING_PROMPT.format(items="、".join(tables)))
return lines
def _record(self, p: SourcePlan, *, named: list[str], picked: list[str], joined: list[str],
missing: list[str]) -> dict[str, Any]:
"""look 事件里一个库的那一块(look 文档 §2.5.7)。表名写全名。"""
given = [*named, *picked, *joined]
synced = getattr(p.source, "schema_synced_at", None)
out: dict[str, Any] = {
"kind": "table", "source": p.name, "source_id": getattr(p.source, "id", None),
"named": [p.qualified(t) for t in named], "picked": [p.qualified(t) for t in picked],
"joined": [p.qualified(t) for t in joined], "missing": missing, "total": p.total,
"catalog_tables": p.catalog_tables,
"schema_synced_at": synced.isoformat() if synced is not None else None,
"catalog_versions": {p.qualified(t): p.entries[t].version for t in given if t in p.entries},
}
if p.unexplored:
out["explored"] = p.explored
return out