Files
AgentLab/backend/app/look/service.py
T
ThearchyHelios 1a6082f033 大库的对象索引、探查范围和后台探查(next-round 03 A1),schema_view 预筛与按需加入、剖析事实的后端(A2、遗留 e 的底座)
为什么:几千张表的库以前用不了——探查每个源最多取 200 个对象、按数据库列出的顺序截断,同步请求、整份替换、不排队;
落在窗口外的表在目录、look、剖析里被说成「表已不在」。

做了什么:
- db/models_sources.py:source_objects(对象索引一行一个对象:范围、原因、结构指纹与 struct_version、first_seen / last_seen /
  removed_at、reflected_at、按需加入的留痕)、introspect_jobs / introspect_tasks(后台探查任务与逐表状态)。
- data/source_objects.py:每种方言一条字典查询建对象索引(失败退回 Inspector,上限 5 万,截断时不判删除);探查范围规则
  (options.introspect_scope:通配、按 name_key、排除优先、显式列出 / 移出;有目录的表、已发布模板写死 SQL 用到的表 pinned;
  命中超上限拒绝并说出命中数);唯一的 rebuild_schema_cache 从范围内已反射的行物化 schema_cache(形状不变,另加 scope 摘要);
  三种「不在」分开说(removed / missing / out_of_scope / reflect_failed);按需加入(当场只读反射、落盘、留痕、进程内按源加锁)。
- data/introspect_jobs.py:探查改成后台任务(core/job_runtime,lane kind introspect,和同源起草、剖析串行;按批反射、逐表记状态、
  可停止、读完的照样物化;重启标中断不续跑)。
- api/datasources.py:POST /introspect 返回 202 和任务,dry_run 同步只刷新索引、返回命中数;/introspect-jobs 读任务、停止;
  /introspect-scope 读、预览、保存规则;/schema?table= 带剖析事实、说清三种「不在」;DataSourceOut 加 schema_scope、introspect_job。
- schema_view:search_tables(BM25 倒排,遮罩列只用列名,范围外只按名字和注释、权重低一档)、profile_lines(剖析事实一节,
  附「只供写查询参考」,遮罩按渲染那一刻重滤,上传表格快照不一致时不给)、absent_detail。
- look:候选超过 PREFILTER_LIMIT 时只给预筛结果 + 已用到的表 + 一跳关联;点名或挑中范围外的表按需加入(每次查看最多 4 张),
  look 记录加 added_to_scope、prefiltered;表详情带剖析事实。
- tools/datasource:db_schema__ 加 keyword、不传 table 时只列前 50 张;点名范围外的表按需加入(每次运行最多 8 张);查询结果
  照实列「未按目录检查的表」(unchecked_tables,不新增检查规则);建工具时读剖析事实。
- 剖析:_relation_ends 分清「范围外,可以加入」和「表结构里没有」;任务记上传表格的 snapshot_id;last_facts。表助手、/schema
  用同一段剖析事实。
- 目录接口:清单只列范围内,absent_reason 分开说;GET /catalog/search 查对象索引,POST /catalog/scope/add 加入范围。
- 第 1 波留下的一处:check_valid_filter 和 stored_valid_filter 拼有效记录条件时按 quote_name 强制加引号。
- 工具参数纠错:空着的可选参数不止一个时,只认拼写明显接近的那一个(db_schema 多了 keyword 以后 tabel 仍能纠成 table)。
- 测试:3,000 张表的合成库(对象索引、缺省范围超上限拒绝、规则圈 400 张后台读完、停止、重启标中断、和剖析排队、同步起草
  409)、删除与结构版本、Inspector 回退、截断不判删除、范围规则、pinned、方言金样;改了同步探查的老测试改为等任务做完。
2026-10-16 14:17:06 +08:00

244 lines
14 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""查看服务(look 文档 §2.5.3,总方案 L2、L3):一行 look → 交回给模型的文字 + 发给前端的记录。
处理顺序:
1. need 没写:回「look 要写 need」,不算次数;次数用完:回「按已查看的结构完成」,不再查看。
2. targets 按类型分组(旧写法 tables:[{source, table}] 读进来换成 kind=table)。写了 name 的点名直取,不调挑选模型;
只写了 kind(和 source)的、或者整个没写 targets 的,按 need 在那一类里挑(没写时挑这个入口的第一类)。
3. 挑选失败或挑出来是空的:交回那一类的目录(status=index),写明原因,请模型点名再看。点名的一个都没认出来时,
按 need 在那几个来源里挑(同只写类别),没认出来的照样写「没找到」;候选里一个能查看的都没有(库还没探查表
结构)时没有目录可交,status=failed,照实说原因。
4. 这一批对象这一轮都已经看过:回「已在上面」,不重新排一遍。
5. 各类给详情,合计受 LOOK_RESULT_CHARS 约束;记一条记录(id、by、need、status、objects、用时)。
系统代为查看(look 文档 D5:发布前修复、升级证据、应用润色)也走这里,by=system,不占模型的次数,记录照样发、照样存。
查看本身出错(status=failed)不挡生成:照实交回原因,模型按已有的信息接着写。
"""
from __future__ import annotations
import logging
import time
from typing import Any
from app.look.base import LOOK_LIMIT, LOOK_RESULT_CHARS, LookBudget, LookKind, LookResult, LookTarget
logger = logging.getLogger(__name__)
# ==========================================================================
# 写给模型的文字
# ==========================================================================
NEED_MISSING_PROMPT = "look 要写 need:一句话说明要查什么。这一次没有查看,也不算次数"
LIMIT_PROMPT = "这一轮的查看次数已用完(最多 {limit} 次),按已经查看的结构完成,不要再 look"
REPEAT_PROMPT = "这些{what}已经在上面的查看结果里了,不再重复给出,直接用上面的"
HEAD_MODEL_PROMPT = "查看结果(第 {n} 次,这一轮最多 {limit} 次)"
HEAD_SYSTEM_PROMPT = "系统已代为查看(不占你的查看次数)"
NEED_PROMPT = "需求:{need}"
INDEX_PROMPT = ("没能按需求挑出{what}:{reason}。下面是候选的{what}目录,从里面点名(targets 写 name)再 look 一次:\n"
"{index}")
EMPTY_PROMPT = ("按这个需求没有挑出相关的{what}。这一步确实要用到{what}的话,从下面的目录里点名再 look;"
"不需要就接着写:\n{index}")
FAILED_PROMPT = "这一次查看出错了:{reason}。按已有的信息接着写"
UNAVAILABLE_PROMPT = "没有可查看的{what}:{reason}。按已有的信息接着写,不要猜{what}的名字和字段"
#: 按需求挑出来是空的:写进 look 事件的 reason,界面上照原话显示
EMPTY_REASON = "按这个需求没有挑出相关的{what}"
UNKNOWN_KIND_PROMPT = "这一轮不能查看这一类对象:{kinds}(能查看的:{allowed})"
GUIDE_HEAD_PROMPT = (
"按需查看(look):要看{labels}的详情时,单独输出一行\n"
' {{"op":"look","need":"一句话说要查什么","targets":[{example}]}}\n'
" 输出这一行就停下,系统把查看结果作为下一条消息交回来,你接着往下写(已经输出的不要重复)。\n"
" · need 必填。targets 里写了 name 的直接取那几个;只写 kind(可以加 source)不写 name 的,按 need 在这一类里挑;"
"整个不写 targets 时按 need 挑{default}。sources 可选:限定在哪几个来源(数据源名、工作流名……)里挑\n"
" · kind 只能是:{kinds}\n"
" · 每轮最多查看 {limit} 次,一次把要用的都点上;这一轮已经看过的不要再看,这次对话里看过的直接点名看"
)
GUIDE_EXAMPLE_PROMPT = {
"table": '{"kind":"table","source":"数据源名","name":"表名"}',
"capability": '{"kind":"capability","name":"能力名"}',
"workflow_version": '{"kind":"workflow_version","source":"工作流名","name":"v3"}',
}
CONTINUE_PROMPT = "\n\n接着输出操作。已经输出、已经落地的操作不要重复{landed}。"
LANDED_PROMPT = "(已落地:{ids})"
def parse_targets(line: dict[str, Any]) -> list[LookTarget]:
"""look 行里的 targets(L2)。旧写法 tables:[{source, table}](look 文档 §2.5.2 的初稿)读进来换成 kind=table。"""
out: list[LookTarget] = []
for t in line.get("targets") or []:
if isinstance(t, dict) and isinstance(t.get("kind"), str):
target: LookTarget = {"kind": t["kind"].strip()}
for key in ("source", "name"):
if isinstance(t.get(key), (str, int)) and str(t[key]).strip():
target[key] = str(t[key]).strip()
out.append(target)
for t in line.get("tables") or []:
if isinstance(t, str) and t.strip():
out.append({"kind": "table", "name": t.strip()})
elif isinstance(t, dict) and isinstance(t.get("table", t.get("name")), str):
target = {"kind": "table", "name": str(t.get("table", t.get("name"))).strip()}
if isinstance(t.get("source"), str) and t["source"].strip():
target["source"] = t["source"].strip()
out.append(target)
return out
def _sources(line: dict[str, Any]) -> list[str] | None:
raw = line.get("sources")
names = [str(s).strip() for s in raw if isinstance(s, str) and s.strip()] if isinstance(raw, list) else []
return names or None
class LookService:
"""一轮生成里的查看:各类对象(LookKind)、这一轮的记录、看过的对象。起草和自查修正共用一份。"""
def __init__(self, kinds: list[LookKind]) -> None:
self.kinds: dict[str, LookKind] = {k.kind: k for k in kinds}
self.records: list[dict[str, Any]] = []
self.texts: list[str] = []
self._seen: set[str] = set()
self._n = 0
@property
def default_kind(self) -> str | None:
return next(iter(self.kinds), None)
async def overview(self) -> str:
"""system 里的概览:每类各自几行,没有内容的类不占地方。"""
blocks = []
for kind in self.kinds.values():
lines = await kind.overview()
if lines:
blocks.append("\n".join(lines))
return "\n\n".join(blocks)
def guide(self) -> str:
"""写给模型的 look 用法(只列这个入口能看的类)。什么时候该看由调用方另写(各入口不一样)。"""
if not self.kinds:
return ""
labels = "、".join(k.label for k in self.kinds.values())
default = self.kinds[self.default_kind].label if self.default_kind else ""
return GUIDE_HEAD_PROMPT.format(labels=labels, example=GUIDE_EXAMPLE_PROMPT.get(self.default_kind or "", ""),
default=default, kinds="、".join(self.kinds), limit=LOOK_LIMIT)
def looked(self) -> str:
"""这一轮所有查看交回过的文字,修正请求里原样带上(look 文档 D8:和起草时看到的是同一份)。"""
return "\n\n".join(self.texts)
def continue_prompt(self, landed: list[str]) -> str:
return CONTINUE_PROMPT.format(landed=LANDED_PROMPT.format(ids="、".join(landed)) if landed else "")
async def look(self, line: dict[str, Any], budget: LookBudget | None = None, *, by: str = "model") -> LookResult:
need = str(line.get("need") or "").strip()
if not need:
return LookResult(NEED_MISSING_PROMPT, None, counted=False)
if budget is not None and budget.used >= budget.limit:
return LookResult(LIMIT_PROMPT.format(limit=budget.limit), None, counted=False)
if budget is not None:
budget.used += 1
self._n += 1
started = time.monotonic()
record: dict[str, Any] = {"op": "look", "id": f"L{self._n}", "by": by, "need": need, "status": "ok"}
head = HEAD_SYSTEM_PROMPT if by == "system" else HEAD_MODEL_PROMPT.format(
n=budget.used if budget else self._n, limit=budget.limit if budget else LOOK_LIMIT)
try:
body, objects, keys = await self._run(line, need, record)
except Exception as e: # noqa: BLE001 - 查看是辅助步骤,出错不挡生成(P3)
from app.core.errors import explain
logger.exception("查看出错:%s", need)
record.update(status="failed", reason=explain(e)[0])
body, objects, keys = FAILED_PROMPT.format(reason=record["reason"]), [], set()
if record["status"] == "ok" and keys and keys <= self._seen and not any(o.get("missing") for o in objects):
# 这一批这一轮都看过了:不重排一遍,也不发事件(模型没看到新东西),编号留给下一次真正的查看
self._n -= 1
what = "、".join(dict.fromkeys(self.kinds[k.split(":", 1)[0]].label for k in keys))
return LookResult(REPEAT_PROMPT.format(what=what), None, counted=True)
self._seen |= keys
record["elapsed_ms"] = int((time.monotonic() - started) * 1000)
record["objects"] = objects
text = "\n".join([head, NEED_PROMPT.format(need=need), "", body])
self.records.append(record)
self.texts.append(text)
return LookResult(text, record)
async def _run(self, line: dict[str, Any], need: str,
record: dict[str, Any]) -> tuple[str, list[dict[str, Any]], set[str]]:
"""真正查看:(交回的正文, objects[], 这一次给出详情的对象的身份)。status / reason 写进 record。"""
targets = parse_targets(line)
sources = _sources(line)
for kind in self.kinds.values():
# 这一次查看开始:按需加入的上限按次数算、留痕要记需求原文和查看编号(只有表这一类用得着)
begin = getattr(kind, "begin", None)
if callable(begin):
begin(need=need, ref=str(record.get("id") or ""))
unknown = [t["kind"] for t in targets if t["kind"] not in self.kinds]
named: dict[str, list[LookTarget]] = {}
picks: dict[str, list[str] | None] = {}
for t in targets:
if t["kind"] not in self.kinds:
continue
if t.get("name"):
named.setdefault(t["kind"], []).append(t)
else:
# 只写了类别:按 need 在这一类里挑;写了 source 的只在那几个来源里挑
scope = picks.setdefault(t["kind"], [])
if scope is not None and t.get("source"):
scope.append(t["source"])
elif not t.get("source"):
picks[t["kind"]] = None
if not targets and self.default_kind:
picks[self.default_kind] = None
parts: list[str] = []
objects: list[dict[str, Any]] = []
keys: set[str] = set()
indexed: list[str] = []
empty: list[str] = []
unavailable: list[str] = []
budget = LOOK_RESULT_CHARS
for kind_name in dict.fromkeys([*named, *picks]):
kind = self.kinds[kind_name]
found, missing = await kind.resolve(named.get(kind_name, [])) if kind_name in named else ([], [])
fallback = bool(missing) and not found and kind_name not in picks
if fallback:
# 点名的一个都没认出来(常见的是拿说明里的中文词当表名):按 need 在那几个来源里挑,和只写类别时一样,
# 不让模型为了拿到真名再 look 一次。没认出来的照样写进「没找到」,不拿挑出来的冒充点名的
picks[kind_name] = list(dict.fromkeys(str(t["source"]) for t in missing if t.get("source"))) or None
if kind_name in picks:
within = picks[kind_name] or sources
chosen, reason = await kind.pick(need, within)
if reason is not None or not chosen:
index = await kind.index(within)
if not index and reason is not None:
# 候选里一个能查看的都没有(库还没探查表结构):没有目录可交,照实说原因,算查看没完成
unavailable.append(UNAVAILABLE_PROMPT.format(what=kind.label, reason=reason))
record["reason"] = reason
elif index or not fallback:
# 点名的来源本身不在这一轮里(目录是空的):「没找到数据源」已经说清楚了,不再补一段空目录
indexed.append(INDEX_PROMPT.format(what=kind.label, reason=reason, index=index) if reason
else EMPTY_PROMPT.format(what=kind.label, index=index))
if reason is not None:
record["reason"] = reason
else:
empty.append(kind.label)
found += [o for o in chosen if kind.key(o) not in {kind.key(f) for f in found}]
if not found and not missing:
continue
text, blocks = await kind.detail(found, budget, missing=missing)
budget = max(budget - len(text), 0)
parts.append(text)
objects += blocks
keys |= {kind.key(o) for o in found}
if unknown:
parts.append(UNKNOWN_KIND_PROMPT.format(kinds="、".join(dict.fromkeys(unknown)),
allowed="、".join(self.kinds)))
parts += indexed + unavailable
if unavailable and not indexed and not keys:
record["status"] = "failed"
elif indexed and not keys:
# 一个对象都没给出详情、交回的是目录:挑选失败(有原因)或按需求没挑出来(返回空)
record["status"] = "index"
record.setdefault("reason", EMPTY_REASON.format(what="、".join(empty)))
else:
# 给出了详情:status 是 ok,reason 只在不是 ok 时带
record.pop("reason", None)
return "\n\n".join(p for p in parts if p), objects, keys