Files
AgentLab/backend/tests/test_catalog_schema_truncated.py
T
ThearchyHelios 1a6082f033 大库的对象索引、探查范围和后台探查(next-round 03 A1),schema_view 预筛与按需加入、剖析事实的后端(A2、遗留 e 的底座)
为什么:几千张表的库以前用不了——探查每个源最多取 200 个对象、按数据库列出的顺序截断,同步请求、整份替换、不排队;
落在窗口外的表在目录、look、剖析里被说成「表已不在」。

做了什么:
- db/models_sources.py:source_objects(对象索引一行一个对象:范围、原因、结构指纹与 struct_version、first_seen / last_seen /
  removed_at、reflected_at、按需加入的留痕)、introspect_jobs / introspect_tasks(后台探查任务与逐表状态)。
- data/source_objects.py:每种方言一条字典查询建对象索引(失败退回 Inspector,上限 5 万,截断时不判删除);探查范围规则
  (options.introspect_scope:通配、按 name_key、排除优先、显式列出 / 移出;有目录的表、已发布模板写死 SQL 用到的表 pinned;
  命中超上限拒绝并说出命中数);唯一的 rebuild_schema_cache 从范围内已反射的行物化 schema_cache(形状不变,另加 scope 摘要);
  三种「不在」分开说(removed / missing / out_of_scope / reflect_failed);按需加入(当场只读反射、落盘、留痕、进程内按源加锁)。
- data/introspect_jobs.py:探查改成后台任务(core/job_runtime,lane kind introspect,和同源起草、剖析串行;按批反射、逐表记状态、
  可停止、读完的照样物化;重启标中断不续跑)。
- api/datasources.py:POST /introspect 返回 202 和任务,dry_run 同步只刷新索引、返回命中数;/introspect-jobs 读任务、停止;
  /introspect-scope 读、预览、保存规则;/schema?table= 带剖析事实、说清三种「不在」;DataSourceOut 加 schema_scope、introspect_job。
- schema_view:search_tables(BM25 倒排,遮罩列只用列名,范围外只按名字和注释、权重低一档)、profile_lines(剖析事实一节,
  附「只供写查询参考」,遮罩按渲染那一刻重滤,上传表格快照不一致时不给)、absent_detail。
- look:候选超过 PREFILTER_LIMIT 时只给预筛结果 + 已用到的表 + 一跳关联;点名或挑中范围外的表按需加入(每次查看最多 4 张),
  look 记录加 added_to_scope、prefiltered;表详情带剖析事实。
- tools/datasource:db_schema__ 加 keyword、不传 table 时只列前 50 张;点名范围外的表按需加入(每次运行最多 8 张);查询结果
  照实列「未按目录检查的表」(unchecked_tables,不新增检查规则);建工具时读剖析事实。
- 剖析:_relation_ends 分清「范围外,可以加入」和「表结构里没有」;任务记上传表格的 snapshot_id;last_facts。表助手、/schema
  用同一段剖析事实。
- 目录接口:清单只列范围内,absent_reason 分开说;GET /catalog/search 查对象索引,POST /catalog/scope/add 加入范围。
- 第 1 波留下的一处:check_valid_filter 和 stored_valid_filter 拼有效记录条件时按 quote_name 强制加引号。
- 工具参数纠错:空着的可选参数不止一个时,只认拼写明显接近的那一个(db_schema 多了 keyword 以后 tabel 仍能纠成 table)。
- 测试:3,000 张表的合成库(对象索引、缺省范围超上限拒绝、规则圈 400 张后台读完、停止、重启标中断、和剖析排队、同步起草
  409)、删除与结构版本、Inspector 回退、截断不判删除、范围规则、pinned、方言金样;改了同步探查的老测试改为等任务做完。
2026-10-16 14:17:06 +08:00

160 lines
7.1 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""探查范围外的表要说出来(数据目录完整性审查 F3 / B4;next-round 03 A1 之后按探查范围说)。
以前探查结构每个数据源最多取 200 张表,多出来的不在表结构缓存里,哪儿都没说。现在全库对象进对象索引,探查范围(规则)
圈出要读结构的那部分;范围外的表在对象索引里有名字。这组守的是:
- 目录清单接口返回 schema_truncated(有范围外的表)、schema_total(对象索引里一共几张),界面据此说明
- 助手的概览写「共 205 张表,探查范围内 200 张」;查看记录(look 事件)的 total 是对象总数,另带 explored(范围内几张)
- 查看结果里写明范围外的点名查看会按需加入;交回的表目录也写明只列了范围内的
- 运行时 db_schema 点名范围外的表:按需加入,给出字段
- 没有范围外的库照旧,不多说一句;升级前只探查了一部分的老缓存照旧说「只探查了」
合成库:205 张空表,探查范围规则只收 t0*、t1*(200 张),测试里临时建,用完连同数据源一起删掉。
"""
from __future__ import annotations
import sqlite3
import uuid
import pytest
from httpx import ASGITransport, AsyncClient
from app.db.base import SessionLocal
from app.db.models import DataSource
from app.main import app
from tests.fixtures.sources import drop_source, introspect_now
WIDE = 205
def _build(path, count: int) -> str:
db = sqlite3.connect(path)
for i in range(count):
db.execute(f"CREATE TABLE t{i:03d} (id INTEGER PRIMARY KEY, amount INTEGER)")
db.commit()
db.close()
return str(path)
@pytest.fixture
async def client():
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
yield c
async def _source(client, path: str, *, include: list[str] | None = None) -> tuple[str, str]:
name = f"wide_{uuid.uuid4().hex[:8]}"
r = await client.post("/api/datasources", json={"name": name, "kind": "sqlite", "database": path,
"readonly": True})
assert r.status_code in (200, 201), r.text
sid = r.json()["id"]
if include:
r = await client.put(f"/api/datasources/{sid}/introspect-scope",
json={"scope": {"mode": "rules", "include": include}})
assert r.status_code == 200, r.text
await introspect_now(client, sid)
return sid, name
@pytest.fixture
async def wide(client, tmp_path):
sid, name = await _source(client, _build(tmp_path / "wide.db", WIDE), include=["t0*", "t1*"])
yield sid, name
await drop_source(sid)
@pytest.fixture
async def narrow(client, tmp_path):
sid, name = await _source(client, _build(tmp_path / "narrow.db", 3))
yield sid, name
await drop_source(sid)
async def _row(sid: str) -> DataSource:
async with SessionLocal() as session:
row = await session.get(DataSource, sid)
session.expunge(row)
return row
async def test_catalog_list_says_how_many_tables_were_not_introspected(client, wide):
sid, _ = wide
body = (await client.get(f"/api/datasources/{sid}/catalog")).json()
assert len(body["tables"]) == 200
assert body["schema_truncated"] is True and body["schema_total"] == WIDE
async def test_catalog_list_of_a_complete_schema(client, narrow):
sid, _ = narrow
body = (await client.get(f"/api/datasources/{sid}/catalog")).json()
assert len(body["tables"]) == 3
assert body["schema_truncated"] is False and body["schema_total"] == 3
async def test_assistant_context_counts_the_real_total(wide):
from app.look.service import LookService
from app.look.tables import TableKind
sid, name = wide
kind = await TableKind.load([await _row(sid)])
looks = LookService([kind])
overview = await looks.overview()
assert f"共 {WIDE} 张表,探查范围内 200 张" in overview
# 点名查看一张范围内的表:记录里 total 是对象总数,另带 explored;文字里说清范围外的点名会按需加入
result = await looks.look({"op": "look", "need": "看一下 t000", "targets": [
{"kind": "table", "source": name, "name": "t000"}]})
(row,) = result.record["objects"]
assert row["total"] == WIDE and row["explored"] == 200 and "added_to_scope" not in row
assert f"共 {WIDE} 张表,探查范围内 200 张" in result.text
# 遇到不认识的表名点名 look 或按 keyword 查,别猜
assert f"db_schema__{name}" in result.text and "keyword" in result.text and "按需加入" in result.text
# 交回的表目录也写明只列了范围内的
assert f"共 {WIDE} 张表,探查范围内 200 张,下面只列范围内的" in await kind.index(None)
async def test_assistant_context_of_a_complete_schema_says_nothing_extra(narrow):
from app.look.service import LookService
from app.look.tables import TableKind
sid, name = narrow
looks = LookService([await TableKind.load([await _row(sid)])])
assert "只探查了" not in await looks.overview()
result = await looks.look({"op": "look", "need": "全部"})
(row,) = result.record["objects"]
assert row["total"] == 3 and "explored" not in row
assert "没有探查到" not in result.text and "只探查了" not in result.text
async def test_schema_tool_adds_an_out_of_scope_table_instead_of_denying_it(wide, narrow):
"""点名一张探查范围外的表不能说「没有」:它在对象索引里,当场读结构、加进范围(每次运行有上限)。以前回「数据源中
没有 X」,模型据此改用别的表,或者编一个答案。"""
from app.tools.datasource import build_datasource_tools
from app.tools.registry import ToolContext, Versions
sid, name = wide
async with SessionLocal() as session:
(tool,) = await build_datasource_tools([f"db_schema__{name}"], ToolContext(data_versions=Versions.CURRENT),
session)
text = await tool.coroutine(table="t203")
assert "amount" in text and "按需加入了探查范围" in text
row = await _row(sid)
assert "t203" in row.schema_cache["tables"]
# 探查完整的库:查不存在的表照旧说没有
async with SessionLocal() as session:
(tool,) = await build_datasource_tools([f"db_schema__{(await _row(narrow[0])).name}"],
ToolContext(data_versions=Versions.CURRENT), session)
assert "没有 nope" in await tool.coroutine(table="nope")
def test_legacy_truncated_cache_still_does_not_deny():
"""升级前只探查了前 200 张的老缓存(没有对象索引):查一张没探查到的表照旧说「只探查了」、请查数据字典。"""
from types import SimpleNamespace
from app.data.introspect import describe_table
row = SimpleNamespace(name="legacy", schema_cache={"tables": {"t000": {"columns": [{"name": "id"}]}},
"truncated": True, "total": 205})
text = describe_table(row, "t300")
assert "中没有" not in text and "只探查了" in text and "information_schema" in text