mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-17 01:31:59 +00:00
1a6082f033
为什么:几千张表的库以前用不了——探查每个源最多取 200 个对象、按数据库列出的顺序截断,同步请求、整份替换、不排队; 落在窗口外的表在目录、look、剖析里被说成「表已不在」。 做了什么: - db/models_sources.py:source_objects(对象索引一行一个对象:范围、原因、结构指纹与 struct_version、first_seen / last_seen / removed_at、reflected_at、按需加入的留痕)、introspect_jobs / introspect_tasks(后台探查任务与逐表状态)。 - data/source_objects.py:每种方言一条字典查询建对象索引(失败退回 Inspector,上限 5 万,截断时不判删除);探查范围规则 (options.introspect_scope:通配、按 name_key、排除优先、显式列出 / 移出;有目录的表、已发布模板写死 SQL 用到的表 pinned; 命中超上限拒绝并说出命中数);唯一的 rebuild_schema_cache 从范围内已反射的行物化 schema_cache(形状不变,另加 scope 摘要); 三种「不在」分开说(removed / missing / out_of_scope / reflect_failed);按需加入(当场只读反射、落盘、留痕、进程内按源加锁)。 - data/introspect_jobs.py:探查改成后台任务(core/job_runtime,lane kind introspect,和同源起草、剖析串行;按批反射、逐表记状态、 可停止、读完的照样物化;重启标中断不续跑)。 - api/datasources.py:POST /introspect 返回 202 和任务,dry_run 同步只刷新索引、返回命中数;/introspect-jobs 读任务、停止; /introspect-scope 读、预览、保存规则;/schema?table= 带剖析事实、说清三种「不在」;DataSourceOut 加 schema_scope、introspect_job。 - schema_view:search_tables(BM25 倒排,遮罩列只用列名,范围外只按名字和注释、权重低一档)、profile_lines(剖析事实一节, 附「只供写查询参考」,遮罩按渲染那一刻重滤,上传表格快照不一致时不给)、absent_detail。 - look:候选超过 PREFILTER_LIMIT 时只给预筛结果 + 已用到的表 + 一跳关联;点名或挑中范围外的表按需加入(每次查看最多 4 张), look 记录加 added_to_scope、prefiltered;表详情带剖析事实。 - tools/datasource:db_schema__ 加 keyword、不传 table 时只列前 50 张;点名范围外的表按需加入(每次运行最多 8 张);查询结果 照实列「未按目录检查的表」(unchecked_tables,不新增检查规则);建工具时读剖析事实。 - 剖析:_relation_ends 分清「范围外,可以加入」和「表结构里没有」;任务记上传表格的 snapshot_id;last_facts。表助手、/schema 用同一段剖析事实。 - 目录接口:清单只列范围内,absent_reason 分开说;GET /catalog/search 查对象索引,POST /catalog/scope/add 加入范围。 - 第 1 波留下的一处:check_valid_filter 和 stored_valid_filter 拼有效记录条件时按 quote_name 强制加引号。 - 工具参数纠错:空着的可选参数不止一个时,只认拼写明显接近的那一个(db_schema 多了 keyword 以后 tabel 仍能纠成 table)。 - 测试:3,000 张表的合成库(对象索引、缺省范围超上限拒绝、规则圈 400 张后台读完、停止、重启标中断、和剖析排队、同步起草 409)、删除与结构版本、Inspector 回退、截断不判删除、范围规则、pinned、方言金样;改了同步探查的老测试改为等任务做完。
160 lines
7.1 KiB
Python
160 lines
7.1 KiB
Python
"""探查范围外的表要说出来(数据目录完整性审查 F3 / B4;next-round 03 A1 之后按探查范围说)。
|
||
|
||
以前探查结构每个数据源最多取 200 张表,多出来的不在表结构缓存里,哪儿都没说。现在全库对象进对象索引,探查范围(规则)
|
||
圈出要读结构的那部分;范围外的表在对象索引里有名字。这组守的是:
|
||
|
||
- 目录清单接口返回 schema_truncated(有范围外的表)、schema_total(对象索引里一共几张),界面据此说明
|
||
- 助手的概览写「共 205 张表,探查范围内 200 张」;查看记录(look 事件)的 total 是对象总数,另带 explored(范围内几张)
|
||
- 查看结果里写明范围外的点名查看会按需加入;交回的表目录也写明只列了范围内的
|
||
- 运行时 db_schema 点名范围外的表:按需加入,给出字段
|
||
- 没有范围外的库照旧,不多说一句;升级前只探查了一部分的老缓存照旧说「只探查了」
|
||
|
||
合成库:205 张空表,探查范围规则只收 t0*、t1*(200 张),测试里临时建,用完连同数据源一起删掉。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import sqlite3
|
||
import uuid
|
||
|
||
import pytest
|
||
from httpx import ASGITransport, AsyncClient
|
||
|
||
from app.db.base import SessionLocal
|
||
from app.db.models import DataSource
|
||
from app.main import app
|
||
from tests.fixtures.sources import drop_source, introspect_now
|
||
|
||
WIDE = 205
|
||
|
||
|
||
def _build(path, count: int) -> str:
|
||
db = sqlite3.connect(path)
|
||
for i in range(count):
|
||
db.execute(f"CREATE TABLE t{i:03d} (id INTEGER PRIMARY KEY, amount INTEGER)")
|
||
db.commit()
|
||
db.close()
|
||
return str(path)
|
||
|
||
|
||
@pytest.fixture
|
||
async def client():
|
||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
|
||
yield c
|
||
|
||
|
||
async def _source(client, path: str, *, include: list[str] | None = None) -> tuple[str, str]:
|
||
name = f"wide_{uuid.uuid4().hex[:8]}"
|
||
r = await client.post("/api/datasources", json={"name": name, "kind": "sqlite", "database": path,
|
||
"readonly": True})
|
||
assert r.status_code in (200, 201), r.text
|
||
sid = r.json()["id"]
|
||
if include:
|
||
r = await client.put(f"/api/datasources/{sid}/introspect-scope",
|
||
json={"scope": {"mode": "rules", "include": include}})
|
||
assert r.status_code == 200, r.text
|
||
await introspect_now(client, sid)
|
||
return sid, name
|
||
|
||
|
||
@pytest.fixture
|
||
async def wide(client, tmp_path):
|
||
sid, name = await _source(client, _build(tmp_path / "wide.db", WIDE), include=["t0*", "t1*"])
|
||
yield sid, name
|
||
await drop_source(sid)
|
||
|
||
|
||
@pytest.fixture
|
||
async def narrow(client, tmp_path):
|
||
sid, name = await _source(client, _build(tmp_path / "narrow.db", 3))
|
||
yield sid, name
|
||
await drop_source(sid)
|
||
|
||
|
||
async def _row(sid: str) -> DataSource:
|
||
async with SessionLocal() as session:
|
||
row = await session.get(DataSource, sid)
|
||
session.expunge(row)
|
||
return row
|
||
|
||
|
||
async def test_catalog_list_says_how_many_tables_were_not_introspected(client, wide):
|
||
sid, _ = wide
|
||
body = (await client.get(f"/api/datasources/{sid}/catalog")).json()
|
||
assert len(body["tables"]) == 200
|
||
assert body["schema_truncated"] is True and body["schema_total"] == WIDE
|
||
|
||
|
||
async def test_catalog_list_of_a_complete_schema(client, narrow):
|
||
sid, _ = narrow
|
||
body = (await client.get(f"/api/datasources/{sid}/catalog")).json()
|
||
assert len(body["tables"]) == 3
|
||
assert body["schema_truncated"] is False and body["schema_total"] == 3
|
||
|
||
|
||
async def test_assistant_context_counts_the_real_total(wide):
|
||
from app.look.service import LookService
|
||
from app.look.tables import TableKind
|
||
|
||
sid, name = wide
|
||
kind = await TableKind.load([await _row(sid)])
|
||
looks = LookService([kind])
|
||
overview = await looks.overview()
|
||
assert f"共 {WIDE} 张表,探查范围内 200 张" in overview
|
||
# 点名查看一张范围内的表:记录里 total 是对象总数,另带 explored;文字里说清范围外的点名会按需加入
|
||
result = await looks.look({"op": "look", "need": "看一下 t000", "targets": [
|
||
{"kind": "table", "source": name, "name": "t000"}]})
|
||
(row,) = result.record["objects"]
|
||
assert row["total"] == WIDE and row["explored"] == 200 and "added_to_scope" not in row
|
||
assert f"共 {WIDE} 张表,探查范围内 200 张" in result.text
|
||
# 遇到不认识的表名点名 look 或按 keyword 查,别猜
|
||
assert f"db_schema__{name}" in result.text and "keyword" in result.text and "按需加入" in result.text
|
||
# 交回的表目录也写明只列了范围内的
|
||
assert f"共 {WIDE} 张表,探查范围内 200 张,下面只列范围内的" in await kind.index(None)
|
||
|
||
|
||
async def test_assistant_context_of_a_complete_schema_says_nothing_extra(narrow):
|
||
from app.look.service import LookService
|
||
from app.look.tables import TableKind
|
||
|
||
sid, name = narrow
|
||
looks = LookService([await TableKind.load([await _row(sid)])])
|
||
assert "只探查了" not in await looks.overview()
|
||
result = await looks.look({"op": "look", "need": "全部"})
|
||
(row,) = result.record["objects"]
|
||
assert row["total"] == 3 and "explored" not in row
|
||
assert "没有探查到" not in result.text and "只探查了" not in result.text
|
||
|
||
|
||
async def test_schema_tool_adds_an_out_of_scope_table_instead_of_denying_it(wide, narrow):
|
||
"""点名一张探查范围外的表不能说「没有」:它在对象索引里,当场读结构、加进范围(每次运行有上限)。以前回「数据源中
|
||
没有 X」,模型据此改用别的表,或者编一个答案。"""
|
||
from app.tools.datasource import build_datasource_tools
|
||
from app.tools.registry import ToolContext, Versions
|
||
|
||
sid, name = wide
|
||
async with SessionLocal() as session:
|
||
(tool,) = await build_datasource_tools([f"db_schema__{name}"], ToolContext(data_versions=Versions.CURRENT),
|
||
session)
|
||
text = await tool.coroutine(table="t203")
|
||
assert "amount" in text and "按需加入了探查范围" in text
|
||
row = await _row(sid)
|
||
assert "t203" in row.schema_cache["tables"]
|
||
|
||
# 探查完整的库:查不存在的表照旧说没有
|
||
async with SessionLocal() as session:
|
||
(tool,) = await build_datasource_tools([f"db_schema__{(await _row(narrow[0])).name}"],
|
||
ToolContext(data_versions=Versions.CURRENT), session)
|
||
assert "没有 nope" in await tool.coroutine(table="nope")
|
||
|
||
|
||
def test_legacy_truncated_cache_still_does_not_deny():
|
||
"""升级前只探查了前 200 张的老缓存(没有对象索引):查一张没探查到的表照旧说「只探查了」、请查数据字典。"""
|
||
from types import SimpleNamespace
|
||
|
||
from app.data.introspect import describe_table
|
||
|
||
row = SimpleNamespace(name="legacy", schema_cache={"tables": {"t000": {"columns": [{"name": "id"}]}},
|
||
"truncated": True, "total": 205})
|
||
text = describe_table(row, "t300")
|
||
assert "中没有" not in text and "只探查了" in text and "information_schema" in text
|