mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 00:21:58 +00:00
大库的 look 预筛与按需加入、剖析事实四处一致、有效记录条件按 quote_name 加引号的测试;对象索引批量更新改走按表的 executemany
为什么:A2 和遗留 e 的行为要有脚本化的验收,且能证明测试抓得住回归(去掉遮罩重滤、去掉 quote_name 强制引号后对应用例都失败, 已回跑确认)。 - tests/test_look_large.py:3,000 张表的合成库,探查范围 300 张 + 5 张有目录的表,其余只在对象索引里(注释写进索引,模拟 MySQL / PG 的表注释)。10 条脚本化需求挑中 ≥ 9 条、挑选提示词 < 20,000 字(挑选用按字面重合打分的假模型,只看得到交给它的 那份目录);挑中范围外的表按需加入、记 added_to_scope 和留痕;点名的每次查看最多 4 张、超了照实说;db_schema__ 的 keyword、 只列前 50 张、每次运行最多 8 张;查询照实列「未按目录检查的表」;目录搜索和「加入范围」;剖析的关系指向范围外时不说 「表结构里没有」。 - tests/test_profile_facts.py:look、运行时 db_schema、/schema、表助手四处同一段剖析事实和固定提示;剖析之后加遮罩,那一列 的范围不出现;没有剖析结果时表详情一字不差;上传表格快照不一致(含没记快照的)不给;剖析任务开始一张表时记下快照; 「上次剖析」取真正开始过的那一次。 - tests/test_valid_filter_quote.py:存进目录的写法和剖析核对查询都按 quote_name 给 Oracle 小写列加引号。 - source_objects.refresh_index:注释、类型、行数估算有变的行改用按表的 executemany(ORM 的按主键批量更新要求参数里写主键 列名,3,000 张表的库第二次刷新索引时报错)。 - db_schema__ 不传 table 时说清没列出的里有几张在探查范围外。
This commit is contained in:
@@ -480,11 +480,13 @@ async def refresh_index(session: AsyncSession, source_id: str, schema: str, resu
|
||||
for chunk in _chunks(new_rows):
|
||||
await session.execute(insert(_SO), chunk)
|
||||
if changed:
|
||||
stmt = (update(_SO).where(_SO.id == bindparam("_id"))
|
||||
.values(kind=bindparam("kind"), comment=bindparam("comment"), est_rows=bindparam("est_rows"))
|
||||
.execution_options(synchronize_session=False))
|
||||
# 按表(不按 ORM 实体)的 executemany:ORM 的按主键批量更新要求参数里写主键列的名字
|
||||
table = SourceObject.__table__
|
||||
stmt = (update(table).where(table.c.id == bindparam("_id"))
|
||||
.values(kind=bindparam("kind"), comment=bindparam("comment"), est_rows=bindparam("est_rows")))
|
||||
conn = await session.connection()
|
||||
for chunk in _chunks(changed):
|
||||
await session.execute(stmt, chunk)
|
||||
await conn.execute(stmt, chunk)
|
||||
still = [r.id for name, r in existing.items() if name in seen]
|
||||
for chunk in _chunks(still):
|
||||
await session.execute(update(_SO).where(_SO.id.in_(chunk)).values(last_seen_at=now)
|
||||
|
||||
@@ -854,7 +854,8 @@ def _schema_tables(table: str | list[str] | None) -> list[str]:
|
||||
#: db_schema 不传 table 时最多列几张(几千张表的库整份列出来没法看,也塞不进上下文)、按 keyword 检索给几个
|
||||
SCHEMA_LIST_LIMIT = 50
|
||||
SCHEMA_KEYWORD_LIMIT = 30
|
||||
SCHEMA_MORE_PROMPT = "…另有 {n} 张没有列出:用 keyword 按要查的内容检索,或者 table 直接点名"
|
||||
SCHEMA_MORE_PROMPT = "…另有 {n} 张没有列出{outside}:用 keyword 按要查的内容检索,或者 table 直接点名"
|
||||
SCHEMA_MORE_OUTSIDE = "(其中 {n} 张在探查范围外,点名时按需加入)"
|
||||
SCHEMA_KEYWORD_HEAD = "数据源「{source}」里和「{keyword}」相关的表(按相关程度排):"
|
||||
SCHEMA_KEYWORD_NONE = "数据源「{source}」里没有检索到和「{keyword}」相关的表:换个说法,或者 table 直接点名"
|
||||
SCHEMA_OUTSIDE_MARK = "(探查范围外:table 点名它会按需加入)"
|
||||
@@ -908,7 +909,9 @@ def _make_schema_tool(source: Any, *, fixed: bool = False,
|
||||
lines = [f" {n}" + ("(按原样导入、未经规整,不能直接对列求和)" if n in raw else "")
|
||||
+ (REPORTED_TOTAL_MARK if n in totals else "") for n in shown]
|
||||
if total > len(shown):
|
||||
lines.append(" " + SCHEMA_MORE_PROMPT.format(n=total - len(shown)))
|
||||
beyond = total - len(tables)
|
||||
lines.append(" " + SCHEMA_MORE_PROMPT.format(
|
||||
n=total - len(shown), outside=SCHEMA_MORE_OUTSIDE.format(n=beyond) if beyond > 0 else ""))
|
||||
return head + ":\n" + "\n".join(lines)
|
||||
|
||||
return StructuredTool(
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
"""大库下的 look、db_schema、目录搜索(docs/design/next-round/03-数据接入与目录维护.md A2)。
|
||||
|
||||
3,000 张表的合成库(tests/fixtures/catalog/large.py):探查范围 ord_*、sal_* 加上 5 张有数据目录的目标表,其余在范围外,
|
||||
对象索引里只有名字和注释。这组守的是:
|
||||
|
||||
- look 的挑表在候选超过 PREFILTER_LIMIT 时只给预筛结果(BM25)、已用到的表、一跳关联:10 条脚本化需求至少 9 条挑中,
|
||||
挑选提示词不到 20,000 字;挑中范围外的表按需加入,look 记录写 added_to_scope;
|
||||
- 点名范围外的表按需加入、留痕(谁、哪一轮、哪一次查看、需求原文);每次查看最多 ON_DEMAND_PER_LOOK 张,超了照实说;
|
||||
- 运行时 db_schema__:keyword 用同一个检索;不传 table 只列前 50 张;点名范围外的按需加入,每次运行最多 ON_DEMAND_PER_RUN 张;
|
||||
- 查询照实列「未按目录检查的表」;
|
||||
- 目录:GET /catalog/search 找得到范围外的表,POST /catalog/scope/add 加进来(记进规则的显式列出)。
|
||||
|
||||
挑选模型用一个按需求和目录行的字面重合打分的假模型(不调真模型):它只能看到交给它的那份目录,预筛漏掉的它也挑不到。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.data import catalog
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import DataSource
|
||||
from app.db.models_sources import SourceObject
|
||||
from app.look.base import ON_DEMAND_PER_LOOK, ON_DEMAND_PER_RUN, PICK_LIMIT
|
||||
from app.look.pick import PICK_HUMAN_PROMPT, PICK_SYSTEM_PROMPT, PICK_USED_PROMPT
|
||||
from app.look.service import LookService
|
||||
from app.look.tables import TableKind
|
||||
from app.main import app
|
||||
from tests.fixtures.catalog import large
|
||||
from tests.fixtures.sources import drop_source, introspect_now
|
||||
|
||||
|
||||
class OverlapPicker:
|
||||
"""假的挑选模型:按需求里的二字片段在目录行里出现的次数给每行打分,挑前 PICK_LIMIT 行。记下每次的提示词字数。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.prompts: list[int] = []
|
||||
self.calls = 0
|
||||
|
||||
async def pick(self, *, need: str, what: str, line: str, index: str, used: str = ""):
|
||||
human = PICK_HUMAN_PROMPT.format(need=need, used=PICK_USED_PROMPT.format(items=used) if used else "",
|
||||
what=what, line=line, index=index)
|
||||
self.prompts.append(len(PICK_SYSTEM_PROMPT) + len(human))
|
||||
self.calls += 1
|
||||
grams = {need[i:i + 2] for i in range(len(need) - 1) if re.match(r"[一-鿿]{2}", need[i:i + 2])}
|
||||
source, scored = None, []
|
||||
for row in index.splitlines():
|
||||
if m := re.match(r"数据源「(.+?)」", row):
|
||||
source = m.group(1)
|
||||
continue
|
||||
if not row.strip() or "|" not in row and not re.match(r"^[\w.]+", row):
|
||||
continue
|
||||
name = row.split("|", 1)[0].strip()
|
||||
score = sum(1 for g in grams if g in row)
|
||||
if score:
|
||||
scored.append((score, source, name))
|
||||
scored.sort(key=lambda x: -x[0])
|
||||
return [(s, n) for _score, s, n in scored[:PICK_LIMIT]]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def client():
|
||||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
|
||||
yield c
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def big_db(tmp_path_factory) -> str:
|
||||
return str(large.build(tmp_path_factory.mktemp("large-look") / "large.db"))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def big(client, big_db):
|
||||
"""探查范围 ord_*、sal_*(300 张)+ 5 张有目录的目标表(pinned);对象索引里写上全部注释(SQLite 没有表注释)。"""
|
||||
name = f"large_{uuid.uuid4().hex[:8]}"
|
||||
r = await client.post("/api/datasources", json={"name": name, "kind": "sqlite", "database": big_db,
|
||||
"readonly": True})
|
||||
sid = r.json()["id"]
|
||||
async with SessionLocal() as session:
|
||||
for t in large.TARGETS:
|
||||
if t.cataloged:
|
||||
await catalog.write_entry(session, sid, t.name, {"label": {"value": t.comment, "source": "human",
|
||||
"status": "confirmed"}},
|
||||
if_version=0, actor=None)
|
||||
r = await client.put(f"/api/datasources/{sid}/introspect-scope",
|
||||
json={"scope": {"mode": "rules", "include": ["ord_*", "sal_*"]}})
|
||||
assert r.status_code == 200
|
||||
out = await introspect_now(client, sid)
|
||||
assert out["table_count"] == 305
|
||||
# 探查之后再写注释:刷新对象索引会照 SQLite 的字典(没有注释)改回去
|
||||
notes = large.comments()
|
||||
async with SessionLocal() as session:
|
||||
for obj in (await session.execute(select(SourceObject).where(SourceObject.source_id == sid))).scalars():
|
||||
obj.comment = notes.get(obj.name)
|
||||
await session.commit()
|
||||
yield sid, name
|
||||
await drop_source(sid)
|
||||
|
||||
|
||||
async def _row(sid: str) -> DataSource:
|
||||
async with SessionLocal() as session:
|
||||
row = await session.get(DataSource, sid)
|
||||
session.expunge(row)
|
||||
return row
|
||||
|
||||
|
||||
async def test_prefiltered_pick_hits_at_least_9_of_10_with_a_short_prompt(big):
|
||||
sid, name = big
|
||||
picker = OverlapPicker()
|
||||
kind = await TableKind.load([await _row(sid)], picker=picker, trail={"actor": "tester", "conversation_id": "c1"})
|
||||
looks = LookService([kind])
|
||||
hits = 0
|
||||
added: list[str] = []
|
||||
for target in large.TARGETS:
|
||||
result = await looks.look({"op": "look", "need": target.need})
|
||||
objects = (result.record or {}).get("objects") or []
|
||||
picked = {n for o in objects for n in o.get("picked", [])}
|
||||
if target.name in picked:
|
||||
hits += 1
|
||||
for o in objects:
|
||||
assert o.get("prefiltered", 0) <= 120 + 40
|
||||
added += o.get("added_to_scope", [])
|
||||
assert hits >= 9, f"挑中 {hits}/10"
|
||||
assert picker.prompts and max(picker.prompts) < 20_000, f"挑选提示词 {max(picker.prompts)} 字"
|
||||
out_of_scope = {t.name for t in large.TARGETS if not t.cataloged}
|
||||
assert out_of_scope & set(added), "挑中范围外的表时按需加入,记在 added_to_scope"
|
||||
rows = {r.name: r for r in (await _objects(sid)) if r.name in out_of_scope and r.in_scope}
|
||||
assert rows and all(r.added_via == "look" and r.added_by == "tester" and r.added_conversation_id == "c1"
|
||||
and r.added_need and r.added_ref for r in rows.values())
|
||||
|
||||
|
||||
async def _objects(sid: str) -> list[SourceObject]:
|
||||
async with SessionLocal() as session:
|
||||
return list((await session.execute(select(SourceObject).where(SourceObject.source_id == sid))).scalars())
|
||||
|
||||
|
||||
async def test_naming_out_of_scope_tables_adds_them_with_a_per_look_cap(big):
|
||||
sid, name = big
|
||||
kind = await TableKind.load([await _row(sid)], picker=OverlapPicker(), trail={"actor": "tester"})
|
||||
looks = LookService([kind])
|
||||
wanted = ["shift_roster", "complaint_ticket", "waybill_track", "pur_hdr_east", "pur_line_east", "pur_log_east"]
|
||||
result = await looks.look({"op": "look", "need": "看这几张表的字段",
|
||||
"targets": [{"kind": "table", "source": name, "name": n} for n in wanted]})
|
||||
(obj,) = result.record["objects"]
|
||||
assert len(obj["added_to_scope"]) == ON_DEMAND_PER_LOOK == 4
|
||||
assert set(obj["named"]) == set(obj["added_to_scope"])
|
||||
assert "按需加入探查范围" in result.text and f"一次查看最多按需加入 {ON_DEMAND_PER_LOOK} 张" in result.text
|
||||
assert sorted(obj["missing"]) == sorted(wanted[4:]), "超了上限的照实写没找到、说清原因"
|
||||
assert "biz_date" in result.text, "加进来的表当场给字段"
|
||||
row = await _row(sid)
|
||||
assert all(n in row.schema_cache["tables"] for n in obj["added_to_scope"]), "落盘,下一轮、运行时都看得到"
|
||||
# 下一次查看重新算上限
|
||||
again = await looks.look({"op": "look", "need": "再看两张",
|
||||
"targets": [{"kind": "table", "source": name, "name": n} for n in wanted[4:]]})
|
||||
assert len(again.record["objects"][0]["added_to_scope"]) == 2
|
||||
|
||||
|
||||
async def test_db_schema_keyword_list_limit_and_per_run_cap(big):
|
||||
from app.tools.datasource import build_datasource_tools
|
||||
from app.tools.registry import ToolContext, Versions
|
||||
|
||||
sid, name = big
|
||||
async with SessionLocal() as session:
|
||||
tools = await build_datasource_tools([f"db_schema__{name}", f"db_query__{name}"],
|
||||
ToolContext(run_id="run-cap-1", node_id="n1",
|
||||
data_versions=Versions.CURRENT), session)
|
||||
schema = next(t for t in tools if t.name.startswith("db_schema__"))
|
||||
query = next(t for t in tools if t.name.startswith("db_query__"))
|
||||
listed = await schema.coroutine()
|
||||
assert listed.count("\n ") == 51 and "另有 2950 张没有列出(其中 2695 张在探查范围外" in listed and "keyword" in listed
|
||||
found = await schema.coroutine(keyword="员工排班")
|
||||
assert "shift_roster|员工排班表(探查范围外:table 点名它会按需加入)" in found
|
||||
|
||||
names = [f"pur_{e}_east" for e in ("hdr", "line", "log", "snap", "daily", "monthly", "cfg", "map", "hist")]
|
||||
text = await schema.coroutine(table=names)
|
||||
assert text.count("按需加入了探查范围") == ON_DEMAND_PER_RUN == 8
|
||||
assert f"这次运行最多按需加入 {ON_DEMAND_PER_RUN} 张" in text, "第 9 张照实说上限用完了"
|
||||
added = [r for r in await _objects(sid) if r.added_run_id == "run-cap-1"]
|
||||
assert len(added) == 8 and all(r.added_via == "db_schema" and r.added_ref == "n1" for r in added)
|
||||
# 查询工具看到的是同一份表结构:加进来的表不再列成「未按目录检查的表」,范围外的照实列
|
||||
import json
|
||||
|
||||
out = json.loads(await query.coroutine(sql="SELECT a.id FROM pur_hdr_east a JOIN pur_tmp_east b ON a.id = b.id"))
|
||||
assert out.get("unchecked_tables") == ["pur_tmp_east"]
|
||||
# 删掉了、从来没有的分开说
|
||||
assert "没有 no_such_table" in await schema.coroutine(table="no_such_table")
|
||||
|
||||
|
||||
async def test_catalog_search_and_add_to_scope(client, big):
|
||||
sid, name = big
|
||||
r = await client.get(f"/api/datasources/{sid}/catalog/search", params={"q": "投诉"})
|
||||
body = r.json()
|
||||
assert body["indexed"] is True and body["total"] >= 1
|
||||
hit = next(i for i in body["items"] if i["name"] == "complaint_ticket")
|
||||
assert hit["in_scope"] is False and hit["absent_reason"] == "out_of_scope" and hit["comment"] == "客户投诉工单"
|
||||
r = await client.get(f"/api/datasources/{sid}/catalog/search", params={"q": "ORD_HDR"})
|
||||
assert r.json()["items"][0]["name"].startswith("ord_hdr") and r.json()["items"][0]["in_scope"] is True
|
||||
|
||||
r = await client.post(f"/api/datasources/{sid}/catalog/scope/add",
|
||||
json={"tables": ["complaint_ticket", "nope_table", "ord_hdr_east"]})
|
||||
assert r.status_code == 200, r.text
|
||||
out = r.json()
|
||||
assert out["added"] == ["complaint_ticket"] and out["already"] == ["ord_hdr_east"]
|
||||
assert [x["name"] for x in out["refused"]] == ["nope_table"]
|
||||
row = await _row(sid)
|
||||
assert "complaint_ticket" in row.schema_cache["tables"]
|
||||
assert "complaint_ticket" in row.options["introspect_scope"]["tables"], "记进规则的显式列出,下次探查也带上"
|
||||
listed = (await client.get(f"/api/datasources/{sid}/catalog")).json()
|
||||
assert any(t["table_name"] == "complaint_ticket" for t in listed["tables"])
|
||||
scope = (await client.get(f"/api/datasources/{sid}/introspect-scope")).json()
|
||||
trail = next(t for t in scope["on_demand"] if t["name"] == "complaint_ticket")
|
||||
assert trail["via"] == "catalog" and trail["scope_reason"] == "explicit"
|
||||
|
||||
|
||||
async def test_out_of_scope_parent_is_not_called_missing(big):
|
||||
from app.data import catalog_profile
|
||||
|
||||
sid, _ = big
|
||||
row = await _row(sid)
|
||||
from app.data import catalog_profile_jobs
|
||||
|
||||
async with SessionLocal() as session:
|
||||
outside = await catalog_profile_jobs.outside_of(session, row)
|
||||
tables = row.schema_cache["tables"]
|
||||
rel = {"id": "r1", "columns": ["ord_id"], "to_table": "pur_hdr_west", "to_columns": ["id"], "status": "proposed",
|
||||
"source": "name"}
|
||||
checks = catalog_profile.plan_checks("ord_hdr_east", tables["ord_hdr_east"], {"relations": [rel]},
|
||||
tables=tables, masked=set(), outside=outside)
|
||||
(relation,) = [c for c in checks if c.kind == "relation"]
|
||||
assert relation.skip[0] == "out_of_scope" and "探查范围外" in relation.skip[1] and "重新探查" not in relation.skip[1]
|
||||
rel["to_table"] = "really_missing"
|
||||
checks = catalog_profile.plan_checks("ord_hdr_east", tables["ord_hdr_east"], {"relations": [rel]},
|
||||
tables=tables, masked=set(), outside=outside)
|
||||
assert [c for c in checks if c.kind == "relation"][0].skip[0] == "missing"
|
||||
@@ -0,0 +1,178 @@
|
||||
"""剖析事实进 look 的表详情(docs/design/next-round/05-已知遗留.md (e))。
|
||||
|
||||
剖析记下的行数估算和依据、各日期列的范围以前只在 catalog_profile_tasks 里,助手写 SQL 时不知道表有多大、数据到哪天。现在
|
||||
经 schema_view.profile_lines 成一节,look、运行时 db_schema、/schema、表助手四处同一段文字,附「只供写查询参考……不能
|
||||
作为回答里的数」。这组守的是:
|
||||
|
||||
- 四处文字一致;没有剖析结果时表详情和以前一字不差;
|
||||
- 遮罩按渲染那一刻重滤:剖析之后才给某一列加遮罩,那一列的范围不出现;
|
||||
- 上传表格只在剖析时的快照和这次读的快照一致时给,不一致(含升级前没记快照的)写「剖析早于当前版本,不给」;
|
||||
- 剖析任务开始一张表时记下上传表格的快照;
|
||||
- 「上次剖析」取真正开始过的最近一次(同表清单)。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
|
||||
from app.data import catalog_profile_jobs, schema_view
|
||||
from app.data.schema_view import PROFILE_HINT_PROMPT, PROFILE_STALE_PROMPT
|
||||
from app.db.base import SessionLocal, new_id
|
||||
from app.db.models import CatalogProfileJob, CatalogProfileTask, DataSource
|
||||
from app.main import app
|
||||
from tests.fixtures.sources import drop_source, introspect_now
|
||||
|
||||
FACT_LINE = "数据剖析(2026-10-01):约 12,345 行(数据库统计估算);order_date 从 2024-01-01 到 2026-09-30;paid_at 从 2024-01-02 到 2026-09-29 18:00:01"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def client():
|
||||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
|
||||
yield c
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def shop(client, tmp_path):
|
||||
path = tmp_path / "shop.db"
|
||||
db = sqlite3.connect(path)
|
||||
db.execute("CREATE TABLE orders (id INTEGER PRIMARY KEY, order_date TEXT, paid_at TEXT, amount REAL)")
|
||||
db.execute("CREATE TABLE stores (id INTEGER PRIMARY KEY, name TEXT)")
|
||||
db.commit()
|
||||
db.close()
|
||||
name = f"facts_{uuid.uuid4().hex[:8]}"
|
||||
r = await client.post("/api/datasources", json={"name": name, "kind": "sqlite", "database": str(path)})
|
||||
sid = r.json()["id"]
|
||||
await introspect_now(client, sid)
|
||||
yield sid, name
|
||||
await drop_source(sid)
|
||||
|
||||
|
||||
async def _profiled(sid: str, *, started: bool = True, snapshot: str | None = None, rows: int = 12345,
|
||||
at: str = "2026-10-01T08:00:00+00:00") -> None:
|
||||
"""记一次剖析:orders 有行数估算和两个日期列的范围。"""
|
||||
when = datetime.fromisoformat(at)
|
||||
async with SessionLocal() as session:
|
||||
job = CatalogProfileJob(id=new_id(), source_id=sid, status="succeeded", scope="selected",
|
||||
created_at=when, finished_at=when)
|
||||
session.add(job)
|
||||
await session.flush()
|
||||
session.add(CatalogProfileTask(
|
||||
job_id=job.id, seq=0, table_name="orders", status="ok" if started else "cancelled",
|
||||
row_estimate={"rows": rows, "method": "stats", "at_least": None} if started else None,
|
||||
date_ranges=[{"column": "order_date", "min": "2024-01-01", "max": "2026-09-30"},
|
||||
{"column": "paid_at", "min": "2024-01-02 00:00:00", "max": "2026-09-29 18:00:01"}]
|
||||
if started else None,
|
||||
snapshot_id=snapshot, started_at=when if started else None, finished_at=when))
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def _row(sid: str) -> DataSource:
|
||||
async with SessionLocal() as session:
|
||||
row = await session.get(DataSource, sid)
|
||||
session.expunge(row)
|
||||
return row
|
||||
|
||||
|
||||
async def _four_places(client, sid: str, name: str) -> dict[str, str]:
|
||||
from app.data import catalog_assistant
|
||||
from app.look.service import LookService
|
||||
from app.look.tables import TableKind
|
||||
from app.tools.datasource import build_datasource_tools
|
||||
from app.tools.registry import ToolContext, Versions
|
||||
|
||||
row = await _row(sid)
|
||||
looks = LookService([await TableKind.load([row])])
|
||||
look = (await looks.look({"op": "look", "need": "订单", "targets": [
|
||||
{"kind": "table", "source": name, "name": "orders"}]})).text
|
||||
async with SessionLocal() as session:
|
||||
(tool,) = await build_datasource_tools([f"db_schema__{name}"], ToolContext(data_versions=Versions.CURRENT),
|
||||
session)
|
||||
assistant = (await catalog_assistant.prepare(session, row, row, "orders")).system
|
||||
tool_text = await tool.coroutine(table="orders")
|
||||
api = (await client.get(f"/api/datasources/{sid}/schema", params={"table": "orders"})).json()["detail"]
|
||||
return {"look": look, "db_schema": tool_text, "/schema": api, "assistant": assistant}
|
||||
|
||||
|
||||
async def test_the_same_profile_facts_in_four_places(client, shop):
|
||||
sid, name = shop
|
||||
before = (await client.get(f"/api/datasources/{sid}/schema", params={"table": "orders"})).json()["detail"]
|
||||
await _profiled(sid)
|
||||
places = await _four_places(client, sid, name)
|
||||
for where, text in places.items():
|
||||
assert FACT_LINE in text, f"{where} 里没有那一段剖析事实:{text[-400:]}"
|
||||
assert PROFILE_HINT_PROMPT in text, f"{where} 里没有「只供写查询参考」"
|
||||
after = places["/schema"]
|
||||
assert after.startswith(before) and after != before, "剖析事实是追加的一节,表详情原有的部分不动"
|
||||
|
||||
# 剖析之后才给 paid_at 加遮罩:渲染那一刻重滤,它的范围不出现
|
||||
r = await client.patch(f"/api/datasources/{sid}", json={"options": {"mask_columns": "paid_at"}})
|
||||
assert r.status_code == 200, r.text
|
||||
for where, text in (await _four_places(client, sid, name)).items():
|
||||
assert "paid_at 从" not in text, f"{where} 给了遮罩列的范围"
|
||||
assert "order_date 从 2024-01-01 到 2026-09-30" in text, where
|
||||
|
||||
|
||||
async def test_without_profile_results_the_detail_is_unchanged(shop):
|
||||
sid, _ = shop
|
||||
row = await _row(sid)
|
||||
plain = schema_view.table_detail(row, "orders")
|
||||
async with SessionLocal() as session:
|
||||
facts = await catalog_profile_jobs.last_facts(session, sid)
|
||||
assert facts == {}
|
||||
assert schema_view.table_detail(row, "orders", profile=facts) == plain
|
||||
assert schema_view.table_detail(row, "orders", profile={"stores": {"row_estimate": {"rows": 1}}}) == plain
|
||||
|
||||
|
||||
async def test_last_facts_follow_the_last_profile_that_really_started(shop):
|
||||
sid, _ = shop
|
||||
await _profiled(sid, rows=500, at="2026-09-01T08:00:00+00:00")
|
||||
await _profiled(sid, started=False, at="2026-10-02T08:00:00+00:00")
|
||||
async with SessionLocal() as session:
|
||||
facts = await catalog_profile_jobs.last_facts(session, sid)
|
||||
assert await catalog_profile_jobs.last_facts(session, sid, []) == {}
|
||||
assert facts["orders"]["row_estimate"]["rows"] == 500 and facts["orders"]["at"].startswith("2026-09-01"), \
|
||||
"没开始就停了的那一项不挡掉上一次真正剖析的结果"
|
||||
|
||||
|
||||
def test_upload_sources_only_get_facts_of_the_current_snapshot():
|
||||
facts = {"at": "2026-10-01T00:00:00+00:00", "snapshot_id": "snap-a",
|
||||
"row_estimate": {"rows": 3, "method": "count", "at_least": None}, "date_ranges": []}
|
||||
assert schema_view.profile_lines(facts, set(), "snap-a", upload=True)[0].endswith("3 行(逐行数出)")
|
||||
assert schema_view.profile_lines(facts, set(), "snap-b", upload=True) == [" " + PROFILE_STALE_PROMPT]
|
||||
assert schema_view.profile_lines({**facts, "snapshot_id": None}, set(), "snap-a", upload=True) \
|
||||
== [" " + PROFILE_STALE_PROMPT], "升级前没记快照的剖析,上传表格在重新剖析之前不给"
|
||||
at_least = {**facts, "row_estimate": {"rows": None, "method": "count", "at_least": 100000}}
|
||||
assert "至少 100,000 行(数到上限为止)" in schema_view.profile_lines(at_least, set(), None)[0]
|
||||
unknown = {**facts, "row_estimate": {"rows": None, "method": "unknown", "at_least": None}}
|
||||
assert schema_view.profile_lines(unknown, set(), None) == [], "什么都没有就不出这一节"
|
||||
|
||||
view = SimpleNamespace(name="up", origin="upload", snapshot_id="snap-b", options={}, kind="sqlite",
|
||||
schema_cache={"tables": {"t": {"qualified": "t", "columns": [{"name": "id", "type": "INTEGER"}]}}})
|
||||
text = schema_view.table_detail(view, "t", profile={"t": facts})
|
||||
assert PROFILE_STALE_PROMPT in text and "3 行" not in text
|
||||
|
||||
|
||||
async def test_profile_job_records_the_snapshot_of_an_upload_table(shop):
|
||||
sid, _ = shop
|
||||
from app.data.catalog_profile_jobs import _ProfileRun
|
||||
|
||||
async with SessionLocal() as session:
|
||||
job = CatalogProfileJob(id=new_id(), source_id=sid, status="running", scope="selected",
|
||||
created_at=datetime.now(timezone.utc))
|
||||
session.add(job)
|
||||
await session.flush()
|
||||
session.add(CatalogProfileTask(job_id=job.id, seq=0, table_name="orders", status="pending"))
|
||||
await session.commit()
|
||||
run = _ProfileRun(job.id)
|
||||
run.cx = SimpleNamespace(source=SimpleNamespace(snapshot_id="snap-z"))
|
||||
async with SessionLocal() as session:
|
||||
await run._start_table(session, 0)
|
||||
async with SessionLocal() as session:
|
||||
task = (await session.get(CatalogProfileJob, job.id)) and (await session.execute(
|
||||
__import__("sqlalchemy").select(CatalogProfileTask).where(CatalogProfileTask.job_id == job.id))).scalar_one()
|
||||
assert task.snapshot_id == "snap-z" and task.status == "running"
|
||||
@@ -0,0 +1,39 @@
|
||||
"""有效记录条件按 quote_name 强制加引号(第 1 波留下的一处,next-round 03 C1 ③)。
|
||||
|
||||
Oracle 库里本来就是小写的列(建表时写了 "status")探查时记了 quote_name。拼有效记录条件时只按 SqlDialect.quote(要加才加)
|
||||
写成不带引号的 status,Oracle 转成大写 STATUS,找的是另一列。存进目录的写法(stored_valid_filter)和剖析核对查询
|
||||
(check_valid_filter)都要按表结构里的 quote_name 加引号——两处是同一条。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
from app.data import catalog_profile, sqlcheck
|
||||
|
||||
META = {"qualified": "ANALYTICS.visits", "schema": "ANALYTICS", "quote_name": True,
|
||||
"columns": [{"name": "status", "type": "VARCHAR2(10 CHAR)", "nullable": True, "quote_name": True},
|
||||
{"name": "amount", "type": "NUMBER(12, 2)", "nullable": True}]}
|
||||
|
||||
|
||||
def test_stored_filter_quotes_columns_recorded_with_quote_name():
|
||||
parsed = sqlcheck.parse_valid_filter("status <> 'X' AND amount > 0", META, kind="oracle", table="visits")
|
||||
assert parsed.problem is None
|
||||
assert sqlcheck.stored_valid_filter(parsed, "oracle", META) == "\"status\" <> 'X' AND amount > 0"
|
||||
# 不给表结构时照旧按方言要加才加(老的调用方)
|
||||
assert sqlcheck.stored_valid_filter(parsed, "oracle") == "status <> 'X' AND amount > 0"
|
||||
|
||||
|
||||
async def test_the_profile_check_quotes_the_same_way(monkeypatch):
|
||||
seen: list[str] = []
|
||||
|
||||
async def fake(source, sql, **kw):
|
||||
seen.append(sql)
|
||||
return SimpleNamespace(rows=[[10, 2]])
|
||||
|
||||
monkeypatch.setattr(catalog_profile, "run_single_query", fake)
|
||||
source = SimpleNamespace(kind="oracle", enabled=True, options={catalog_profile.PROFILE_OPTION: {"enabled": True}})
|
||||
found = await catalog_profile.check_valid_filter(source, "visits", META, "status <> 'X'")
|
||||
assert found.rows == 10 and len(seen) == 1
|
||||
sql = seen[0]
|
||||
assert "CASE WHEN \"status\" <> 'X' THEN 0" in sql, sql
|
||||
assert "SELECT \"status\" FROM \"ANALYTICS\".\"visits\"" in sql, "取的列、表名也按 quote_name 加引号"
|
||||
Reference in New Issue
Block a user