mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 04:01:58 +00:00
审阅队列与进度按约定改:field_label 写给人看的完整位置(「表的有效记录条件」「列 status 的码值」「指向 members 的关联关系」,和表助手提案、导入冲突的说法同一个函数),列级的项另带 column;排在这里的原因写「被 N 次运行用到」,数的是运行(同「用过这张表的运行」的 total),同一次运行里查了几次、使用次数比运行次数多时两个都写(「被 2 次运行用到,使用 5 次」),table_usage 和优先级照旧按使用次数(同目录清单的「使用 N 次」和排序);进度里的常用 20 张表和起草的默认范围是同一批(使用次数从多到少,用到的不够 20 张时按表结构的顺序补齐),top 只给 tables、items_total、items_confirmed,all 只给约定的四项;表助手背景里写这张表被几次运行用到、查询几次
This commit is contained in:
@@ -57,7 +57,7 @@ from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.data import catalog, catalog_jobs, catalog_profile
|
||||
from app.data.catalog_review import SOURCE_LABEL, slot_path
|
||||
from app.data.catalog_review import SOURCE_LABEL, item_label, slot_path, table_usage_counts
|
||||
from app.data.draft_model import DRAFT_MODEL_PLACE
|
||||
from app.data.engine import SnapshotTampered, masked_columns
|
||||
|
||||
@@ -153,7 +153,8 @@ ASSISTANT_PROMPT_LABELS = {
|
||||
"view": "(视图)",
|
||||
"comment": "数据库注释:{text}",
|
||||
"system": "系统说明:{text}",
|
||||
"usage": "这张表被运行用过 {n} 次。",
|
||||
"usage": "这张表被 {runs} 次运行用到(查询 {n} 次)。",
|
||||
"usage_none": "这张表还没有被运行用到过。",
|
||||
"description": "数据源说明:{text}",
|
||||
"guidance": "数据源的起草要求(使用者补充的背景和要求):{text}",
|
||||
"structure": "表结构:",
|
||||
@@ -469,7 +470,8 @@ async def prepare(session: AsyncSession, row: Any, source: Any, table: str,
|
||||
entries = await catalog.read_catalog(session, source.id)
|
||||
entry = entries.get(table)
|
||||
notes = entry.notes if entry else {}
|
||||
usage = (await catalog.table_usage(session, source)).get(table, 0)
|
||||
usage = await table_usage_counts(session, source)
|
||||
used, runs = usage.queries.get(table, 0), usage.runs.get(table, 0)
|
||||
blank_n, blank_cols = await _last_blank(session, source.id, table)
|
||||
|
||||
lines = [labels["head"], labels["table"].format(source=row.name, kind=row.kind, table=table,
|
||||
@@ -477,7 +479,7 @@ async def prepare(session: AsyncSession, row: Any, source: Any, table: str,
|
||||
comment = str(meta.get("comment") or "").strip()
|
||||
if comment:
|
||||
lines.append((labels["system"] if system_notes else labels["comment"]).format(text=comment))
|
||||
lines.append(labels["usage"].format(n=usage))
|
||||
lines.append(labels["usage"].format(runs=runs, n=used) if used else labels["usage_none"])
|
||||
description = str(getattr(row, "description", "") or "").strip()
|
||||
if description:
|
||||
lines.append(labels["description"].format(text=description))
|
||||
@@ -693,11 +695,9 @@ class _Tools:
|
||||
|
||||
|
||||
def _where(key: catalog.Slot, change: catalog.PatchChange) -> str:
|
||||
"""提案里一项的中文说法:「列 status 的码值」「表的有效记录条件」「指向 members 的关联关系」。"""
|
||||
"""提案里一项的中文说法:「列 status 的码值」「表的有效记录条件」「指向 members 的关联关系」(同审阅队列的 field_label)。"""
|
||||
value = change.after if isinstance(change.after, Mapping) else change.before
|
||||
if key[0] == "r" and isinstance(value, Mapping) and value.get("to_table"):
|
||||
return f"指向 {value['to_table']} 的关联关系"
|
||||
return catalog.describe_slot(key)
|
||||
return item_label(key, value if isinstance(value, Mapping) else None)
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
|
||||
@@ -6,8 +6,8 @@
|
||||
|
||||
- **队列**(review_queue,GET /catalog/review-queue):只列推断(proposed)的项。priority = 使用次数权重 × 风险权重 ×
|
||||
来源权重,从高到低;同分按表名、路径排,结果稳定。
|
||||
- 使用次数权重:1 + log2(1 + 使用次数)。使用次数同目录清单(catalog.table_usage):按对数增长,常用表在前,但不至于
|
||||
让常用表上的中文名压过所有表上影响数字的项。
|
||||
- 使用次数权重:1 + log2(1 + 使用次数)。使用次数同目录清单(catalog.table_usage,清单的「使用 N 次」和排序、起草的默认
|
||||
范围都按它):按对数增长,常用表在前,但不至于让常用表上的中文名压过所有表上影响数字的项。
|
||||
- 风险权重(RISK_WEIGHT):有效记录条件、去重规则、业务日期、关联关系最高(直接改变数字),业务主键、度量类型次之
|
||||
(计数和能不能加总),码值、粒度、表类型再次,中文名、说明、含义、单位最低。
|
||||
- 来源权重(SOURCE_WEIGHT):模型起草、命名推断是猜的,比数据库注释优先;数据剖析(码值候选、业务日期、没核实的关系)
|
||||
@@ -15,14 +15,19 @@
|
||||
- 两组(kind):facts 影响数字的项(有效记录条件、去重规则、业务日期、关联关系、业务主键、度量类型);names 名称和说明
|
||||
(中文名、说明、含义、单位、码值、粒度、表类型)。表的「说明」和中文名、含义一样只是说明文字,归 names。
|
||||
- 表结构里已经没有的表、列上的项不列(确认了也没有用,目录页上另有清理的提示)。
|
||||
- reasons 用中文说为什么排在这里:常用表(使用 N 次)、影响数字的项(有效记录条件)、模型起草,未确认……
|
||||
- field_label 是这一项给人看的位置:「表的有效记录条件」「列 status 的码值」「指向 members 的关联关系」(item_label,
|
||||
和表助手提案、导入冲突里的说法一致);列级的项另给 column。
|
||||
- table_usage 是所在表的使用次数(同清单)。reasons 用中文说为什么排在这里:常用表(被 N 次运行用到)、影响数字的项
|
||||
(有效记录条件)、模型起草,未确认……「被 N 次运行用到」数的是运行(同 /usage 的 total);同一次运行里查了几次、
|
||||
使用次数比运行次数多时两个都写(「被 2 次运行用到,使用 5 次」),和清单、运行列表上的数都对得上。
|
||||
- **批量审阅**(review_batch,POST /catalog/review/batch):逐表按 if_version 乐观锁,同一张表的几条在一次写入里完成,
|
||||
一张表冲突、写不进去不影响别的表;路径不对、表不存在逐条报(results 和请求一一对应)。审阅的规则同单项审阅
|
||||
(catalog.review_item)。
|
||||
- **被哪些运行用过**(table_runs,GET /catalog/{table}/usage):取法同使用次数(catalog.query_snapshot_tables),按运行
|
||||
去重,新的在前。
|
||||
- **进度**(review_progress,GET /catalog/review-progress):使用次数前 TOP_N 张表(只算被运行用到过的)、全部、影响
|
||||
数字的项各自的确认情况。项的范围和队列一样:表结构里没有的表、列不算。
|
||||
去重,新的在前;total 是运行次数。
|
||||
- **进度**(review_progress,GET /catalog/review-progress):常用表(使用次数前 TOP_N 张,和起草对话框「使用次数最多的
|
||||
20 张表」是同一批:catalog.default_tables 的取法,用到的不够 TOP_N 张时按表结构的顺序补齐)、全部、影响数字的项各自的
|
||||
确认情况。项的范围和队列一样:表结构里没有的表、列不算。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -44,7 +49,7 @@ KINDS = ("all", "facts", "names")
|
||||
QUEUE_LIMIT_MAX = 200
|
||||
BATCH_MAX = 200
|
||||
RUNS_LIMIT_MAX = 100
|
||||
#: 进度里的「常用表」取使用次数前几张
|
||||
#: 进度里的「常用表」取使用次数前几张(同起草的默认范围 catalog.DRAFT_DEFAULT_TABLES)
|
||||
TOP_N = 20
|
||||
|
||||
#: 每一项属于哪一组、风险几档:键是 (t, 字段) / (c, 字段) / (r, ""),档 4 最高
|
||||
@@ -64,9 +69,10 @@ SOURCE_WEIGHT = {"llm": 1.5, "name": 1.5, "profile": 1.25, "comment": 1.0}
|
||||
SOURCE_LABEL = {"comment": "数据库注释", "fk": "外键约束", "name": "命名推断", "profile": "数据剖析", "llm": "模型起草",
|
||||
"human": "人工填写"}
|
||||
RELATION_LABEL = "关联关系"
|
||||
#: reasons 的说法
|
||||
REASON_TOP = "常用表(使用 {n} 次)"
|
||||
REASON_USED = "使用 {n} 次"
|
||||
#: reasons 的说法:常用表(被几次运行用到);使用次数比运行次数多(同一次运行里查了几次)时两个都写
|
||||
REASON_TOP = "常用表({used})"
|
||||
USED_RUNS = "被 {runs} 次运行用到"
|
||||
USED_RUNS_AND_QUERIES = "被 {runs} 次运行用到,使用 {n} 次"
|
||||
REASON_RISK = {4: "影响数字的项({label})", 3: "影响计数和汇总的项({label})", 2: "帮助理解取值的项({label})",
|
||||
1: "名称和说明({label})"}
|
||||
REASON_PENDING_CODES = "{n} 个码值的含义待填写"
|
||||
@@ -109,8 +115,8 @@ def field_name(key: Slot) -> str:
|
||||
return "relation" if key[0] == "r" else key[-1]
|
||||
|
||||
|
||||
def field_label(key: Slot) -> str:
|
||||
"""字段的界面叫法:中文名、码值、有效记录条件、关联关系……"""
|
||||
def field_text(key: Slot) -> str:
|
||||
"""字段的界面叫法(不带列名):中文名、码值、有效记录条件、关联关系……"""
|
||||
if key[0] == "t":
|
||||
return catalog.TABLE_FIELD_LABEL.get(key[1], key[1])
|
||||
if key[0] == "c":
|
||||
@@ -118,6 +124,13 @@ def field_label(key: Slot) -> str:
|
||||
return RELATION_LABEL
|
||||
|
||||
|
||||
def item_label(key: Slot, item: Mapping[str, Any] | None = None) -> str:
|
||||
"""一项给人看的位置:「表的有效记录条件」「列 status 的码值」「指向 members 的关联关系」(不知道指向哪张表时写编号)。"""
|
||||
if key[0] == "r" and isinstance(item, Mapping) and item.get("to_table"):
|
||||
return f"指向 {item['to_table']} 的{RELATION_LABEL}"
|
||||
return catalog.describe_slot(key)
|
||||
|
||||
|
||||
def item_value(key: Slot, item: Mapping[str, Any]) -> Any:
|
||||
"""一项的值:关联关系是两端、基数和覆盖率,别的是 value。"""
|
||||
if key[0] == "r":
|
||||
@@ -183,18 +196,49 @@ def priority(usage: int, key: Slot, source: Any) -> float:
|
||||
|
||||
|
||||
def top_tables(tables: Mapping[str, Any], usage: Mapping[str, int]) -> list[str]:
|
||||
"""使用次数前 TOP_N 张表:只算被运行用到过的,同样多的按表结构里的顺序。"""
|
||||
"""使用次数前 TOP_N 张表(同 catalog.default_tables,起草对话框的「使用次数最多的 20 张表」):使用次数从多到少,
|
||||
一样多的(包括都没用过的)按表结构里的顺序。表结构里不到 TOP_N 张时就是全部。"""
|
||||
order = {name: i for i, name in enumerate(tables)}
|
||||
used = [name for name in tables if usage.get(name, 0) > 0]
|
||||
return sorted(used, key=lambda n: (-usage[n], order[n]))[:TOP_N]
|
||||
return sorted(tables, key=lambda n: (-usage.get(n, 0), order[n]))[:TOP_N]
|
||||
|
||||
|
||||
def _reasons(key: Slot, item: Mapping[str, Any], usage: int, top: bool) -> list[str]:
|
||||
@dataclass(frozen=True)
|
||||
class Usage:
|
||||
"""每张表用得多不多(没被用过的表不出现)。
|
||||
|
||||
queries:使用次数,同 catalog.table_usage(目录清单的「使用 N 次」、清单的排序、起草的默认范围);
|
||||
runs:用到它的运行有几次,同「被哪些运行用过」(table_runs)的 total。同一次运行里查了几次这张表,queries 比 runs 多。
|
||||
"""
|
||||
|
||||
queries: dict[str, int]
|
||||
runs: dict[str, int]
|
||||
|
||||
|
||||
async def table_usage_counts(session: AsyncSession, source: Any) -> Usage:
|
||||
"""一次读出每张表的使用次数和运行次数(同一份查询快照引用:catalog.query_snapshot_tables)。"""
|
||||
queries: Counter[str] = Counter()
|
||||
runs: dict[str, set[str]] = {}
|
||||
for run, hit in await catalog.query_snapshot_tables(session, source):
|
||||
queries.update(hit)
|
||||
for name in hit:
|
||||
runs.setdefault(name, set()).add(run)
|
||||
return Usage(dict(queries), {name: len(ids) for name, ids in runs.items()})
|
||||
|
||||
|
||||
def usage_text(queries: int, runs: int) -> str:
|
||||
"""「被 2 次运行用到」;使用次数比运行次数多时「被 2 次运行用到,使用 5 次」。"""
|
||||
if queries > runs:
|
||||
return USED_RUNS_AND_QUERIES.format(runs=f"{runs:,}", n=f"{queries:,}")
|
||||
return USED_RUNS.format(runs=f"{runs:,}")
|
||||
|
||||
|
||||
def _reasons(key: Slot, item: Mapping[str, Any], usage: int, runs: int, top: bool) -> list[str]:
|
||||
out: list[str] = []
|
||||
if usage > 0:
|
||||
out.append((REASON_TOP if top else REASON_USED).format(n=f"{usage:,}"))
|
||||
used = usage_text(usage, runs)
|
||||
out.append(REASON_TOP.format(used=used) if top else used)
|
||||
_, tier = FIELD_GROUP[field_of(key)]
|
||||
out.append(REASON_RISK[tier].format(label=field_label(key)))
|
||||
out.append(REASON_RISK[tier].format(label=field_text(key)))
|
||||
if key[0] == "c" and key[2] == "codes" and isinstance(item.get("value"), Mapping):
|
||||
pending = sum(1 for v in item["value"].values() if not str(v).strip())
|
||||
if pending:
|
||||
@@ -225,14 +269,14 @@ async def review_queue(session: AsyncSession, source: Any, *, kind: str = "all",
|
||||
if table:
|
||||
wanted = catalog.resolve_table_name(cache, table) or table
|
||||
entries = await catalog.read_catalog(session, source.id)
|
||||
usage = await catalog.table_usage(session, source)
|
||||
top = set(top_tables(tables, usage))
|
||||
usage = await table_usage_counts(session, source)
|
||||
top = set(top_tables(tables, usage.queries))
|
||||
items: list[dict[str, Any]] = []
|
||||
for name, entry in entries.items():
|
||||
meta = tables.get(name)
|
||||
if meta is None or (wanted is not None and name != wanted):
|
||||
continue
|
||||
used = usage.get(name, 0)
|
||||
used = usage.queries.get(name, 0)
|
||||
for key, item in live_slots(entry.notes, meta):
|
||||
if item.get("status") != "proposed":
|
||||
continue
|
||||
@@ -244,7 +288,7 @@ async def review_queue(session: AsyncSession, source: Any, *, kind: str = "all",
|
||||
"table_name": name,
|
||||
"path": slot_path(key),
|
||||
"field": field_name(key),
|
||||
"field_label": field_label(key),
|
||||
"field_label": item_label(key, item),
|
||||
# 列级的项是哪一列(表级的项、关联关系为 null)
|
||||
"column": key[1] if key[0] == "c" else None,
|
||||
"value": value,
|
||||
@@ -253,7 +297,7 @@ async def review_queue(session: AsyncSession, source: Any, *, kind: str = "all",
|
||||
"status": item.get("status"),
|
||||
"note": item.get("note") if isinstance(item.get("note"), str) and item.get("note") else None,
|
||||
"priority": round(priority(used, key, item.get("source")), 4),
|
||||
"reasons": _reasons(key, item, used, name in top),
|
||||
"reasons": _reasons(key, item, used, usage.runs.get(name, 0), name in top),
|
||||
"table_usage": used,
|
||||
"version": entry.version,
|
||||
})
|
||||
@@ -367,7 +411,8 @@ async def table_runs(session: AsyncSession, source: Any, table: str, *, limit: i
|
||||
|
||||
|
||||
async def review_progress(session: AsyncSession, source: Any) -> dict[str, Any]:
|
||||
"""审阅进度:常用 TOP_N 张表(只算被运行用到过的)、全部、影响数字的项。项按状态计数;verified(有确证)另给一项。"""
|
||||
"""审阅进度:常用 TOP_N 张表(top_tables)、全部、影响数字的项,项按状态计数。全部的 items_total 里还有有确证的项
|
||||
(不单列)。"""
|
||||
tables = (getattr(source, "schema_cache", None) or {}).get("tables") or {}
|
||||
entries = await catalog.read_catalog(session, source.id)
|
||||
usage = await catalog.table_usage(session, source)
|
||||
@@ -388,17 +433,15 @@ async def review_progress(session: AsyncSession, source: Any) -> dict[str, Any]:
|
||||
facts[status] += 1
|
||||
return {
|
||||
"top_n": TOP_N,
|
||||
"top": {"tables": len(top), "items_total": sum(head.values()), "items_confirmed": head["confirmed"],
|
||||
"items_proposed": head["proposed"], "items_verified": head["verified"]},
|
||||
"top": {"tables": len(top), "items_total": sum(head.values()), "items_confirmed": head["confirmed"]},
|
||||
"all": {"items_total": sum(every.values()), "items_confirmed": every["confirmed"],
|
||||
"items_proposed": every["proposed"], "items_rejected": every["rejected"],
|
||||
"items_verified": every["verified"]},
|
||||
"items_proposed": every["proposed"], "items_rejected": every["rejected"]},
|
||||
"facts": {"proposed": facts["proposed"], "confirmed": facts["confirmed"]},
|
||||
}
|
||||
|
||||
|
||||
__all__ = ["BATCH_MAX", "BatchItem", "CODE_CONFLICT", "CODE_INVALID", "CODE_PATH", "CODE_TABLE_MISSING",
|
||||
"FIELD_GROUP", "KINDS", "QUEUE_LIMIT_MAX", "RISK_WEIGHT", "RUNS_LIMIT_MAX", "SOURCE_LABEL", "SOURCE_WEIGHT",
|
||||
"TOP_N", "display", "field_label", "field_name", "field_of", "item_value", "live_slots", "priority",
|
||||
"relation_text", "review_batch", "review_progress", "review_queue", "slot_path", "table_runs", "top_tables",
|
||||
"usage_weight"]
|
||||
"TOP_N", "Usage", "display", "field_name", "field_of", "field_text", "item_label", "item_value",
|
||||
"live_slots", "priority", "relation_text", "review_batch", "review_progress", "review_queue", "slot_path",
|
||||
"table_runs", "table_usage_counts", "top_tables", "usage_text", "usage_weight"]
|
||||
|
||||
@@ -39,7 +39,7 @@ from typing import Any
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.data import catalog, catalog_jobs
|
||||
from app.data.catalog_review import field_label, slot_path
|
||||
from app.data.catalog_review import item_label, slot_path
|
||||
from app.data.engine import masked_columns
|
||||
from app.data.names import name_key
|
||||
|
||||
@@ -358,7 +358,7 @@ def plan_table(table_name: str, target: str, file_notes: Mapping[str, Any], targ
|
||||
plan.keep += 1
|
||||
else:
|
||||
plan.conflicts.append({
|
||||
"path": slot_path(mapped), "label": _label(mapped, item),
|
||||
"path": slot_path(mapped), "label": item_label(mapped, item),
|
||||
"file_value": _shown_value(mapped, item), "target_value": _shown_value(mapped, have),
|
||||
"file_status": status, "target_status": have.get("status"),
|
||||
"why": _conflict_why(str(status), str(have.get("status"))),
|
||||
@@ -371,14 +371,6 @@ def plan_table(table_name: str, target: str, file_notes: Mapping[str, Any], targ
|
||||
return plan
|
||||
|
||||
|
||||
def _label(key: catalog.Slot, item: Mapping[str, Any]) -> str:
|
||||
if key[0] == "t":
|
||||
return f"表的{field_label(key)}"
|
||||
if key[0] == "c":
|
||||
return f"列 {key[1]} 的{field_label(key)}"
|
||||
return f"指向 {item.get('to_table')} 的关联关系"
|
||||
|
||||
|
||||
def _shown_value(key: catalog.Slot, item: Mapping[str, Any]) -> Any:
|
||||
if key[0] == "r":
|
||||
return {k: item.get(k) for k in ("columns", "to_table", "to_columns", "cardinality", "coverage")}
|
||||
|
||||
@@ -335,7 +335,7 @@ async def test_background_has_structure_catalog_and_hides_masked_values(client,
|
||||
assert rule in system
|
||||
# 表结构、目录(来源和状态)、使用次数、起草要求、数据源说明
|
||||
for fact in ("order_no", "status", "amount", "member_id → members.id", "主键", "每笔订单一行", "人工填写", "已确认",
|
||||
"模型起草", "已驳回", "被运行用过 0 次", "金额一律按元", "虚构的网店库", "数据剖析:全表 30 行"):
|
||||
"模型起草", "已驳回", "还没有被运行用到过", "金额一律按元", "虚构的网店库", "数据剖析:全表 30 行"):
|
||||
assert fact in system, fact
|
||||
# 别的表指过来的关联关系
|
||||
assert "members.id → orders.member_id" in system
|
||||
@@ -351,6 +351,26 @@ async def test_background_has_structure_catalog_and_hides_masked_values(client,
|
||||
assert "没开剖析" not in system
|
||||
|
||||
|
||||
async def test_background_says_how_many_runs_used_the_table(client, tmp_path, monkeypatch):
|
||||
"""使用次数的口径同审阅队列:被几次运行用到(同「用过这张表的运行」的 total),查询几次(同目录清单的「使用 N 次」)。"""
|
||||
from app.db.models import Artifact, Run
|
||||
|
||||
sid = await _source(client, tmp_path)
|
||||
async with SessionLocal() as session:
|
||||
name = (await session.get(DataSource, sid)).name
|
||||
for i, n in enumerate((2, 1)):
|
||||
rid = uuid.uuid4().hex
|
||||
session.add(Run(id=rid, graph={}, workflow_name=f"周报{i}", status="succeeded"))
|
||||
for _ in range(n):
|
||||
session.add(Artifact(id=uuid.uuid4().hex, run_id=rid, kind=catalog.QUERY_SNAPSHOT_KIND,
|
||||
meta={"source_id": sid, "source": name, "tables": ["orders"]}))
|
||||
await session.commit()
|
||||
model = Scripted([Turn(text=["好。"])])
|
||||
use(monkeypatch, model)
|
||||
await _talk(client, sid, _ask("看看"))
|
||||
assert "这张表被 2 次运行用到(查询 3 次)。" in model.seen[0][0].content
|
||||
|
||||
|
||||
async def test_long_catalogs_are_cut_with_a_note(client, tmp_path, monkeypatch):
|
||||
sid = await _source(client, tmp_path)
|
||||
monkeypatch.setattr(catalog_assistant, "CATALOG_ITEMS_SHOWN", 2)
|
||||
|
||||
@@ -2,11 +2,13 @@
|
||||
|
||||
- GET /catalog/review-queue:只列推断(proposed)的项,按「使用次数权重 × 风险权重 × 来源权重」从高到低,同分按表名、
|
||||
路径;facts(有效记录条件、去重规则、业务日期、关联关系、业务主键、度量类型)/ names(中文名、说明、含义、单位、码值、
|
||||
粒度、表类型)两组;按表筛选;分页;reasons 写为什么排在这里;表结构里已经没有的表、列不列
|
||||
粒度、表类型)两组;按表筛选;分页;field_label 是给人看的位置(「列 status 的码值」);reasons 写为什么排在这里
|
||||
(被几次运行用到,和 /usage 的 total 一致);table_usage 是使用次数(同目录清单);表结构里已经没有的表、列不列
|
||||
- POST /catalog/review/batch:逐表按 if_version 乐观锁,一张表冲突不影响别的表;同一张表的几条一次写入;路径不对、表不存在
|
||||
逐条报
|
||||
- GET /catalog/{table}/usage:用过这张表的运行,新的在前,取法同 catalog.table_usage
|
||||
- GET /catalog/review-progress:常用 20 张表、全部、影响数字的项各自的确认情况
|
||||
- GET /catalog/review-progress:常用 20 张表(使用次数前 20 张,同起草的默认范围:用到的不够 20 张时按表结构的顺序
|
||||
补齐)、全部、影响数字的项各自的确认情况
|
||||
|
||||
表名、列名、业务数字都是虚构的。
|
||||
"""
|
||||
@@ -188,11 +190,11 @@ async def test_queue_lists_only_proposed_items_by_priority(client):
|
||||
rel_id = _member_rel()["id"]
|
||||
assert (first["table_name"], first["path"]) in {("orders", "valid_filter"), ("orders", f"relations.{rel_id}")}
|
||||
vf = next(i for i in items if i["path"] == "valid_filter")
|
||||
assert vf["field"] == "valid_filter" and vf["field_label"] == "有效记录条件"
|
||||
assert vf["field"] == "valid_filter" and vf["field_label"] == "表的有效记录条件"
|
||||
assert vf["value"] == "status <> 9" and vf["display"] == "status <> 9"
|
||||
assert vf["source"] == "llm" and vf["note"] == "剖析核对:排除 3.2% 的行(共 12,345 行)"
|
||||
assert vf["table_usage"] == 37 and vf["version"] == 1
|
||||
assert vf["reasons"] == ["常用表(使用 37 次)", "影响数字的项(有效记录条件)", "模型起草,未确认"]
|
||||
assert vf["reasons"] == ["常用表(被 37 次运行用到)", "影响数字的项(有效记录条件)", "模型起草,未确认"]
|
||||
# 常用表上的中文名排在没人用的表上的中文名前面;同一张表上影响数字的项排在中文名前面
|
||||
order = [(i["table_name"], i["path"]) for i in items]
|
||||
assert order.index(("orders", "label")) < order.index(("coupons", "label"))
|
||||
@@ -203,7 +205,7 @@ async def test_queue_values_are_displayed_in_ui_words(client):
|
||||
sid = await _source()
|
||||
items = {(i["table_name"], i["path"]): i for i in (await client.get(f"{_base(sid)}/review-queue")).json()["items"]}
|
||||
rel = items[("orders", f"relations.{_member_rel()['id']}")]
|
||||
assert rel["field"] == "relation" and rel["field_label"] == "关联关系"
|
||||
assert rel["field"] == "relation" and rel["field_label"] == "指向 members 的关联关系"
|
||||
assert rel["value"] == {"columns": ["member_id"], "to_table": "members", "to_columns": ["id"],
|
||||
"cardinality": "many_to_one", "coverage": 0.98}
|
||||
assert rel["display"] == "member_id → members.id(多对一,覆盖率 98%)"
|
||||
@@ -211,15 +213,39 @@ async def test_queue_values_are_displayed_in_ui_words(client):
|
||||
kind = items[("orders", "kind")]
|
||||
assert kind["display"] == "明细表" and kind["value"] == "fact"
|
||||
codes = items[("orders", "columns.status.codes")]
|
||||
assert codes["field"] == "codes" and codes["field_label"] == "码值" and codes["column"] == "status"
|
||||
assert codes["field"] == "codes" and codes["field_label"] == "列 status 的码值" and codes["column"] == "status"
|
||||
assert codes["display"] == "1=已支付、9=含义待填写"
|
||||
assert "1 个码值的含义待填写" in codes["reasons"] and "数据剖析,未确认" in codes["reasons"]
|
||||
assert items[("orders", "columns.amount.measure")]["display"] == "可累加"
|
||||
assert items[("members", "keys")]["display"] == "id"
|
||||
assert items[("members", "keys")]["display"] == "id" and items[("members", "keys")]["field_label"] == "表的业务主键"
|
||||
assert items[("orders", "columns.amount.measure")]["field_label"] == "列 amount 的度量类型"
|
||||
assert items[("members", "label")]["reasons"] == ["名称和说明(中文名)", "模型起草,未确认"]
|
||||
assert items[("coupons", "label")]["reasons"] == ["名称和说明(中文名)", "数据库注释,未确认"]
|
||||
|
||||
|
||||
async def test_queue_reasons_count_runs_like_the_usage_list(client):
|
||||
"""同一次运行里查了几次这张表:原因写「被几次运行用到」(和 /usage 的 total 一样),table_usage 是使用次数(和目录清单的
|
||||
「使用 N 次」一样),两个数不一样时原因里两个都写;优先级按使用次数(同清单的排序)。"""
|
||||
sid = await _source()
|
||||
async with SessionLocal() as session:
|
||||
name = (await session.get(DataSource, sid)).name
|
||||
runs = await _use(sid, name, {"orders": 2, "members": 1})
|
||||
async with SessionLocal() as session:
|
||||
for _ in range(3):
|
||||
session.add(Artifact(id=uuid.uuid4().hex, run_id=runs[0], kind=catalog.QUERY_SNAPSHOT_KIND,
|
||||
meta={"source_id": sid, "source": name, "tables": ["orders"]}))
|
||||
await session.commit()
|
||||
usage = await catalog.table_usage(session, await session.get(DataSource, sid))
|
||||
assert usage["orders"] == 5
|
||||
items = {(i["table_name"], i["path"]): i for i in (await client.get(f"{_base(sid)}/review-queue")).json()["items"]}
|
||||
vf = items[("orders", "valid_filter")]
|
||||
assert vf["table_usage"] == 5
|
||||
assert vf["reasons"][0] == "常用表(被 2 次运行用到,使用 5 次)"
|
||||
assert items[("members", "label")]["reasons"][0] == "常用表(被 1 次运行用到)"
|
||||
assert vf["priority"] == round(catalog_review.priority(5, ("t", "valid_filter"), "llm"), 4)
|
||||
assert (await client.get(f"{_base(sid)}/orders/usage")).json()["total"] == 2
|
||||
|
||||
|
||||
async def test_queue_filters_by_kind_and_table_and_pages(client):
|
||||
sid = await _source(usage={"orders": 3})
|
||||
facts = (await client.get(f"{_base(sid)}/review-queue", params={"kind": "facts"})).json()
|
||||
@@ -338,20 +364,34 @@ async def test_table_usage_lists_runs_newest_first(client):
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_review_progress_counts_top_tables_all_and_facts(client):
|
||||
async def test_review_progress_counts_top_tables_all_and_facts(client, monkeypatch):
|
||||
sid = await _source(usage={"orders": 5})
|
||||
r = await client.get(f"{_base(sid)}/review-progress")
|
||||
assert r.status_code == 200, r.text
|
||||
body = r.json()
|
||||
assert body["top_n"] == 20
|
||||
# 常用表只算被运行用到过的表:这里只有 orders(9 项:表结构里没有的列不算)
|
||||
assert body["top"] == {"tables": 1, "items_total": 9, "items_confirmed": 1, "items_proposed": 7,
|
||||
"items_verified": 0}
|
||||
assert body["all"] == {"items_total": 13, "items_confirmed": 1, "items_proposed": 11, "items_rejected": 1,
|
||||
"items_verified": 0}
|
||||
# 表结构里只有 3 张表:都算常用表(使用次数前 20 张);表结构里没有的表、列不算
|
||||
assert body["top"] == {"tables": 3, "items_total": 13, "items_confirmed": 1}
|
||||
assert body["all"] == {"items_total": 13, "items_confirmed": 1, "items_proposed": 11, "items_rejected": 1}
|
||||
# 影响数字的项:有效记录条件、关联关系、度量类型、业务主键
|
||||
assert body["facts"] == {"proposed": 4, "confirmed": 0}
|
||||
await client.post(f"{_base(sid)}/review/batch", json={"items": [
|
||||
{"table_name": "orders", "path": "valid_filter", "action": "confirm", "if_version": 1}]})
|
||||
after = (await client.get(f"{_base(sid)}/review-progress")).json()
|
||||
assert after["top"]["items_confirmed"] == 2 and after["facts"] == {"proposed": 3, "confirmed": 1}
|
||||
# 只取前 2 张:用得多的在前,没人用的按表结构的顺序补(同起草的默认范围)
|
||||
monkeypatch.setattr(catalog_review, "TOP_N", 2)
|
||||
used = await _source(usage={"coupons": 1})
|
||||
top = (await client.get(f"{_base(used)}/review-progress")).json()["top"]
|
||||
assert top == {"tables": 2, "items_total": 1 + 9, "items_confirmed": 1}
|
||||
|
||||
|
||||
def test_top_tables_are_the_draft_default_scope():
|
||||
"""常用表和起草对话框「使用次数最多的 20 张表」是同一批:使用次数从多到少,一样多(包括都没用过)的按表结构里的顺序。"""
|
||||
tables = {f"t{i:02d}": {} for i in range(30)}
|
||||
usage = {"t29": 4, "t05": 4, "t17": 9}
|
||||
top = catalog_review.top_tables(tables, usage)
|
||||
assert catalog_review.TOP_N == catalog.DRAFT_DEFAULT_TABLES == 20
|
||||
assert top[:3] == ["t17", "t05", "t29"] and len(top) == 20
|
||||
assert top[3:] == [n for n in tables if n not in usage][:17]
|
||||
assert catalog_review.top_tables({"a": {}, "b": {}}, {}) == ["a", "b"]
|
||||
|
||||
Reference in New Issue
Block a user