mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-22 16:22:00 +00:00
46a2988dc1
- 重构问数据执行架构为「快速直查(fast)」与「深度分析(deep)」双模式,内置语义预命中(Grounding)注入指标卡片、标准实体编码与表目录血缘 - 快速直查免工作流生成单节点执行,新增零额外模型调用的数字证据归一与封存(direct_evidence),打通可点击数字溯源与 /verify 校验 - 深度分析与助手编排接入跨源本体穿透(自动升级重跑与一键沿本体排查关联数据源),优化数据源范围智能收敛与往返缓存 - 补齐端到端回归测试矩阵与中英文全量文档说明
843 lines
38 KiB
Python
843 lines
38 KiB
Python
"""多表 / 参数化 SQL 模板语义指标(calc_mode="sql_template")与后台口径文档一键拆解工作流(SemanticDecomposeJob)端到端测试。"""
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import json
|
||
import sqlite3
|
||
import uuid
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
import pytest
|
||
from httpx import ASGITransport, AsyncClient
|
||
from langchain_core.messages import AIMessage
|
||
from sqlalchemy import delete, select
|
||
|
||
from app.chat import job as chat_job
|
||
from app.chat.grounding import ground_question
|
||
from app.data import catalog, catalog_assistant, semantic_decompose
|
||
from app.data.engine import CATALOG_PROFILE_OPTION, MASK_COLUMNS_OPTION
|
||
from app.data.semantic_metrics import get_metric, metric_to_dict
|
||
from app.db.base import SessionLocal
|
||
from app.db.models import ConversationTurn, Document, Run
|
||
from app.db.models_chat import ChatTurnJob
|
||
from app.db.models_metrics import SemanticDecomposeJob, SemanticMetric
|
||
from app.db.models_ontology import CanonicalEntity
|
||
from app.engine.evidence import iter_segments
|
||
from app.engine.runner import run_manager
|
||
from app.main import app
|
||
from app.providers import mock_model
|
||
from app.tools.metrics import METRIC_QUERY
|
||
from tests.fixtures.sources import drop_source, sqlite_source
|
||
|
||
|
||
@pytest.fixture(autouse=True)
|
||
async def _lifecycle():
|
||
await run_manager.setup()
|
||
yield
|
||
await semantic_decompose.shutdown()
|
||
await chat_job.shutdown()
|
||
await run_manager.shutdown()
|
||
async with SessionLocal() as session:
|
||
await session.execute(delete(SemanticDecomposeJob))
|
||
await session.execute(delete(SemanticMetric))
|
||
await session.execute(delete(CanonicalEntity).where(CanonicalEntity.key.like("mes_test_%")))
|
||
await session.execute(delete(Document).where(Document.title.like("%OEE%")))
|
||
await session.commit()
|
||
|
||
|
||
@pytest.fixture
|
||
async def client():
|
||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
|
||
yield c
|
||
|
||
|
||
async def _create_mes_fixture(tmp_path: Path) -> dict[str, str]:
|
||
"""构造还原工业 OEE / TEEP 跨表场景的 SQLite 测试库(盲列 ATTRIBUTE01..34 + 不合格记录表 + 模具系数表 + 产线节拍表 + 遮罩列)。"""
|
||
db_path = tmp_path / f"mes_oee_{uuid.uuid4().hex[:6]}.db"
|
||
conn = sqlite3.connect(db_path)
|
||
try:
|
||
conn.executescript(
|
||
"""
|
||
CREATE TABLE mes_shift_daily (
|
||
id INTEGER PRIMARY KEY,
|
||
stat_date TEXT NOT NULL,
|
||
line_code TEXT NOT NULL,
|
||
prod_code TEXT NOT NULL,
|
||
ATTRIBUTE01 REAL NOT NULL, -- 计划开班时长(秒)
|
||
ATTRIBUTE02 REAL NOT NULL, -- 计划内停机时长(秒)
|
||
ATTRIBUTE03 REAL NOT NULL, -- 故障与换模损失时长(秒)
|
||
ATTRIBUTE10 REAL NOT NULL, -- 合格入库数(件)
|
||
ATTRIBUTE34 REAL NOT NULL, -- 良品率(底表直接存百分数,如 98.5)
|
||
secret_phone TEXT,
|
||
is_valid INTEGER NOT NULL DEFAULT 1
|
||
);
|
||
|
||
CREATE TABLE mes_bhgsj (
|
||
id INTEGER PRIMARY KEY,
|
||
stat_date TEXT NOT NULL,
|
||
line_code TEXT NOT NULL,
|
||
scrap_qty REAL NOT NULL
|
||
);
|
||
|
||
CREATE TABLE mes_jzhmjxs (
|
||
id INTEGER PRIMARY KEY,
|
||
prod_code TEXT NOT NULL,
|
||
mjxs REAL NOT NULL
|
||
);
|
||
|
||
CREATE TABLE iot_dn_productionline (
|
||
line_code TEXT PRIMARY KEY,
|
||
line_name TEXT NOT NULL,
|
||
beat REAL NOT NULL -- 标准节拍(秒/件)
|
||
);
|
||
|
||
INSERT INTO iot_dn_productionline (line_code, line_name, beat) VALUES
|
||
('L1', '一号冲压产线', 10.0),
|
||
('L2', '二号焊接产线', 20.0);
|
||
|
||
INSERT INTO mes_jzhmjxs (prod_code, mjxs) VALUES
|
||
('P100', 1.2);
|
||
-- P200 故意不在模具系数表中,验证 COALESCE(mj.mjxs, 0.5) 兜底系数
|
||
|
||
INSERT INTO mes_shift_daily (
|
||
stat_date, line_code, prod_code,
|
||
ATTRIBUTE01, ATTRIBUTE02, ATTRIBUTE03, ATTRIBUTE10, ATTRIBUTE34,
|
||
secret_phone, is_valid
|
||
) VALUES
|
||
('2026-03-01', 'L1', 'P100', 28800, 3600, 1200, 1800, 96.0, '13900001111', 1),
|
||
('2026-03-01', 'L2', 'P200', 28800, 1800, 2000, 2000, 95.0, '13900002222', 1),
|
||
('2026-03-02', 'L1', 'P100', 28800, 3600, 600, 2000, 98.0, '13900003333', 1),
|
||
('2026-03-02', 'L1', 'P100', 99999, 0, 0, 9999, 100.0,'13900004444', 0);
|
||
|
||
INSERT INTO mes_bhgsj (stat_date, line_code, scrap_qty) VALUES
|
||
('2026-03-01', 'L1', 200),
|
||
('2026-03-01', 'L2', 100),
|
||
('2026-03-02', 'L1', 50);
|
||
"""
|
||
)
|
||
conn.commit()
|
||
finally:
|
||
conn.close()
|
||
|
||
src_name = f"mes_prod_{uuid.uuid4().hex[:6]}"
|
||
sid = await sqlite_source(
|
||
src_name,
|
||
db_path,
|
||
options={
|
||
CATALOG_PROFILE_OPTION: {"enabled": True},
|
||
MASK_COLUMNS_OPTION: "secret_phone",
|
||
},
|
||
schema=True,
|
||
)
|
||
ent_key = f"mes_test_line_{uuid.uuid4().hex[:4]}"
|
||
async with SessionLocal() as session:
|
||
session.add(
|
||
CanonicalEntity(
|
||
key=ent_key,
|
||
label="制造产线",
|
||
aliases=["产线", "生产线"],
|
||
members=[
|
||
{
|
||
"id": "m_l1",
|
||
"value": "LINE_01",
|
||
"label": "一号冲压产线",
|
||
"aliases": ["一号产线", "1线"],
|
||
"source_codes": {src_name: "L1"},
|
||
},
|
||
{
|
||
"id": "m_l2",
|
||
"value": "LINE_02",
|
||
"label": "二号焊接产线",
|
||
"aliases": ["二号产线", "2线"],
|
||
"source_codes": {src_name: "L2"},
|
||
},
|
||
],
|
||
bindings=[
|
||
{
|
||
"id": "b_line",
|
||
"source_id": sid,
|
||
"source_name": src_name,
|
||
"table_name": "mes_shift_daily",
|
||
"column_name": "line_code",
|
||
"status": "confirmed",
|
||
"source": "human",
|
||
}
|
||
],
|
||
version=1,
|
||
)
|
||
)
|
||
await session.commit()
|
||
|
||
return {"source_id": sid, "source_name": src_name, "entity_key": ent_key}
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_multi_table_sql_template_metric_crud_preview_and_guardrails(client: AsyncClient, tmp_path: Path):
|
||
"""验证多表 JOIN + CTE + COALESCE + {{period_days}} 参数化模板指标的创建、安全拦截、实体转码试算与 restricted 脱敏。"""
|
||
fx = await _create_mes_fixture(tmp_path)
|
||
sid = fx["source_id"]
|
||
ent_key = fx["entity_key"]
|
||
try:
|
||
# 1) 安全红线拦截:写操作、未知占位符、不存在物理表、敏感遮罩列均必须 422 拒绝
|
||
for bad_tpl in (
|
||
"DELETE FROM mes_shift_daily",
|
||
"SELECT {{evil_slot}} SUM(s.ATTRIBUTE10) FROM mes_shift_daily s",
|
||
"SELECT SUM(s.ATTRIBUTE10) FROM mes_nonexistent_tbl s",
|
||
"SELECT COUNT(s.secret_phone) FROM mes_shift_daily s",
|
||
):
|
||
r_bad = await client.post(
|
||
"/api/semantic-metrics",
|
||
json={
|
||
"key": "bad_tpl_metric",
|
||
"name": "非法模板测试",
|
||
"source_id": sid,
|
||
"calc_mode": "sql_template",
|
||
"sql_template": bad_tpl,
|
||
},
|
||
)
|
||
assert r_bad.status_code == 422, f"未拦截非法模板: {bad_tpl} -> {r_bad.text}"
|
||
|
||
# 2) 创建多表 OEE 精算指标(CTE + 多表 LEFT JOIN + COALESCE 0.5 模具兜底系数 + 参数化槽位)
|
||
oee_tpl = """
|
||
WITH mj AS (
|
||
SELECT prod_code, AVG(mjxs) AS mjxs
|
||
FROM mes_jzhmjxs
|
||
GROUP BY prod_code
|
||
),
|
||
sc AS (
|
||
SELECT stat_date, line_code, SUM(scrap_qty) AS bad_qty
|
||
FROM mes_bhgsj
|
||
GROUP BY stat_date, line_code
|
||
)
|
||
SELECT
|
||
{{select_dims}}
|
||
ROUND(
|
||
SUM((s.ATTRIBUTE10 + COALESCE(sc.bad_qty, 0)) * pl.beat * COALESCE(mj.mjxs, 0.5))
|
||
* 100.0
|
||
/ NULLIF(SUM(s.ATTRIBUTE01 - s.ATTRIBUTE02), 0)
|
||
* (SUM(s.ATTRIBUTE10) * 1.0 / NULLIF(SUM(s.ATTRIBUTE10 + COALESCE(sc.bad_qty, 0)), 0)),
|
||
2
|
||
) AS mes_oee_multi
|
||
FROM mes_shift_daily s
|
||
JOIN iot_dn_productionline pl ON pl.line_code = s.line_code
|
||
LEFT JOIN mj ON mj.prod_code = s.prod_code
|
||
LEFT JOIN sc ON sc.stat_date = s.stat_date AND sc.line_code = s.line_code
|
||
WHERE 1=1 {{and_where}}
|
||
{{group_by}}
|
||
{{order_by}}
|
||
""".strip()
|
||
|
||
r_create = await client.post(
|
||
"/api/semantic-metrics",
|
||
json={
|
||
"key": "mes_oee_multi",
|
||
"name": "设备综合效率 OEE(跨表精算)",
|
||
"aliases": ["OEE", "综合效率"],
|
||
"unit": "%",
|
||
"source_id": sid,
|
||
"calc_mode": "sql_template",
|
||
"sql_template": oee_tpl,
|
||
"date_column": "s.stat_date",
|
||
"filter_sql": "s.is_valid = 1",
|
||
"dimensions": [
|
||
{"key": "line_code", "label": "产线", "column": "s.line_code", "entity_key": ent_key},
|
||
],
|
||
},
|
||
)
|
||
assert r_create.status_code == 201, r_create.text
|
||
created_metric = r_create.json()["metric"]
|
||
assert created_metric["calc_mode"] == "sql_template"
|
||
assert created_metric["table_name"] == "mes_shift_daily" # 自动从模板首张物理表识别
|
||
assert "[多表模板]" in created_metric["sql_expression"]
|
||
|
||
# restricted=True 下必须隐藏底层 sql_template
|
||
async with SessionLocal() as session:
|
||
row = await get_metric(session, created_metric["id"])
|
||
restricted_dict = metric_to_dict(row, restricted=True)
|
||
assert "sql_template" not in restricted_dict
|
||
assert "table_name" not in restricted_dict
|
||
|
||
# 3) 在线试算:按产线分组 + 实体中文别名过滤(「一号产线」-> L1,且过滤掉 is_valid=0 脏数据)
|
||
r_prev_l1 = await client.post(
|
||
"/api/semantic-metrics/preview",
|
||
json={
|
||
"metric_id": created_metric["id"],
|
||
"date_from": "2026-03-01",
|
||
"date_to": "2026-03-01",
|
||
"filters": {"line_code": "一号产线"},
|
||
},
|
||
)
|
||
assert r_prev_l1.status_code == 200, r_prev_l1.text
|
||
prev_l1 = r_prev_l1.json()
|
||
assert prev_l1["entity_alignments"][0]["resolved_code"] == "L1"
|
||
# L1 2026-03-01: (1800+200)*10*1.2 / (28800-3600) * (1800/2000) * 100 = 24000 / 25200 * 0.9 * 100 = 85.71
|
||
assert prev_l1["rows"] == [[85.71]]
|
||
|
||
# L2 2026-03-01 走 COALESCE(mj.mjxs, 0.5) 兜底:(2000+100)*20*0.5 / (28800-1800) * (2000/2100) * 100 = 20000 / 27000 * 100 = 74.07
|
||
r_prev_group = await client.post(
|
||
"/api/semantic-metrics/preview",
|
||
json={
|
||
"metric_id": created_metric["id"],
|
||
"date_from": "2026-03-01",
|
||
"date_to": "2026-03-01",
|
||
"group_by": ["line_code"],
|
||
},
|
||
)
|
||
assert r_prev_group.status_code == 200, r_prev_group.text
|
||
grouped_rows = {r[0]: r[1] for r in r_prev_group.json()["rows"]}
|
||
assert grouped_rows == {"L1": 85.71, "L2": 74.07}
|
||
|
||
# 4) 验证 {{period_days}} 动态自然周期天数模板(TEEP 日历秒数 = {{period_days}} * 86400 * 产线条数)
|
||
teep_tpl = """
|
||
SELECT
|
||
{{select_dims}}
|
||
ROUND(
|
||
SUM(s.ATTRIBUTE01 - s.ATTRIBUTE02 - s.ATTRIBUTE03) * 100.0
|
||
/ NULLIF(({{period_days}}) * 86400.0 * COUNT(DISTINCT s.line_code), 0),
|
||
2
|
||
) AS mes_teep
|
||
FROM mes_shift_daily s
|
||
WHERE 1=1 {{and_where}}
|
||
{{group_by}}
|
||
{{order_by}}
|
||
""".strip()
|
||
|
||
r_teep = await client.post(
|
||
"/api/semantic-metrics",
|
||
json={
|
||
"key": "mes_teep",
|
||
"name": "全员有效生产率 TEEP",
|
||
"unit": "%",
|
||
"source_id": sid,
|
||
"calc_mode": "sql_template",
|
||
"sql_template": teep_tpl,
|
||
"date_column": "s.stat_date",
|
||
"filter_sql": "s.is_valid = 1",
|
||
"dimensions": [{"key": "line_code", "label": "产线", "column": "s.line_code"}],
|
||
},
|
||
)
|
||
assert r_teep.status_code == 201, r_teep.text
|
||
teep_id = r_teep.json()["metric"]["id"]
|
||
|
||
# 7 天区间(2026-03-01 ~ 2026-03-07):{{period_days}} 必须确定性编译为 7
|
||
r_teep_7d = await client.post(
|
||
"/api/semantic-metrics/preview",
|
||
json={
|
||
"metric_id": teep_id,
|
||
"date_from": "2026-03-01",
|
||
"date_to": "2026-03-07",
|
||
},
|
||
)
|
||
assert r_teep_7d.status_code == 200, r_teep_7d.text
|
||
assert "(7) * 86400.0" in r_teep_7d.json()["sql"]
|
||
|
||
# 按日趋势(group_by_date=True):单日每行 {{period_days}} 必须确定性编译为 1
|
||
r_teep_daily = await client.post(
|
||
"/api/semantic-metrics/preview",
|
||
json={
|
||
"metric_id": teep_id,
|
||
"date_from": "2026-03-01",
|
||
"date_to": "2026-03-02",
|
||
"group_by_date": True,
|
||
},
|
||
)
|
||
assert r_teep_daily.status_code == 200, r_teep_daily.text
|
||
assert "(1) * 86400.0" in r_teep_daily.json()["sql"]
|
||
assert len(r_teep_daily.json()["rows"]) == 2
|
||
finally:
|
||
await drop_source(sid)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fast_direct_ask_with_sql_template_metric_produces_clickable_evidence(
|
||
client: AsyncClient,
|
||
tmp_path: Path,
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
):
|
||
"""验证快速直查(depth='fast')命中多表 sql_template 指标后,经 metric_query 取数并生成可点击数字证据。"""
|
||
fx = await _create_mes_fixture(tmp_path)
|
||
sid = fx["source_id"]
|
||
ent_key = fx["entity_key"]
|
||
try:
|
||
oee_tpl = (
|
||
"WITH mj AS (SELECT prod_code, AVG(mjxs) AS mjxs FROM mes_jzhmjxs GROUP BY prod_code), "
|
||
"sc AS (SELECT stat_date, line_code, SUM(scrap_qty) AS bad_qty FROM mes_bhgsj GROUP BY stat_date, line_code) "
|
||
"SELECT {{select_dims}} ROUND(SUM((s.ATTRIBUTE10 + COALESCE(sc.bad_qty, 0)) * pl.beat * COALESCE(mj.mjxs, 0.5)) * 100.0 "
|
||
"/ NULLIF(SUM(s.ATTRIBUTE01 - s.ATTRIBUTE02), 0) "
|
||
"* (SUM(s.ATTRIBUTE10) * 1.0 / NULLIF(SUM(s.ATTRIBUTE10 + COALESCE(sc.bad_qty, 0)), 0)), 2) AS mes_oee_fast "
|
||
"FROM mes_shift_daily s "
|
||
"JOIN iot_dn_productionline pl ON pl.line_code = s.line_code "
|
||
"LEFT JOIN mj ON mj.prod_code = s.prod_code "
|
||
"LEFT JOIN sc ON sc.stat_date = s.stat_date AND sc.line_code = s.line_code "
|
||
"WHERE 1=1 {{and_where}} {{group_by}} {{order_by}}"
|
||
)
|
||
r_create = await client.post(
|
||
"/api/semantic-metrics",
|
||
json={
|
||
"key": "mes_oee_fast",
|
||
"name": "产线跨表OEE",
|
||
"aliases": ["跨表OEE"],
|
||
"unit": "%",
|
||
"source_id": sid,
|
||
"calc_mode": "sql_template",
|
||
"sql_template": oee_tpl,
|
||
"date_column": "s.stat_date",
|
||
"filter_sql": "s.is_valid = 1",
|
||
"dimensions": [{"key": "line_code", "label": "产线", "column": "s.line_code", "entity_key": ent_key}],
|
||
},
|
||
)
|
||
assert r_create.status_code == 201
|
||
|
||
# 验证语义预命中包含 calc_mode="sql_template" 提示
|
||
async with SessionLocal() as session:
|
||
bundle = await ground_question(session, question="查询 2026-03-01 一号产线的跨表OEE")
|
||
assert any(m["key"] == "mes_oee_fast" and m["calc_mode"] == "sql_template" for m in bundle.matched_metrics)
|
||
assert "多表参数化 SQL 模板" in bundle.system_context
|
||
|
||
def _decide_oee(self: Any, messages: list[Any]) -> AIMessage:
|
||
has_tool_res = any(getattr(m, "type", "") == "tool" for m in messages)
|
||
if not has_tool_res:
|
||
return AIMessage(
|
||
content="",
|
||
tool_calls=[
|
||
{
|
||
"id": "call_oee_tpl",
|
||
"name": METRIC_QUERY,
|
||
"args": {
|
||
"metric_key": "mes_oee_fast",
|
||
"date_from": "2026-03-01",
|
||
"date_to": "2026-03-01",
|
||
"filters": {"line_code": "一号产线"},
|
||
},
|
||
}
|
||
],
|
||
)
|
||
return AIMessage(
|
||
content="2026-03-01 一号冲压产线的跨表精算 OEE 为 **[[v:Q1.r0.mes_oee_fast]]**%。"
|
||
)
|
||
|
||
monkeypatch.setattr(mock_model.MockChatModel, "_decide", _decide_oee)
|
||
|
||
conv_res = await client.post("/api/conversations", json={"kind": "chat", "title": "跨表OEE直查测试"})
|
||
conv_id = conv_res.json()["id"]
|
||
|
||
r_turn = await client.post(
|
||
"/api/chat/turns",
|
||
json={
|
||
"conversation_id": conv_id,
|
||
"question": "查询 2026-03-01 一号产线的跨表OEE",
|
||
"datasource_ids": [sid],
|
||
"depth": "fast",
|
||
},
|
||
)
|
||
assert r_turn.status_code == 202, r_turn.text
|
||
turn_id = r_turn.json()["turn_id"]
|
||
await chat_job.manager.wait(turn_id, timeout=30.0)
|
||
|
||
async with SessionLocal() as session:
|
||
job = await session.get(ChatTurnJob, turn_id)
|
||
turn_row = await session.get(ConversationTurn, turn_id)
|
||
run_row = await session.get(Run, job.run_id)
|
||
|
||
assert turn_row is not None and turn_row.status == "done"
|
||
assert "85.71" in (turn_row.answer or "")
|
||
assert "[[v:" not in (turn_row.answer or "")
|
||
|
||
doc_art_id = run_row.output["_evidence"]["doc_artifact"]
|
||
doc = (await client.get(f"/api/artifacts/{doc_art_id}")).json()["content"]
|
||
assert doc["stats"]["numbers_cited"] >= 1
|
||
segs = list(iter_segments(doc))
|
||
assert any(s.get("kind") == "number" and "85.71" in str(s.get("text") or "") for s in segs)
|
||
|
||
verify_res = await client.get(f"/api/runs/{job.run_id}/verify")
|
||
assert verify_res.status_code == 200 and verify_res.json()["ok"] is True
|
||
finally:
|
||
await drop_source(sid)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_background_document_decompose_workflow_and_one_click_apply(
|
||
client: AsyncClient,
|
||
tmp_path: Path,
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
):
|
||
"""验证业务口径文档一键拆解后台任务(阶段进度 + 字段符号链式注入 + 知识库归档 + 一键应用四层资产)。"""
|
||
fx = await _create_mes_fixture(tmp_path)
|
||
sid = fx["source_id"]
|
||
src_name = fx["source_name"]
|
||
|
||
captured_prompts: list[tuple[str, str]] = []
|
||
|
||
class _FakeDecomposeLLM:
|
||
async def ainvoke(self, msgs: list[Any]) -> AIMessage:
|
||
sys_txt = str(msgs[0].content)
|
||
usr_txt = str(msgs[1].content)
|
||
captured_prompts.append((sys_txt, usr_txt))
|
||
|
||
if "数据目录与字段字典拆解专家" in sys_txt:
|
||
payload = {
|
||
"tables": [
|
||
{
|
||
"table": "mes_shift_daily",
|
||
"changes": [
|
||
{"path": "label", "value": "产线班次日度生产宽表", "reason": "文档总表说明"},
|
||
{
|
||
"path": "business_date",
|
||
"value": {"column": "stat_date", "rule": "按班次归属自然日"},
|
||
"reason": "文档第 0 节",
|
||
},
|
||
{"path": "valid_filter", "value": "is_valid = 1", "reason": "仅统计有效班次"},
|
||
{"path": "columns.attribute01.label", "value": "计划开班时长", "reason": "文档符号 T_open"},
|
||
{"path": "columns.attribute01.unit", "value": "秒", "reason": "文档符号 T_open"},
|
||
{"path": "columns.attribute01.measure", "value": "flow", "reason": "时长可累加"},
|
||
{"path": "columns.ATTRIBUTE34.label", "value": "班次良品率", "reason": "文档符号 Q_rate"},
|
||
{"path": "columns.ATTRIBUTE34.unit", "value": "%", "reason": "百分数"},
|
||
{"path": "columns.ATTRIBUTE34.measure", "value": "stock", "reason": "比率不可直接跨日相加"},
|
||
{
|
||
"path": "columns.ATTRIBUTE34.note",
|
||
"value": "底表直接存百分数如 98.5,计算时需除以 100,切勿重复乘 100",
|
||
"reason": "文档存储陷阱提示",
|
||
},
|
||
# 故意混入遮罩列测试自动拦截
|
||
{"path": "columns.secret_phone.label", "value": "手机号", "reason": "测试遮罩过滤"},
|
||
],
|
||
}
|
||
],
|
||
"guardrail_notes": [
|
||
{
|
||
"title": "TEEP 物理红线",
|
||
"rule": "TEEP 超过 100% 必属配置或源数据错误;OEE 超过 200% 多为模具系数异常。",
|
||
"category": "redline",
|
||
}
|
||
],
|
||
}
|
||
return AIMessage(content=json.dumps(payload, ensure_ascii=False))
|
||
|
||
if "语义指标建模专家" in sys_txt:
|
||
payload = {
|
||
"metrics": [
|
||
{
|
||
"key": "mes_time_utilization",
|
||
"name": "时间稼动率(单表)",
|
||
"aliases": ["时间稼动率"],
|
||
"description": "实际开机时长除以负荷时长",
|
||
"unit": "%",
|
||
"calc_mode": "expression",
|
||
"table_name": "mes_shift_daily",
|
||
"sql_expression": "ROUND(SUM(ATTRIBUTE01 - ATTRIBUTE02 - ATTRIBUTE03) * 100.0 / NULLIF(SUM(ATTRIBUTE01 - ATTRIBUTE02), 0), 2)",
|
||
"sql_template": "",
|
||
"date_column": "stat_date",
|
||
"filter_sql": "is_valid = 1",
|
||
"dimensions": [{"key": "line_code", "label": "产线", "column": "line_code"}],
|
||
},
|
||
{
|
||
"key": "mes_teep_doc",
|
||
"name": "全员有效生产率 TEEP(文档拆解)",
|
||
"aliases": ["TEEP"],
|
||
"description": "联合产线节拍与动态周期天数计算 TEEP",
|
||
"unit": "%",
|
||
"calc_mode": "sql_template",
|
||
"table_name": "mes_shift_daily",
|
||
"sql_expression": "",
|
||
"sql_template": (
|
||
"SELECT {{select_dims}} "
|
||
"ROUND(SUM(s.ATTRIBUTE10 * pl.beat) * 100.0 / NULLIF(({{period_days}}) * 86400.0 * COUNT(DISTINCT s.line_code), 0), 2) AS mes_teep_doc "
|
||
"FROM mes_shift_daily s JOIN iot_dn_productionline pl ON pl.line_code = s.line_code "
|
||
"WHERE 1=1 {{and_where}} {{group_by}} {{order_by}}"
|
||
),
|
||
"date_column": "s.stat_date",
|
||
"filter_sql": "s.is_valid = 1",
|
||
"dimensions": [{"key": "line_code", "label": "产线", "column": "s.line_code"}],
|
||
},
|
||
{
|
||
# 故意混入不存在列的脏指标,验证自动过滤进 skipped
|
||
"key": "mes_hallucinated",
|
||
"name": "臆造列指标",
|
||
"calc_mode": "expression",
|
||
"table_name": "mes_shift_daily",
|
||
"sql_expression": "SUM(NON_EXISTENT_COL)",
|
||
},
|
||
]
|
||
}
|
||
return AIMessage(content=json.dumps(payload, ensure_ascii=False))
|
||
|
||
# 标准业务实体拆解专家
|
||
ent_payload = {
|
||
"operations": [
|
||
{
|
||
"op": "upsert_entity",
|
||
"entity_key": "mes_test_product_model",
|
||
"label": "产品型号",
|
||
"description": "车间生产产品型号标准实体",
|
||
"aliases": ["品号", "机种"],
|
||
"reason": "多表通过 prod_code 关联模具系数",
|
||
},
|
||
{
|
||
"op": "upsert_binding",
|
||
"entity_key": "mes_test_product_model",
|
||
"entity_label": "产品型号",
|
||
"source_name": src_name,
|
||
"table_name": "mes_shift_daily",
|
||
"column_name": "prod_code",
|
||
"note": "班次产品编码",
|
||
"reason": "关联模具系数表",
|
||
},
|
||
]
|
||
}
|
||
return AIMessage(content=json.dumps(ent_payload, ensure_ascii=False))
|
||
|
||
async def _fake_resolve_model(_session: Any):
|
||
return _FakeDecomposeLLM(), "mock/decompose-pro"
|
||
|
||
monkeypatch.setattr(catalog_assistant, "resolve_model", _fake_resolve_model)
|
||
|
||
spec_md = """# TEEP / OEE 及相关指标公式总表
|
||
## 0. 字段符号对照
|
||
- `ATTRIBUTE01`:计划开班时长(秒)
|
||
- `ATTRIBUTE34`:班次良品率(底表存百分数,如 98.5)
|
||
## 1. 跨表 TEEP 公式
|
||
自然日历时间 = 统计周期天数 × 86400 × 产线条数
|
||
## 2. 判异红线
|
||
TEEP > 100% 必属源数据错误。
|
||
"""
|
||
try:
|
||
# 1) 测试上传附件接口可用
|
||
r_att = await client.post(
|
||
"/api/semantic-metrics/attachments",
|
||
files={"file": ("oee_extra_notes.md", b"# Extra Note\nprod_code P100 = Model A\n", "text/markdown")},
|
||
)
|
||
assert r_att.status_code == 200, r_att.text
|
||
att_id = r_att.json()["attachment_id"]
|
||
|
||
# 2) 发起后台一键拆解任务(202 Accepted)
|
||
r_start = await client.post(
|
||
"/api/semantic-metrics/decompose-jobs",
|
||
json={
|
||
"source_id": sid,
|
||
"scopes": ["catalog", "metrics", "entities"],
|
||
"attachment_ids": [att_id],
|
||
"doc_title": "TEEP 与 OEE 指标公式总表",
|
||
"doc_text": spec_md,
|
||
"save_to_kb": True,
|
||
"note": "重点提取单表稼动率与跨表 TEEP",
|
||
},
|
||
)
|
||
assert r_start.status_code == 202, r_start.text
|
||
job_id = r_start.json()["job"]["id"]
|
||
|
||
# 3) 轮询直至后台任务完成(验证进度 100% 与 4 个阶段全部成功)
|
||
final_job: dict[str, Any] | None = None
|
||
for _ in range(50):
|
||
r_job = await client.get(f"/api/semantic-metrics/decompose-jobs/{job_id}")
|
||
assert r_job.status_code == 200
|
||
j = r_job.json()["job"]
|
||
if j["status"] in ("succeeded", "failed", "cancelled"):
|
||
final_job = j
|
||
break
|
||
await asyncio.sleep(0.04)
|
||
|
||
assert final_job is not None
|
||
assert final_job["status"] == "succeeded", final_job.get("error")
|
||
assert final_job["progress"] == 100
|
||
assert [s["status"] for s in final_job["steps"]] == ["succeeded", "succeeded", "succeeded", "succeeded"]
|
||
|
||
# 验证 active 横幅接口返回最新待审阅任务
|
||
r_active = await client.get("/api/semantic-metrics/decompose-jobs/active", params={"source_id": sid})
|
||
assert r_active.status_code == 200
|
||
assert r_active.json()["latest"]["id"] == job_id
|
||
|
||
# 核心断言:阶段 2 提取出的 ATTRIBUTE01(计划开班时长,大写对齐!)与 ATTRIBUTE34 存储说明已在内存里链式传给阶段 3!
|
||
metric_stage_prompt = next(u for s, u in captured_prompts if "语义指标建模专家" in s)
|
||
assert "ATTRIBUTE01 (REAL) — 计划开班时长,单位:秒,度量:flow" in metric_stage_prompt
|
||
assert "底表直接存百分数" in metric_stage_prompt
|
||
|
||
res = final_job["result"]
|
||
assert len(res["kb_documents"]) >= 1
|
||
assert len(res["guardrail_notes"]) == 1
|
||
assert len(res["metric_candidates"]) == 2
|
||
assert len(res["metric_skipped"]) == 1
|
||
# 确认 secret_phone 遮罩列被自动拦截,attribute01 大小写已自动对齐为 ATTRIBUTE01
|
||
cat_paths = [c["path"] for c in res["catalog_patches"][0]["changes"]]
|
||
assert "columns.ATTRIBUTE01.label" in cat_paths
|
||
assert all("secret_phone" not in p for p in cat_paths)
|
||
assert len(res["entity_changes"]) == 2
|
||
|
||
# 4) 一键应用审阅结果
|
||
r_apply = await client.post(
|
||
f"/api/semantic-metrics/decompose-jobs/{job_id}/apply",
|
||
json={"overwrite_existing_metrics": True},
|
||
)
|
||
assert r_apply.status_code == 200, r_apply.text
|
||
applied = r_apply.json()["applied"]
|
||
assert applied["catalog_applied_count"] >= 8
|
||
assert applied["metrics_created_count"] == 2
|
||
assert applied["entities_applied_count"] == 2
|
||
|
||
# 5) 核实三层资产均已真实落库且可立即执行在线试算
|
||
async with SessionLocal() as session:
|
||
cat_entry = await catalog.read_entry(session, sid, "mes_shift_daily")
|
||
assert cat_entry is not None
|
||
vis = catalog.visible_notes(cat_entry.notes)
|
||
assert vis["columns"]["ATTRIBUTE01"]["label"]["value"] == "计划开班时长"
|
||
assert vis["columns"]["ATTRIBUTE34"]["measure"]["value"] == "stock"
|
||
|
||
ent_row = (
|
||
await session.execute(select(CanonicalEntity).where(CanonicalEntity.key == "mes_test_product_model"))
|
||
).scalar_one_or_none()
|
||
assert ent_row is not None
|
||
assert ent_row.label == "产品型号"
|
||
|
||
r_try_teep = await client.post(
|
||
"/api/semantic-metrics/preview",
|
||
json={
|
||
"metric_id": "mes_teep_doc",
|
||
"date_from": "2026-03-01",
|
||
"date_to": "2026-03-02",
|
||
},
|
||
)
|
||
assert r_try_teep.status_code == 200, r_try_teep.text
|
||
assert r_try_teep.json()["row_count"] == 1
|
||
assert r_try_teep.json()["rows"][0][0] > 0
|
||
finally:
|
||
await drop_source(sid)
|
||
|
||
|
||
async def test_oracle_dialect_sql_compilation_and_validation(client: AsyncClient) -> None:
|
||
"""回归验证 Oracle 方言下的多表 WITH CTE 模板、单表表达式、ROWNUM 行数限制、标识符不误加小写双引号与 DATE 字面量。"""
|
||
import sqlglot
|
||
|
||
from app.data.guard import check as guard_check
|
||
from app.data.semantic_metrics import compile_metric_sql, create_metric
|
||
from app.db.models import DataSource
|
||
|
||
async with SessionLocal() as session:
|
||
ora_ds = DataSource(
|
||
name=f"oracle_mes_{uuid.uuid4().hex[:6]}",
|
||
kind="oracle",
|
||
host="127.0.0.1",
|
||
port=1521,
|
||
database="ORCL",
|
||
username="mes_ro",
|
||
password="",
|
||
readonly=True,
|
||
options={"schema": "MES_PROD"},
|
||
schema_cache={
|
||
"schema": "MES_PROD",
|
||
"tables": {
|
||
"mes_kpi_shift_fact": {
|
||
"qualified": "MES_PROD.mes_kpi_shift_fact",
|
||
"schema": "MES_PROD",
|
||
"columns": [
|
||
{"name": "factory_code", "type": "VARCHAR2(32 CHAR)", "nullable": False},
|
||
{"name": "object_id", "type": "VARCHAR2(32 CHAR)", "nullable": False},
|
||
{"name": "start_date_time", "type": "DATE", "nullable": False},
|
||
{"name": "attribute01", "type": "NUMBER(18, 2)", "nullable": True},
|
||
{"name": "attribute05", "type": "NUMBER(18, 2)", "nullable": True},
|
||
{"name": "attribute14", "type": "NUMBER(18, 2)", "nullable": True},
|
||
{"name": "attribute15", "type": "NUMBER(18, 2)", "nullable": True},
|
||
{"name": "attribute54", "type": "NUMBER(18, 2)", "nullable": True},
|
||
{"name": "lower_quoted_col", "type": "VARCHAR2(16 CHAR)", "nullable": True, "quote_name": True},
|
||
],
|
||
},
|
||
"iot_prod_line_dim": {
|
||
"qualified": "MES_PROD.iot_prod_line_dim",
|
||
"schema": "MES_PROD",
|
||
"columns": [
|
||
{"name": "line_code", "type": "VARCHAR2(32 CHAR)", "nullable": False},
|
||
{"name": "beat", "type": "VARCHAR2(32 CHAR)", "nullable": True},
|
||
],
|
||
},
|
||
},
|
||
},
|
||
)
|
||
session.add(ora_ds)
|
||
await session.flush()
|
||
|
||
tpl = (
|
||
"WITH ln AS (SELECT line_code, AVG(TO_NUMBER(beat)) AS m_line FROM MES_PROD.iot_prod_line_dim GROUP BY line_code) "
|
||
"SELECT {{select_dims}} "
|
||
"ROUND(SUM((k.attribute14 + k.attribute15) * k.attribute54) * 100.0 / NULLIF(SUM(ln.m_line * k.attribute54), 0) "
|
||
"* (SUM(GREATEST(k.attribute01 - k.attribute05, 0)) / NULLIF({{period_days}} * 86400 * COUNT(DISTINCT k.object_id), 0)), 2) AS u_plan "
|
||
"FROM MES_PROD.mes_kpi_shift_fact k LEFT JOIN ln ON ln.line_code = k.object_id "
|
||
"WHERE 1=1 {{and_where}} {{group_by}} {{order_by}}"
|
||
)
|
||
metric_row, _ = await create_metric(
|
||
session,
|
||
{
|
||
"key": "ora_u_plan",
|
||
"name": "计划综合利用率(Oracle跨表)",
|
||
"source_id": ora_ds.id,
|
||
"calc_mode": "sql_template",
|
||
"table_name": "mes_kpi_shift_fact",
|
||
"sql_template": tpl,
|
||
"date_column": "k.start_date_time",
|
||
"dimensions": [
|
||
{"key": "factory_code", "label": "工厂", "column": "k.factory_code"},
|
||
{"key": "lower_col", "label": "特殊小写列", "column": "k.lower_quoted_col"},
|
||
],
|
||
},
|
||
)
|
||
|
||
# 1) 按工厂维度分组(无日期区间):验证无 LIMIT、无 CAST AS TEXT、无小写引号 "k"."factory_code"、WITH 保持在最外层
|
||
sql_grouped, _ = compile_metric_sql(
|
||
metric_row,
|
||
source=ora_ds,
|
||
group_by=["factory_code"],
|
||
limit=30,
|
||
)
|
||
assert "LIMIT" not in sql_grouped.upper()
|
||
assert "CAST(" not in sql_grouped.upper()
|
||
assert '"k"."factory_code"' not in sql_grouped
|
||
assert "k.factory_code" in sql_grouped
|
||
assert "TO_CHAR(k.start_date_time, 'YYYY-MM-DD')" in sql_grouped
|
||
assert sql_grouped.startswith("WITH ln AS")
|
||
assert "WHERE ROWNUM <= 30" in sql_grouped
|
||
guard_check(sql_grouped, readonly=True, dialect="oracle", source_name=ora_ds.name)
|
||
sqlglot.parse_one(sql_grouped, read="oracle")
|
||
|
||
# 2) 带日期区间 + quote_name=True 的特殊小写列分组:验证 Oracle DATE 字面量与跨天 < DATE 'next_day'
|
||
sql_dated, _ = compile_metric_sql(
|
||
metric_row,
|
||
source=ora_ds,
|
||
date_from="2026-11-01",
|
||
date_to="2026-11-07",
|
||
group_by=["lower_col"],
|
||
limit=20,
|
||
)
|
||
assert "k.start_date_time >= DATE '2026-11-01'" in sql_dated
|
||
assert "k.start_date_time < DATE '2026-11-08'" in sql_dated
|
||
assert 'k."lower_quoted_col"' in sql_dated
|
||
assert "7 * 86400" in sql_dated
|
||
sqlglot.parse_one(sql_dated, read="oracle")
|
||
|
||
# 3) 单表 expression 模式:验证自动补齐 MES_PROD. 前缀、按表达式而非 GROUP BY 1 分组、套外层 ROWNUM
|
||
expr_row, _ = await create_metric(
|
||
session,
|
||
{
|
||
"key": "ora_open_hours",
|
||
"name": "计划开班总秒数",
|
||
"source_id": ora_ds.id,
|
||
"calc_mode": "expression",
|
||
"table_name": "mes_kpi_shift_fact",
|
||
"sql_expression": "SUM(attribute01)",
|
||
"date_column": "start_date_time",
|
||
"dimensions": [{"key": "factory_code", "label": "工厂", "column": "factory_code"}],
|
||
},
|
||
)
|
||
sql_expr, _ = compile_metric_sql(
|
||
expr_row,
|
||
source=ora_ds,
|
||
group_by_date=True,
|
||
group_by=["factory_code"],
|
||
limit=15,
|
||
)
|
||
assert "FROM MES_PROD.mes_kpi_shift_fact" in sql_expr
|
||
assert "GROUP BY 1" not in sql_expr
|
||
assert "GROUP BY TO_CHAR(start_date_time, 'YYYY-MM-DD'), factory_code" in sql_expr
|
||
assert "LIMIT" not in sql_expr.upper()
|
||
assert "WHERE ROWNUM <= 15" in sql_expr
|
||
sqlglot.parse_one(sql_expr, read="oracle")
|
||
await session.rollback()
|
||
|