diff --git a/backend/app/engine/dual_check_facts.py b/backend/app/engine/dual_check_facts.py index d33b839c..112ed028 100644 --- a/backend/app/engine/dual_check_facts.py +++ b/backend/app/engine/dual_check_facts.py @@ -4,7 +4,10 @@ 对得上的事实只有三种(data/catalog.py 的表级项): - 两边按不同的时间列筛时间 → 业务日期(business_date):被采纳的一边按哪一列筛,业务日期就记哪一列。只有一边按时间 - 筛的不算:审核人判断的是要不要限定时间,没在两列之间选过 + 筛的不算:审核人判断的是要不要限定时间,没在两列之间选过。两边的列都得在表结构里声明为日期 / 时间类型(DATE、 + DATETIME、TIMESTAMP、TIMESTAMPTZ、TIME,带精度、带时区的写法也算):差异说明认时间列只看列名和常量像不像日期, + 文本列(SQLite 常把日期存成文字)、整数列(Unix 秒、20260901 这类日期键)、名字带 time 的时长列都可能被说成「按它筛 + 时间」,认不准存的是不是日期,不记 - 一边多一个排除无效记录的条件 → 有效记录条件(valid_filter):被采纳的一边有这个条件才记,记的就是这个条件 - 聚合只差去不去重、被采纳的一边去重 → 去重规则(dedup)。采纳了不去重的一边不记:那是这个指标自己的算法(数行), 写成表的「不去重」会把别的指标带偏 @@ -55,6 +58,7 @@ from typing import Any from sqlalchemy import select from app.data import catalog +from app.data.names import name_key from app.db.base import SessionLocal from app.db.models import DataSource @@ -92,6 +96,11 @@ _MARKS_INVALID = _INVALID_WORDS | {"del", "removed", "hidden", "archived", "inte _MARKS_VALID = frozenset({"valid", "enabled", "active", "effective"}) #: 取值和指标名称、口径说明比对时,数字至少这么长才比(0、1 这种标记值到处都是) _SCOPE_DIGITS = 2 +#: 表结构里声明的日期 / 时间类型(业务日期只记这样的列):DATE、DATETIME、TIMESTAMP、TIMESTAMPTZ、TIME、TIMETZ,可以带 +#: 精度(TIMESTAMP(3)、DATETIME(6))、带时区(TIMESTAMP WITH TIME ZONE、TIME WITHOUT TIME ZONE、Oracle 的 WITH LOCAL +#: TIME ZONE),大小写不论。整个类型名都得是这个样子:DATE_KEY、INTERVAL 这类名字里带着 date、time 的不算 +_TEMPORAL_TYPE = re.compile(r"(?:date|datetime|timestamp|timestamptz|time|timetz)(?:\s*\(\s*\d+\s*\))?" + r"(?:\s+with(?:out)?\s+(?:local\s+)?time\s+zone)?", re.IGNORECASE) @dataclass(frozen=True) @@ -131,8 +140,8 @@ def derive(candidate: Mapping[str, Any], schema_cache: Mapping[str, Any] | None, candidate:{metric 指标名, choice "a" | "b", source 数据源名, dialect, sql: {a, b}(比较的两条 SQL), diff: {equivalent, fallback, items: [FacetDiff.as_dict()]}, scope: {texts, columns}}。scope 是这个指标自己的范围: texts 是口径说明、所在分组的取值、本次运行的输入(指标名称总算在内),columns 是分组的列;有效记录条件用它排除 - 指标自己的筛选范围。notes 是这个数据源的目录({表名: notes}),认码值的意思用。只看差异说明,不重新比 SQL:审核人 - 看到的就是这份说明。 + 指标自己的筛选范围。schema_cache 是数据源的表结构:认表,业务日期还要看两边的列声明的类型。notes 是这个数据源的 + 目录({表名: notes}),认码值的意思用。只看差异说明,不重新比 SQL:审核人看到的就是这份说明。 """ name = str(candidate.get("metric") or "") side = candidate.get("choice") @@ -148,7 +157,7 @@ def derive(candidate: Mapping[str, Any], schema_cache: Mapping[str, Any] | None, item = items[0] aspect = str(item.get("aspect") or "") if aspect == "time": - fact, why = _business_date(item, side) + fact, why = _business_date(item, side, schema_cache) elif aspect == "filter": fact, why = _valid_filter(item, side, candidate, schema_cache, notes or {}) elif aspect == "aggregate": @@ -170,7 +179,10 @@ def derive(candidate: Mapping[str, Any], schema_cache: Mapping[str, Any] | None, metric_id=str(candidate.get("metric_id") or "")), None -def _business_date(item: Mapping[str, Any], side: str) -> tuple[tuple[str, str, Any] | None, str | None]: +def _business_date(item: Mapping[str, Any], side: str, schema_cache: Mapping[str, Any] | None + ) -> tuple[tuple[str, str, Any] | None, str | None]: + """两边按不同的列筛时间、被采纳的一边按哪一列:(表, business_date, {column}) 或 (None, 为什么不记)。两边的列都要在 + 表结构里声明为日期 / 时间类型(_temporal_problem),被采纳的一边先看:它有问题说它,没问题再看另一边。""" other = "b" if side == "a" else "a" detail = item.get("detail") if isinstance(item.get("detail"), Mapping) else {} @@ -187,9 +199,39 @@ def _business_date(item: Mapping[str, Any], side: str) -> tuple[tuple[str, str, return None, "有一边按好几列筛时间,无法确定业务日期按哪一列" if (mine[0]["table"], mine[0]["column"]) == (theirs[0]["table"], theirs[0]["column"]): return None, "两边按同一列筛时间,没有可记的业务日期" + for label, ref in ((SIDE_LABEL[side], mine[0]), (SIDE_LABEL[other], theirs[0])): + if (problem := _temporal_problem(schema_cache, ref)) is not None: + return None, f"{label}按时间筛的列「{ref['table']}.{ref['column']}」{problem},无法确认是日期 / 时间类型" return (str(mine[0]["table"]), "business_date", {"column": str(mine[0]["column"])}), None +def _temporal_problem(schema_cache: Mapping[str, Any] | None, ref: Mapping[str, Any]) -> str | None: + """按时间筛的这一列在表结构里是不是声明成了日期 / 时间类型(_TEMPORAL_TYPE):是的为 None,不是的交回它怎么了 + (查不到、没声明类型、声明成了别的类型),接在列名后面读。 + + 表按 catalog.resolve_table_name 认,列先按原样、再按 name_key 认(差异说明里的名字是规范化过的,SQLite 不加引号的 + 名字一律小写)。按 name_key 认出好几列的说对应好几列:哪一列是它说不准。""" + cache = schema_cache if isinstance(schema_cache, Mapping) else None + table = catalog.resolve_table_name(cache, str(ref["table"])) + tables = cache.get("tables") if cache is not None else None + meta = tables.get(table) if isinstance(tables, Mapping) and table is not None else None + columns = [c for c in meta.get("columns") or [] if isinstance(c, Mapping) and c.get("name")] \ + if isinstance(meta, Mapping) else [] + column = str(ref["column"]) + found = [c for c in columns if str(c["name"]) == column] \ + or [c for c in columns if name_key(str(c["name"])) == name_key(column)] + if not found: + return "不在数据源的表结构里" + if len(found) > 1: + return "在表结构里对应好几列" + declared = " ".join(str(found[0].get("type") or "").split()) + if not declared: + return "在表结构里没有声明类型" + if not _TEMPORAL_TYPE.fullmatch(declared): + return f"在表结构里声明为 {declared}" + return None + + def _valid_filter(item: Mapping[str, Any], side: str, candidate: Mapping[str, Any], schema_cache: Mapping[str, Any] | None, notes: Mapping[str, Mapping[str, Any]]) -> tuple[tuple[str, str, Any] | None, str | None]: diff --git a/backend/tests/fixtures/dual_check_shop.py b/backend/tests/fixtures/dual_check_shop.py index 565ca784..3d14ab28 100644 --- a/backend/tests/fixtures/dual_check_shop.py +++ b/backend/tests/fixtures/dual_check_shop.py @@ -9,6 +9,9 @@ - 付款客户数(按支付日、排除退款):按行计数 3,去重计数 2 另有渠道(channel)和测试单标记(is_test,都是 0)两列:写数据目录的测试拿它们凑「指标自己的筛选范围」和标记列。 + +下单日、支付日两列声明为 DATE(SQLite 照样存成文字,按字符串比的先后和按日期比的一样):双盲复算只在两边的时间列都 +声明为日期 / 时间类型时才把选择记成业务日期。build(date_type="TEXT") 建一份声明成文本的,看不记的那一种。 """ from __future__ import annotations @@ -50,11 +53,11 @@ BY_REGION = (f"SELECT region, SUM(amount) AS gmv FROM orders WHERE status <> 're DEFINITION = "已支付订单的金额合计:按支付日期(paid_date)归属到当周,不含退款" -def build(path: Path) -> Path: +def build(path: Path, *, date_type: str = "DATE") -> Path: db = sqlite3.connect(path) db.executescript( "CREATE TABLE orders (id INTEGER PRIMARY KEY, region TEXT, customer_id INTEGER, amount REAL, status TEXT, " - "order_date TEXT, paid_date TEXT, channel TEXT, is_test INTEGER);") + f"order_date {date_type}, paid_date {date_type}, channel TEXT, is_test INTEGER);") db.executemany("INSERT INTO orders VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", ROWS) db.commit() db.close() diff --git a/backend/tests/test_dual_check_facts.py b/backend/tests/test_dual_check_facts.py index de39d7d9..e845a710 100644 --- a/backend/tests/test_dual_check_facts.py +++ b/backend/tests/test_dual_check_facts.py @@ -2,8 +2,11 @@ - 只差一个方面、又对得上目录事实时才写:两边按不同的时间列筛 → 业务日期,一边多一个排除无效记录的条件 → 有效记录 条件,聚合只差去不去重、被采纳的一边去重 → 去重规则;按被采纳那一边的写法记 +- 业务日期只在两边的列都在表结构里声明为日期 / 时间类型(DATE、DATETIME、TIMESTAMP、TIME,带精度、带时区的写法也算) + 时才写:文本、整数(Unix 秒、20260901 这类日期键)、表结构里查不到、没声明类型的不写,原因说清是哪一列 - 差了不止一处、有兜底说明、两条 SQL 规范化后一致、差在别的方面、被采纳的一边没有可记的写法、只有一边按时间筛:不写, - 照实说原因 + 照实说原因。同一列上的时间范围不同、跨两张表的条件,差异说明只给兜底,同样不写;说明以后真把跨两张表的条件说成一条 + 筛选条件,derive 自己那道「只涉及一张表」的检查也拦得下 - 指标自己的筛选范围(只看华东、只看线上)不是有效记录条件:取值出现在指标名称、口径说明、所在分组、运行输入里的, 条件用到分组的列的,不是在排除退款、作废、测试这类无效记录的,都不写 - 写的是人工填写、已确认,备注写明运行、节点、指标;同样的说法已确认过就不再写(不升版本),确认过的不同说法不覆盖; @@ -40,8 +43,8 @@ SCHEMA = {"tables": { "orders": {"name": "orders", "columns": [ {"name": "id", "type": "INTEGER"}, {"name": "region", "type": "TEXT"}, {"name": "customer_id", "type": "INTEGER"}, {"name": "amount", "type": "REAL"}, - {"name": "status", "type": "TEXT"}, {"name": "order_date", "type": "TEXT"}, - {"name": "paid_date", "type": "TEXT"}, {"name": "channel", "type": "TEXT"}, + {"name": "status", "type": "TEXT"}, {"name": "order_date", "type": "DATE"}, + {"name": "paid_date", "type": "DATE"}, {"name": "channel", "type": "TEXT"}, {"name": "is_test", "type": "INTEGER"}], "primary_key": ["id"]}, "customers": {"name": "customers", "columns": [ {"name": "id", "type": "INTEGER"}, {"name": "level", "type": "TEXT"}, {"name": "status", "type": "TEXT"}], @@ -169,13 +172,33 @@ def test_table_names_are_dropped_from_the_recorded_filter(): assert why is None and fact.value == "status <> 'refunded'" +#: 订单关联客户;盲算另多一个比较两张表的列的条件 +JOINED = (f"SELECT SUM(o.amount) AS gmv FROM orders o JOIN customers c ON o.customer_id = c.id " + f"WHERE o.paid_date {WEEK}") + + def test_a_condition_across_two_tables_is_not_recorded(): - joined = (f"SELECT SUM(o.amount) AS gmv FROM orders o JOIN customers c ON o.customer_id = c.id " - f"WHERE o.paid_date {WEEK}") - picked = candidate(joined, f"{joined} AND o.status <> c.status", "b") - assert [i["aspect"] for i in picked["diff"]["items"]] == ["filter"] + """几张表交错的条件,差异说明只给兜底(归纳不成具体差异):审核人选的是整体哪边对,不写。""" + picked = candidate(JOINED, f"{JOINED} AND o.status <> c.status", "b") + assert picked["diff"]["fallback"] is True and [i["aspect"] for i in picked["diff"]["items"]] == ["other"] fact, why = derive(picked, SCHEMA) - assert fact is None and "不是只涉及一张表的列" in why + assert fact is None and "未能归纳" in why and "销售额" in why + + +def test_a_filter_spanning_two_tables_is_not_a_valid_filter(): + """差异说明真把跨两张表的条件说成一条筛选条件(以后说明放宽了、或者别的来源交来的说明):derive 自己也拦得下—— + 有效记录条件记在一张表名下,用到两张表的列的写进哪张都不对。差异说明直接构造:从 SQL 走现在只有兜底。""" + picked = candidate(JOINED, f"{JOINED} AND o.status <> c.status", "b") + picked["diff"] = {"equivalent": False, "fallback": False, "items": [{ + "aspect": "filter", "a": None, "b": "o.status <> c.status", + "text": "盲算只保留 o.status <> c.status 的行,原流程没有这个条件", + "detail": {"columns": [{"table": "customers", "column": "status"}, {"table": "orders", "column": "status"}]}, + }]} + fact, why = derive(picked, SCHEMA) + assert fact is None and "不是只涉及一张表的列" in why and "销售额" in why + # 同样的条件只涉及一张表时这道检查放行(后面照常看是不是在排除无效记录):拦下它的确实是「两张表」 + picked["diff"]["items"][0]["detail"] = {"columns": [{"table": "orders", "column": "status"}]} + assert "不是只涉及一张表的列" not in (derive(picked, SCHEMA)[1] or "") def test_counting_distinct_becomes_the_dedup_rule(): @@ -207,16 +230,96 @@ def test_equivalent_sql_or_fallback_or_other_aspects_are_not_recorded(): assert [i["aspect"] for i in joined["diff"]["items"]] == ["aggregate"] assert "不只是去不去重" in derive(joined, SCHEMA)[1] + # 同一列上的时间范围不同:差异说明只给兜底(说不清差的是哪一段),不写 ranged = candidate(RIGHT, RIGHT.replace("'2026-09-07'", "'2026-09-06'"), "b") - assert [i["aspect"] for i in ranged["diff"]["items"]] == ["time_range"] - assert "时间范围" in derive(ranged, SCHEMA)[1] + assert ranged["diff"]["fallback"] is True and [i["aspect"] for i in ranged["diff"]["items"]] == ["other"] + assert "未能归纳" in derive(ranged, SCHEMA)[1] + + # 只差一个方面、可那个方面对不上目录里的三项(关联方式):不写,原因说清差在哪 + left = (f"SELECT SUM(o.amount) AS gmv FROM orders o LEFT JOIN customers c ON c.id = o.customer_id " + f"WHERE o.paid_date {WEEK}") + joins = candidate(left, left.replace("LEFT JOIN", "JOIN"), "b") + assert joins["diff"]["fallback"] is False and [i["aspect"] for i in joins["diff"]["items"]] == ["join"] + fact, why = derive(joins, SCHEMA) + assert fact is None and "两边差在关联" in why and "对不上数据目录里的业务日期、有效记录条件或去重规则" in why assert "没有采用任何一边" in derive({**candidate(BY_ORDER_DATE, RIGHT, "b"), "choice": "neither"}, SCHEMA)[1] def test_a_table_missing_from_the_schema_is_not_recorded(): - fact, why = derive(candidate(BY_ORDER_DATE, RIGHT, "b"), {"tables": {"refunds": {"columns": []}}}) - assert fact is None and "不在数据源的表结构里" in why + """认出了事实(排除退款的有效记录条件),表却不在数据源的表结构里:不写。""" + fact, why = derive(candidate(WITH_REFUNDS, RIGHT, "b"), {"tables": {"refunds": {"columns": []}}}) + assert fact is None and "涉及的表「orders」不在数据源的表结构里" in why + + +def schema_with(**types: str | None) -> dict: + """SCHEMA 的订单表里几列换成给定的声明类型;给 None 的列从表结构里拿掉。""" + orders = SCHEMA["tables"]["orders"] + columns = [{**c, "type": types[c["name"]]} if c["name"] in types else c + for c in orders["columns"] if not (c["name"] in types and types[c["name"]] is None)] + return {"tables": {**SCHEMA["tables"], "orders": {**orders, "columns": columns}}} + + +@pytest.mark.parametrize("order_date, paid_date", [ + ("DATE", "DATE"), + ("TIMESTAMP", "TIMESTAMP"), + ("datetime", "DateTime(6)"), + ("TIMESTAMPTZ", "timestamp(3) with time zone"), + ("TIMESTAMP WITHOUT TIME ZONE", "TIMESTAMP(6) WITH LOCAL TIME ZONE"), + ("time", "TIME WITH TIME ZONE"), +]) +def test_a_business_date_is_written_when_both_columns_are_declared_as_dates_or_times(order_date, paid_date): + """两边的列都在表结构里声明为日期 / 时间类型(带精度、带时区、大小写不同都认):照常记被采纳的那一列。""" + fact, why = derive(candidate(BY_ORDER_DATE, RIGHT, "b"), schema_with(order_date=order_date, paid_date=paid_date)) + assert why is None and (fact.item, fact.value) == ("business_date", {"column": "paid_date"}) + + +@pytest.mark.parametrize("types, column, said", [ + # SQLite 常把日期存成文字:认不准存的是不是日期 + ({"paid_date": "TEXT"}, "orders.paid_date", "在表结构里声明为 TEXT"), + ({"paid_date": "VARCHAR(10)"}, "orders.paid_date", "在表结构里声明为 VARCHAR(10)"), + # 没被采纳的一边也得是:它要是按 Unix 秒、20260901 这类整数筛的,审核人并没有在两个日期列之间选 + ({"order_date": "INTEGER"}, "orders.order_date", "在表结构里声明为 INTEGER"), + ({"order_date": "BIGINT"}, "orders.order_date", "在表结构里声明为 BIGINT"), + # 名字里带着 date、time 的别的类型也不算 + ({"paid_date": "DATE_KEY"}, "orders.paid_date", "在表结构里声明为 DATE_KEY"), + ({"paid_date": "INTERVAL"}, "orders.paid_date", "在表结构里声明为 INTERVAL"), + ({"paid_date": ""}, "orders.paid_date", "在表结构里没有声明类型"), + ({"paid_date": None}, "orders.paid_date", "不在数据源的表结构里"), +]) +def test_a_business_date_is_not_written_unless_both_columns_are_dates_or_times(types, column, said): + """任一边的列不是声明的日期 / 时间类型(文本、整数、查不到、没声明类型):不写,原因说清是哪一列。选哪一边都一样: + 被采纳的那一边有问题说它,没问题就说另一边的。""" + for choice in ("a", "b"): + fact, why = derive(candidate(BY_ORDER_DATE, RIGHT, choice), schema_with(**types)) + assert fact is None, (types, choice) + assert why.startswith("指标「销售额」") and why.endswith(",无法确认是日期 / 时间类型,不写入数据目录"), why + assert f"按时间筛的列「{column}」{said}" in why, why + + +@pytest.mark.parametrize("schema", [None, {}, {"tables": {}}, {"tables": {"customers": SCHEMA["tables"]["customers"]}}]) +def test_a_business_date_is_not_written_without_the_table_structure(schema): + """没有表结构(还没探查、探查失败、表不在里面):确认不了两列的类型,不写。""" + fact, why = derive(candidate(BY_ORDER_DATE, RIGHT, "b"), schema) + assert fact is None and "按时间筛的列「orders.paid_date」不在数据源的表结构里,无法确认是日期 / 时间类型" in why + + +def test_the_type_check_follows_the_names_in_the_table_structure(): + """差异说明里的表名、列名是规范化过的(小写):表结构里写成大写的照样认得出,类型照它声明的看。""" + upper = {"tables": {"ORDERS": {"name": "ORDERS", "columns": [ + {"name": "STATUS", "type": "TEXT"}, {"name": "ORDER_DATE", "type": "DATE"}, {"name": "PAID_DATE", "type": "TEXT"}, + {"name": "AMOUNT", "type": "REAL"}]}}} + fact, why = derive(candidate(BY_ORDER_DATE, RIGHT, "a"), upper) + assert fact is None and "「orders.paid_date」在表结构里声明为 TEXT" in why + + +def test_the_preview_follows_the_type_check(): + """审批卡上的预告和真写同一套判断:一边的列是文本,选哪一边都预告不写(认不出事实,不带写到哪),原因同 derive。""" + schema = schema_with(paid_date="TEXT") + said = preview(candidate(BY_ORDER_DATE, RIGHT, "a"), schema, {}) + for side in ("a", "b"): + assert said[side] == {"write": False, "reason": derive(candidate(BY_ORDER_DATE, RIGHT, side), schema)[1]} + assert "「orders.paid_date」在表结构里声明为 TEXT" in said[side]["reason"] # --------------------------------------------------------------------------