From 7f949cce9243e4f0b73c29a248661972ec8bbc6a Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Sat, 17 Oct 2026 07:59:19 +0800 Subject: [PATCH 01/15] =?UTF-8?q?=E5=8F=96=E5=80=BC=E8=A7=82=E5=AF=9F?= =?UTF-8?q?=EF=BC=9A=E4=B8=9A=E5=8A=A1=E6=97=A5=E6=9C=9F=E6=9C=89=E6=9C=AA?= =?UTF-8?q?=E6=9D=A5=E5=80=BC=E6=97=B6=E4=B8=8D=E5=86=8D=E8=AF=AF=E6=8A=A5?= =?UTF-8?q?=E9=87=8F=E8=B7=8C=E3=80=81=E7=BC=BA=E5=A4=A9=EF=BC=9B=E3=80=8C?= =?UTF-8?q?=E4=BB=8A=E5=A4=A9=E3=80=8D=E7=BB=9F=E4=B8=80=E6=8C=89=E6=9C=8D?= =?UTF-8?q?=E5=8A=A1=E7=AB=AF=E6=89=80=E5=9C=A8=E6=97=B6=E5=8C=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 验收复现：业务日期最晚到明天、今天只写了几行，记出一条 volume_drop。原因有两层： - 比较器 value_drift.daily() 取 last = max(days)，只在 last >= today 时退一天；max 落在明天时退到的是今天这个 还没写完的日子。改成最近一个整天 = min(有数据的最后一天, 昨天)。 - 按日行数的查询 day_volume_sql 只有下界，未来的日子也进了窗口。加上界：数到观察当天（含），第二天起不算 （< 第二天，时间列上当天的时分秒照样算进来，文字存的日期按字符串比同样成立）。签名改成必须传 until。 缺天同样的问题也修了：今天以后的空日子不算缺天；以前那次观察只看到它自己那天（data.until）为止，免得它 窗口里未来的空日子被当成「以前就有过缺天」，挡住这一次真缺的天。断档遇到未来值时离今天是负数，只会报不出、 不会误报，观察本身照实记最新业务日期（目录里也照实给人看），不截未来值，写进了模块说明。 「今天」：原来窗口用 UTC 日期、比较器用 now.date()（也是 UTC），业务日期是本地日期，东八区凌晨 0–8 点会差 一天。统一成服务端所在时区（catalog_profile_observe.local_today，和待办汇总 notify.digest_at 同一个口径）， 写进模块说明。 测试：test_value_drift 两条（未来值不算最近整天、未来空日子不算缺天且历史按 until 截）；test_catalog_observe 两条（SQLite 真跑剖析任务，明天的行不进窗口、不记量跌；local_today 按 TZ 取本地日期）。去掉修复回跑都挂。 金样（tests/fixtures/dialect/*.sql）的 day_volume 一行随上界更新；scripts/dialect_it.py 加了一条「只数到观察 当天」的真库核对。 --- backend/app/data/catalog_profile.py | 22 ++++++---- backend/app/data/catalog_profile_observe.py | 22 ++++++---- backend/app/data/value_drift.py | 26 +++++++---- backend/tests/fixtures/dialect/mariadb.sql | 2 +- backend/tests/fixtures/dialect/mysql.sql | 2 +- backend/tests/fixtures/dialect/oracle.sql | 2 +- backend/tests/fixtures/dialect/postgresql.sql | 2 +- backend/tests/fixtures/dialect/sqlite.sql | 2 +- backend/tests/test_catalog_observe.py | 43 +++++++++++++++++++ backend/tests/test_catalog_profile_units.py | 2 +- backend/tests/test_profile_dialect_golden.py | 2 +- backend/tests/test_value_drift.py | 27 ++++++++++++ scripts/dialect_it.py | 11 ++++- 13 files changed, 133 insertions(+), 32 deletions(-) diff --git a/backend/app/data/catalog_profile.py b/backend/app/data/catalog_profile.py index ed63b2ed..7ea86279 100644 --- a/backend/app/data/catalog_profile.py +++ b/backend/app/data/catalog_profile.py @@ -45,7 +45,7 @@ import re import time from collections.abc import Awaitable, Callable from dataclasses import asdict, dataclass, field -from datetime import date as _date, datetime, timezone +from datetime import date as _date, datetime, timedelta, timezone from decimal import Decimal from typing import Any, NamedTuple @@ -493,17 +493,23 @@ class SqlDialect: hi = self.date_literal(VALID_DATE_BEFORE, text=text) return f"SELECT MAX({column}) FROM {table} WHERE {column} >= {lo} AND {column} < {hi}" - def _window(self, column: str, since: Any, text: bool) -> str: - """时间窗口的条件：since 起（含），并且早于占位日期 9999-12-31——PostgreSQL 的 infinity、Oracle 的 9999-12-31 - 都大于任何一天，不截掉上端就会在「最近 35 天」里多出一天。""" + def _window(self, column: str, since: Any, text: bool, before: Any = None) -> str: + """时间窗口的条件：since 起（含），并且早于 before（不含；缺省是占位日期 9999-12-31——PostgreSQL 的 infinity、 + Oracle 的 9999-12-31 都大于任何一天，不截掉上端就会在「最近 35 天」里多出一天）。""" return (f"{column} >= {self.date_literal(since, text=text)} " - f"AND {column} < {self.date_literal(VALID_DATE_BEFORE, text=text)}") + f"AND {column} < {self.date_literal(before or VALID_DATE_BEFORE, text=text)}") - def day_volume_sql(self, table: str, column: str, since: Any, *, text: bool = False) -> str: - """最近若干天按天计数：since 起（含）每天几行。只在日期列打头有索引时对大表发（走索引的范围扫描）。""" + def day_volume_sql(self, table: str, column: str, since: Any, until: Any, *, text: bool = False) -> str: + """最近若干天按天计数：since 到 until（都含；until 是观察当天）每天几行。只在日期列打头有索引时对大表发（走索引的 + 范围扫描）。上界是 until 的第二天（不含）：业务日期有未来值（预约、计划类的日期，或者写错的）时，今天以后的日子 + 不进窗口——它们不是「最近的整天」，也不该被当成缺天（data/value_drift）。时间列上「< 第二天」把 until 当天的 + 时分秒都算进来；文字存的日期按字符串比，同样成立。""" + last = until if isinstance(until, _date) else _date.fromisoformat(str(until)) + if isinstance(last, datetime): + last = last.date() day = self.day_trunc(column) return (f"SELECT {day} AS d, COUNT(*) AS n FROM {table} " - f"WHERE {self._window(column, since, text)} GROUP BY {day} ORDER BY d") + f"WHERE {self._window(column, since, text, last + timedelta(days=1))} GROUP BY {day} ORDER BY d") def window_value_counts_sql(self, table: str, column: str, date_column: str, since: Any, n: int, *, text: bool = False) -> str: diff --git a/backend/app/data/catalog_profile_observe.py b/backend/app/data/catalog_profile_observe.py index 78b07a31..0663312e 100644 --- a/backend/app/data/catalog_profile_observe.py +++ b/backend/app/data/catalog_profile_observe.py @@ -8,7 +8,11 @@ - rows：这张表的行数（剖析开头估行数那一条的结果，不另发查询）；口径 stats（数据库统计的估算值）/ count（数到上限）； - codes_observe：码值列的取值分布——**所有**码值列都观察，包括人工确认过、标了已列全的（目录-D6）；只读，不改含义； - freshness：业务日期列（已确认或已验证）的最新日期； -- daily_volume：业务日期最近 35 天每天的行数。 +- daily_volume：业务日期最近 35 天每天的行数，数到观察当天为止（今天以后的日子不进窗口，见 SqlDialect.day_volume_sql）。 + +**「今天」按服务端所在时区**（local_today，和待办汇总的 notify.digest_at 同一个口径）：窗口从哪天起、数到哪天、比较器的 +「今天」（最近一个整天是哪天、断档离今天几天）都用它。业务日期是本地日期，按 UTC 的日期算，东八区凌晨 0–8 点的「今天」 +还是前一天，这几样都会差一天。服务端和业务库不在一个时区时，差的那几个小时照实差着（业务库的时区探查不到）。 **大表的口径**（目录-D7）：小表（行数不超过整表统计的上限）看整表（full）。大表：业务日期已确认或已验证、而且这一列打头 有索引（探查记下的 index_first_columns）时看最近 35 天的窗口（window，走索引的范围扫描）；PostgreSQL / Oracle 没有就块采样 @@ -139,12 +143,13 @@ def sample_percent(rows: int | None, target: int) -> float | None: return next((p for p in SAMPLE_LADDER if p <= want), SAMPLE_LADDER[-1]) -def _today() -> date: - return datetime.now(timezone.utc).date() +def local_today(now: datetime | None = None) -> date: + """观察用的「今天」：服务端所在时区的日期（和待办汇总的 notify.digest_at 同一个口径，见模块说明）。now 缺省是现在。""" + return (now or utcnow()).astimezone().date() def _since() -> date: - return _today() - timedelta(days=WINDOW_DAYS) + return local_today() - timedelta(days=WINDOW_DAYS) def _window_ok(cx: Any, meta: Mapping[str, Any], day: str | None) -> bool: @@ -271,8 +276,9 @@ async def _dates(cx: Any, table: str, kind: str, column: str, meta: Mapping[str, result.observations.append(_observation("freshness", column, {"method": "full"}, {"max": str(value)[:19]}, [])) return - since = _since() - res = await cx.run(d.day_volume_sql(t, c, since, text=text), max_rows=WINDOW_DAYS + 2) + today = local_today() + since = today - timedelta(days=WINDOW_DAYS) + res = await cx.run(d.day_volume_sql(t, c, since, today, text=text), max_rows=WINDOW_DAYS + 2) days: dict[str, int] = {} for row in res.rows: day = str(row[0])[:10] if row and row[0] is not None else "" @@ -280,7 +286,7 @@ async def _dates(cx: Any, table: str, kind: str, column: str, meta: Mapping[str, days[day] = days.get(day, 0) + (cp._as_int(row[1]) or 0) basis = {"method": "window", "column": column, "days": WINDOW_DAYS, "since": since.isoformat()} result.observations.append(_observation("daily_volume", column, basis, - {"since": since.isoformat(), "until": _today().isoformat(), + {"since": since.isoformat(), "until": today.isoformat(), "days": days}, [])) @@ -396,7 +402,7 @@ async def _save_one(session: AsyncSession, source: Any, table: str, obs: Mapping current = vd.Obs(kind, obs["basis_key"], obs["data"], now) past = [vd.Obs(h.kind, h.basis_key, h.data or {}, h.at) for h in history] fresh = 0 - for f in vd.compare(current, past, today=now.date(), known=known, complete=catalog.codes_complete(item)): + for f in vd.compare(current, past, today=local_today(now), known=known, complete=catalog.codes_complete(item)): after = dict(f.after) if isinstance(f.after, Mapping) else {"value": f.after} if f.revoke_complete: version = await _revoke_complete(session, source.id, table, column) diff --git a/backend/app/data/value_drift.py b/backend/app/data/value_drift.py index a165f995..432d947a 100644 --- a/backend/app/data/value_drift.py +++ b/backend/app/data/value_drift.py @@ -13,9 +13,17 @@ 记 info，列出变化最大的几个取值。 - **断档**（date_stalled）：最新业务日期没前进，且这个日期离今天超过 max(2 × 以前前进的中位间隔, 3 天)；要见过至少一次 前进才知道它多久前进一次。 -- **缺天**（date_gap）：窗口里（有数据的第一天到最后一天之间）有日子没数据，而以前同口径的窗口都没有过。 +- **缺天**（date_gap）：窗口里（有数据的第一天到最后一天之间）有日子没数据，而以前同口径的窗口都没有过。只看到观察当天 + 为止：业务日期有未来值（预约、计划类的日期，或者写错的）时，今天以后的空日子不是「没数据」，那几天本来就还没到；以前 + 那次观察也只看到它自己那天（data.until）为止，不然它窗口里未来的空日子会被当成「以前就有过缺天」，挡住这一次真缺的天。 - **量跌**（volume_drop）：最近一个整天的行数不到它前面 28 天中位数的 30%（前 28 天至少 14 天有数据、中位数至少 20 行）。 - 最后一天和观察当天同一天（还在写入）时往前退一天。只看这一次的窗口，不需要历史。 + 最近一个整天是「有数据的最后一天」和「观察当天的前一天」里早的那个：观察当天还在写入，不是整天；业务日期有未来值时 + 有数据的最后一天在今天以后，也不能拿它退一天（那样会退到今天这个不完整的日子上）。只看这一次的窗口，不需要历史。 +- **断档**遇到未来值：最新业务日期在今天以后时离今天是负数，不会报断档（报不出、也不误报）；观察查询本身不截未来值 + （最新业务日期照实记，目录里也照实给人看）。 + +「今天」由调用方传进来（today）：取值观察按服务端所在时区算（data/catalog_profile_observe.local_today），业务日期是本地 +日期，用 UTC 的日期在本地凌晨会差一天。 - **行数**（rows_drop / rows_jump）：比上一次同口径的少了 30% 以上 / 涨到 3 倍以上；依据是数据库统计的估算值时，要连续 两次都这样（和跌、涨之前那一次比）才报。数到上限、只知道「至少多少行」的不比。 """ @@ -177,7 +185,10 @@ def _series(data: Mapping[str, Any]) -> dict[date, int]: return out -def _holes(days: Mapping[date, int]) -> list[date]: +def _holes(days: Mapping[date, int], until: date | None = None) -> list[date]: + """有数据的第一天到最后一天之间没数据的日子；until 给了时只看到这一天为止（今天以后的空日子不算缺天）。""" + if until is not None: + days = {d: n for d, n in days.items() if d <= until} if not days: return [] first, last = min(days), max(days) @@ -190,15 +201,14 @@ def daily(current: Obs, history: Iterable[Obs], *, today: date) -> list[Finding] same, _note = comparable(current, history) days = _series(current.data) out: list[Finding] = [] - holes = _holes(days) - if holes and same and all(not _holes(_series(h.data)) for h in same): + holes = _holes(days, today) + if holes and same and all(not _holes(_series(h.data), _day(h.data.get("until"))) for h in same): out.append(Finding("date_gap", "warning", before={"gaps": 0}, after={"gaps": len(holes), "days": [d.isoformat() for d in holes[:GAPS_SHOWN]]}, key=tuple(d.isoformat() for d in holes))) if days: - last = max(days) - if last >= today: - last -= timedelta(days=1) + # 最近一个整天：有数据的最后一天和昨天里早的那个（今天还在写入；未来的日子不是整天） + last = min(max(days), today - timedelta(days=1)) base = [days.get(last - timedelta(days=i), 0) for i in range(1, VOLUME_BASELINE_DAYS + 1)] present = sum(1 for n in base if n > 0) median = statistics.median(base) if base else 0 diff --git a/backend/tests/fixtures/dialect/mariadb.sql b/backend/tests/fixtures/dialect/mariadb.sql index b837c0b0..75bdc227 100644 --- a/backend/tests/fixtures/dialect/mariadb.sql +++ b/backend/tests/fixtures/dialect/mariadb.sql @@ -22,7 +22,7 @@ SELECT MIN(CASE WHEN paid_at >= DATE '0001-01-02' AND paid_at < DATE '9999-12-31 -- date_window SELECT status, COUNT(*) AS cnt FROM sales.orders WHERE paid_at >= DATE '2026-08-27' AND paid_at < DATE '9999-12-31' AND status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status LIMIT 21; -- day_volume -SELECT DATE(paid_at) AS d, COUNT(*) AS n FROM sales.orders WHERE paid_at >= DATE '2026-08-27' AND paid_at < DATE '9999-12-31' GROUP BY DATE(paid_at) ORDER BY d; +SELECT DATE(paid_at) AS d, COUNT(*) AS n FROM sales.orders WHERE paid_at >= DATE '2026-08-27' AND paid_at < DATE '2026-10-02' GROUP BY DATE(paid_at) ORDER BY d; -- sample -- （这种方言没有这类查询） -- max_date diff --git a/backend/tests/fixtures/dialect/mysql.sql b/backend/tests/fixtures/dialect/mysql.sql index 17f0f5da..57141703 100644 --- a/backend/tests/fixtures/dialect/mysql.sql +++ b/backend/tests/fixtures/dialect/mysql.sql @@ -22,7 +22,7 @@ SELECT MIN(CASE WHEN paid_at >= DATE '0001-01-02' AND paid_at < DATE '9999-12-31 -- date_window SELECT status, COUNT(*) AS cnt FROM sales.orders WHERE paid_at >= DATE '2026-08-27' AND paid_at < DATE '9999-12-31' AND status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status LIMIT 21; -- day_volume -SELECT DATE(paid_at) AS d, COUNT(*) AS n FROM sales.orders WHERE paid_at >= DATE '2026-08-27' AND paid_at < DATE '9999-12-31' GROUP BY DATE(paid_at) ORDER BY d; +SELECT DATE(paid_at) AS d, COUNT(*) AS n FROM sales.orders WHERE paid_at >= DATE '2026-08-27' AND paid_at < DATE '2026-10-02' GROUP BY DATE(paid_at) ORDER BY d; -- sample -- （这种方言没有这类查询） -- max_date diff --git a/backend/tests/fixtures/dialect/oracle.sql b/backend/tests/fixtures/dialect/oracle.sql index ba09c3d6..d8c1de40 100644 --- a/backend/tests/fixtures/dialect/oracle.sql +++ b/backend/tests/fixtures/dialect/oracle.sql @@ -22,7 +22,7 @@ SELECT MIN(CASE WHEN paid_at >= DATE '0001-01-02' AND paid_at < DATE '9999-12-31 -- date_window SELECT * FROM (SELECT status, COUNT(*) AS cnt FROM sales.orders WHERE paid_at >= DATE '2026-08-27' AND paid_at < DATE '9999-12-31' AND status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status) WHERE ROWNUM <= 21; -- day_volume -SELECT TRUNC(paid_at) AS d, COUNT(*) AS n FROM sales.orders WHERE paid_at >= DATE '2026-08-27' AND paid_at < DATE '9999-12-31' GROUP BY TRUNC(paid_at) ORDER BY d; +SELECT TRUNC(paid_at) AS d, COUNT(*) AS n FROM sales.orders WHERE paid_at >= DATE '2026-08-27' AND paid_at < DATE '2026-10-02' GROUP BY TRUNC(paid_at) ORDER BY d; -- sample SELECT * FROM (SELECT status, COUNT(*) AS cnt FROM sales.orders SAMPLE BLOCK (1) WHERE status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status) WHERE ROWNUM <= 21; -- max_date diff --git a/backend/tests/fixtures/dialect/postgresql.sql b/backend/tests/fixtures/dialect/postgresql.sql index 995eb3f3..0a9b9ca4 100644 --- a/backend/tests/fixtures/dialect/postgresql.sql +++ b/backend/tests/fixtures/dialect/postgresql.sql @@ -22,7 +22,7 @@ SELECT MIN(CASE WHEN paid_at >= DATE '0001-01-02' AND paid_at < DATE '9999-12-31 -- date_window SELECT status, COUNT(*) AS cnt FROM sales.orders WHERE paid_at >= DATE '2026-08-27' AND paid_at < DATE '9999-12-31' AND status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status LIMIT 21; -- day_volume -SELECT CAST(paid_at AS DATE) AS d, COUNT(*) AS n FROM sales.orders WHERE paid_at >= DATE '2026-08-27' AND paid_at < DATE '9999-12-31' GROUP BY CAST(paid_at AS DATE) ORDER BY d; +SELECT CAST(paid_at AS DATE) AS d, COUNT(*) AS n FROM sales.orders WHERE paid_at >= DATE '2026-08-27' AND paid_at < DATE '2026-10-02' GROUP BY CAST(paid_at AS DATE) ORDER BY d; -- sample SELECT status, COUNT(*) AS cnt FROM sales.orders TABLESAMPLE SYSTEM (1) WHERE status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status LIMIT 21; -- max_date diff --git a/backend/tests/fixtures/dialect/sqlite.sql b/backend/tests/fixtures/dialect/sqlite.sql index 44e0a904..1947d6d2 100644 --- a/backend/tests/fixtures/dialect/sqlite.sql +++ b/backend/tests/fixtures/dialect/sqlite.sql @@ -22,7 +22,7 @@ SELECT MIN(CASE WHEN paid_at >= '0001-01-02' AND paid_at < '9999-12-31' THEN pai -- date_window SELECT status, COUNT(*) AS cnt FROM sales.orders WHERE paid_at >= '2026-08-27' AND paid_at < '9999-12-31' AND status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status LIMIT 21; -- day_volume -SELECT substr(paid_at, 1, 10) AS d, COUNT(*) AS n FROM sales.orders WHERE paid_at >= '2026-08-27' AND paid_at < '9999-12-31' GROUP BY substr(paid_at, 1, 10) ORDER BY d; +SELECT substr(paid_at, 1, 10) AS d, COUNT(*) AS n FROM sales.orders WHERE paid_at >= '2026-08-27' AND paid_at < '2026-10-02' GROUP BY substr(paid_at, 1, 10) ORDER BY d; -- sample -- （这种方言没有这类查询） -- max_date diff --git a/backend/tests/test_catalog_observe.py b/backend/tests/test_catalog_observe.py index 68d07455..76abfad0 100644 --- a/backend/tests/test_catalog_observe.py +++ b/backend/tests/test_catalog_observe.py @@ -228,6 +228,49 @@ async def test_large_table_basis_window_or_first_rows_and_no_cross_basis_compare await drop_source(b) +async def test_future_business_dates_stay_out_of_the_daily_window(client, tmp_path): + """业务日期最晚到明天、今天只写了几行：按日行数的查询只数到今天（上界，今天以后的不算），不记量跌。以前窗口只有下界， + 明天那一天也进了窗口，比较器再退一天就退到今天这个不完整的日子上，记出一条量跌（下一轮验收）。""" + path = tmp_path / "ahead.db" + _build(path, rows=0) + db = sqlite3.connect(path) + rows = [(TODAY - timedelta(days=d)).isoformat() for d in range(1, 31) for _ in range(30)] + rows += [TODAY.isoformat()] * 3 + [(TODAY + timedelta(days=1)).isoformat()] * 30 + db.executemany("INSERT INTO orders (status, paid_on, amount, secret) VALUES ('paid', ?, 1, 'x')", [(r,) for r in rows]) + db.commit() + db.close() + sid = await _make(client, path) + try: + job_id = await _observe(sid) + got = {(o.kind, o.column_name): o for o in await _obs(sid)} + days = got[("daily_volume", "paid_on")].data["days"] + assert (TODAY + timedelta(days=1)).isoformat() not in days and days[TODAY.isoformat()] == 3 + assert got[("daily_volume", "paid_on")].data["until"] == TODAY.isoformat() + async with SessionLocal() as session: + drops = list((await session.execute(select(SourceChange).where( + SourceChange.source_id == sid, SourceChange.kind == "volume_drop"))).scalars()) + assert drops == [], "今天还没写完、明天的日子不是整天：不算量跌" + assert all(o.job_id == job_id for o in got.values()) + finally: + await drop_source(sid) + + +def test_observation_today_is_the_server_local_date(monkeypatch): + """观察用的「今天」按服务端所在时区（和待办汇总 notify.digest_at 同一个口径）：业务日期是本地日期，UTC 20 点在东八区 + 已经是第二天。""" + import time + from datetime import datetime, timezone + + monkeypatch.setenv("TZ", "Asia/Shanghai") + time.tzset() + try: + assert observe.local_today(datetime(2026, 10, 17, 20, 0, tzinfo=timezone.utc)) == date(2026, 10, 18) + assert observe.local_today(datetime(2026, 10, 17, 15, 0, tzinfo=timezone.utc)) == date(2026, 10, 17) + finally: + monkeypatch.undo() + time.tzset() + + class _FakeCx: """只看拼出来的 SQL：大表、业务日期没有索引。""" diff --git a/backend/tests/test_catalog_profile_units.py b/backend/tests/test_catalog_profile_units.py index 22933754..7ce3c114 100644 --- a/backend/tests/test_catalog_profile_units.py +++ b/backend/tests/test_catalog_profile_units.py @@ -145,7 +145,7 @@ def test_profile_sql_passes_readonly_guard_on_every_dialect(kind): d.value_counts_sql(tbl, col, 21, scan_cap=100_000), d.min_max_sql(tbl, [(col, False)]), d.min_max_sql(tbl, [(col, True)]), - d.day_volume_sql(tbl, col, "2026-08-01"), + d.day_volume_sql(tbl, col, "2026-08-01", "2026-09-04"), d.window_value_counts_sql(tbl, col, col, "2026-08-01", 21), ] sampled = d.sample_value_counts_sql(tbl, col, 1, 21) diff --git a/backend/tests/test_profile_dialect_golden.py b/backend/tests/test_profile_dialect_golden.py index ebdd546e..ffd9b366 100644 --- a/backend/tests/test_profile_dialect_golden.py +++ b/backend/tests/test_profile_dialect_golden.py @@ -69,7 +69,7 @@ def _queries(kind: str) -> list[tuple[str, str | None]]: (d.column(META["orders"], "biz_day"), True)])), ("date_window", d.window_value_counts_sql(orders, status, d.column(META["orders"], "paid_at"), "2026-08-27", 21)), - ("day_volume", d.day_volume_sql(orders, d.column(META["orders"], "paid_at"), "2026-08-27")), + ("day_volume", d.day_volume_sql(orders, d.column(META["orders"], "paid_at"), "2026-08-27", "2026-10-01")), ("sample", d.sample_value_counts_sql(orders, status, 1, 21)), # 第 3 波：取值观察的最新业务日期（文字存的日期按字符串比） ("max_date", d.max_date_sql(orders, d.column(META["orders"], "paid_at"))), diff --git a/backend/tests/test_value_drift.py b/backend/tests/test_value_drift.py index 6fa80ef9..9d45e446 100644 --- a/backend/tests/test_value_drift.py +++ b/backend/tests/test_value_drift.py @@ -98,6 +98,33 @@ def test_volume_drop_against_the_previous_28_days(): assert vd.daily(partial, [], today=TODAY) == [] +def test_future_business_dates_do_not_turn_today_into_the_last_full_day(): + """业务日期有未来值（最晚到明天）：最近一个整天仍是昨天，今天这个还没写完的日子不算量跌；昨天真跌了照样报。以前取 + max(days) 再退一天，max 在明天时就落到了今天（只有几行）上，记出一条量跌。""" + start = TODAY - timedelta(days=30) # start + 30 是今天 + ahead = _days(start, [100] * 30 + [5, 40]) # 昨天 100、今天 5 行、明天 40 行 + assert not [f for f in vd.daily(ahead, [], today=TODAY) if f.kind == "volume_drop"] + [f] = [f for f in vd.daily(_days(start, [100] * 29 + [20, 5, 40]), [], today=TODAY) if f.kind == "volume_drop"] + assert f.after["day"] == (TODAY - timedelta(days=1)).isoformat() and f.after["rows"] == 20 + + +def test_days_after_today_are_not_gaps_now_or_in_history(): + """缺天只看到今天为止：今天以后两天没数据、第三天又有（预约、计划类的日期）不算缺天；以前那次观察窗口里它自己的「今天」 + 以后的空日子也不算以前有过缺天，挡不住这一次真缺的天。""" + start = TODAY - timedelta(days=30) + full = _days(start - timedelta(days=1), [100] * 31) + ahead = _days(start, [100] * 31 + [0, 0, 3]) # 今天之后空两天、第三天 3 行 + assert not [f for f in vd.daily(ahead, [full], today=TODAY) if f.kind == "date_gap"] + then = TODAY - timedelta(days=7) + old = vd.Obs("daily_volume", "window:paid_on:35", { + "until": then.isoformat(), + "days": {**{(then - timedelta(days=i)).isoformat(): 100 for i in range(30)}, + (then + timedelta(days=5)).isoformat(): 2}}, AT) # 那次的「今天」以后空了几天、又有一行 + holed = _days(start, [100] * 10 + [0, 0] + [100] * 19) + [gap] = [f for f in vd.daily(holed, [old], today=TODAY) if f.kind == "date_gap"] + assert gap.after["gaps"] == 2 + + def _rows(n: int, method: str = "count") -> vd.Obs: return vd.Obs("rows", method, {"rows": n, "method": method}, AT) diff --git a/scripts/dialect_it.py b/scripts/dialect_it.py index 86538338..f5621d42 100644 --- a/scripts/dialect_it.py +++ b/scripts/dialect_it.py @@ -50,6 +50,9 @@ SETTINGS = ProfileSettings(enabled=True, table_max_queries=40, table_max_s=60, q N_ORDERS, N_EVENTS_OLD, N_EVENTS_NEW = 40, 10, 1500 BASE = _dt.datetime(2026, 9, 1, 10, 0, 0) WINDOW_FROM = "2026-09-05" +#: 按日行数的上界（观察当天，含）：WINDOW_UNTIL 在全部夹具日期之后；WINDOW_CUT 落在中间，之后的日子不进窗口 +WINDOW_UNTIL = "2026-09-30" +WINDOW_CUT = "2026-09-07" failures: list[str] = [] @@ -80,6 +83,8 @@ EXPECT = { "pay_way": Counter(o["pay_way"] for o in ORDERS), "paid_at": (min(o["paid_at"] for o in VALID), max(o["paid_at"] for o in VALID)), "day_volume": Counter(o["paid_at"].date().isoformat() for o in IN_WINDOW), + "day_volume_cut": Counter(o["paid_at"].date().isoformat() for o in IN_WINDOW + if o["paid_at"].date().isoformat() <= WINDOW_CUT), "window_status": Counter(o["status"] for o in IN_WINDOW), } @@ -354,9 +359,13 @@ async def run_target(t: Target) -> None: # ---- 取值观察的新方法 tbl = d.table(orders, "orders") paid_col = d.column(orders, _key(orders, "paid_at")) - res = await data_engine.run_query(source, d.day_volume_sql(tbl, paid_col, WINDOW_FROM)) + res = await data_engine.run_query(source, d.day_volume_sql(tbl, paid_col, WINDOW_FROM, WINDOW_UNTIL)) got_days = Counter({_day(row[0]): int(row[1]) for row in res.rows}) check(got_days == EXPECT["day_volume"], f"按日行数和手算一致（无效日期不算进窗口）：{dict(got_days)}") + res = await data_engine.run_query(source, d.day_volume_sql(tbl, paid_col, WINDOW_FROM, WINDOW_CUT)) + got_cut = Counter({_day(row[0]): int(row[1]) for row in res.rows}) + check(got_cut == EXPECT["day_volume_cut"], + f"按日行数只数到观察当天（含当天的时分秒，之后的不算）：{dict(got_cut)}") status_col = d.column(orders, _key(orders, "status")) res = await data_engine.run_query(source, d.window_value_counts_sql(tbl, status_col, paid_col, WINDOW_FROM, 21)) got_window = Counter({str(row[0]).rstrip(" "): int(row[1]) for row in res.rows}) From caeab0fad687e98622068eedd163489c92f35dc7 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Sat, 17 Oct 2026 08:17:12 +0800 Subject: [PATCH 02/15] =?UTF-8?q?SQL=20=E5=B7=AE=E5=BC=82=E8=AF=B4?= =?UTF-8?q?=E6=98=8E=E8=AE=A4=E5=BE=97=E5=90=8C=E4=B8=80=E5=88=97=E4=B8=8A?= =?UTF-8?q?=E7=9A=84=E6=9D=A1=E4=BB=B6=E6=94=B9=E5=86=99=EF=BC=9A=E8=AF=B4?= =?UTF-8?q?=E6=B8=85=E3=80=8C=E8=BF=99=E6=AC=A1=E5=AF=B9=20X=20=E7=9A=84?= =?UTF-8?q?=E6=9D=A1=E4=BB=B6=E6=98=AF=20=E2=80=A6=EF=BC=8C=E5=8F=A3?= =?UTF-8?q?=E5=BE=84=E6=98=AF=20=E2=80=A6=E3=80=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 验收：口径 status = 'paid'、这次 status IN ('paid', 'refunded')，explain_sql_diff 只给兜底「两条 SQL 的写法不同， 未能归纳成具体差异」。原因是差异说明只说四类，同一列上两边各写一套条件（等号对 IN、取值集合不同、范围端点不同） 不在其中：_extra_filters 看到另一边也用了这一列就放弃。 改法：_Explainer 加 _rewrites，排在 _times 之后、_extra_filters 之前。同一张实表的同一列两边 WHERE 里都有条件、 至少一边有没配上的，就说「A 对 X 的条件是 …，B 是 …」；两边都只是取值集合、又有共同的取值时补「（这次多了 'refunded'）」。两边都是时间条件时 aspect 是 time_range（FacetDiff 说明里早有这个方面、engine/compare 也按它 排序），否则是 filter 且 a、b 两边都有——dual_check_facts 对两边都有的 filter 本来就说「两边各有不同的筛选条件」 不写目录，ledger/stats、评测页的方面叫法都已认得这两个名字。 为了不说出不存在的不同，只在这些条件都满足时说：两边用到这一列的条件都只涉及这一列、比的是那一列本身； 按取值逐段算过两边留下的取值确实不同（_Range.compare 加 never_null：表结构里确定不为空的列不拿空值比； x >= 100 AND x >= 50 对 x >= 100 这类冗余写法算出来一样，不说）；不在 LEFT JOIN 可空一侧；WHERE 之外（ON、 HAVING、CASE、聚合参数）都没用到这一列（SUM(CASE WHEN status = 'paid' …) 配上 IN 可能和只筛 paid 一样）；不是 认不准的时间窗口；没换表；有认不出列的条件时整段不说。 只改说明、不碰等价判定：说明只在已经判了不等价之后才跑。新测试把 _rewrites 换成什么都不做，逐对（语料全部 加几对改写、两种先后、给不给表结构）比等价与否和判定的级别，一字不差；IN 只有一个值、IN 列表去重排序照样判等价。 语料执行守门（意思一样的对不许说出具体不同）全过。 测试：test_sql_semantics 新增三组（各方言说两边、可能不影响结果的不说、等价判定不变）；原来断言这两类只给 兜底的用例改成「算出来的值上换了值」（LOWER(status)）仍只给兜底；test_sql_text_diff、test_dual_check_facts 跟上 （同一列上的时间范围现在是 time_range，照样不写目录；换了筛选的值不写目录）。去掉 _rewrites 的调用回跑，新测试 挂 5 条。 双盲复算的前端夹具 dual_check.json 里「说不清的」那个变体原来就是同一列上的时间范围，现在说得出了：换成套着 DATE() 的时间范围（认不出取值范围，仍只给兜底），test_dual_check_fixture 重新生成，check-verify 对应的说法和 高亮那一行的断言跟上。 --- backend/app/data/sql_semantics.py | 138 +++++++++++++++++++--- backend/tests/test_dual_check_facts.py | 16 ++- backend/tests/test_dual_check_fixture.py | 9 +- backend/tests/test_sql_semantics.py | 106 +++++++++++++++-- backend/tests/test_sql_text_diff.py | 4 +- frontend/src/dev/fixtures/dual_check.json | 8 +- scripts/check-verify.mjs | 8 +- 7 files changed, 246 insertions(+), 43 deletions(-) diff --git a/backend/app/data/sql_semantics.py b/backend/app/data/sql_semantics.py index 1ab1848e..8ec28094 100644 --- a/backend/app/data/sql_semantics.py +++ b/backend/app/data/sql_semantics.py @@ -111,7 +111,7 @@ PIPES_AS_CONCAT 的库这两处的意思不同，sqlglot 的 MySQL 读法和这 **具体说法（见 _Explainer）。** 只在两边的规范形式都是「简单形状」时说（_simple_block：整条语句就是一段 SELECT，没有 CTE、子查询、UNION 这类合并、窗口函数、DISTINCT ON；FROM 里只有实表，连接只有内连接、CROSS JOIN、逗号和 LEFT JOIN； LIMIT 是至少为 1 的常数；没有认不准的名字、SELECT 列表里没有 sqlglot 不认识的函数；不认列的那一级里原文没有在 -WHERE / GROUP BY 里用列别名的），而且只说四类： +WHERE / GROUP BY 里用列别名的），而且只说五类： - 时间列（time）：两边各在一张底层表的一列上按时间筛、列不同（另一边哪儿都没用这一列筛选，两边的时间条件都不是恒为 真的，时间列不在 LEFT JOIN 的可空一侧）；一边按时间筛了、另一边哪儿都没有可能是时间窗口的条件的（见下），也说成 @@ -129,6 +129,13 @@ WHERE / GROUP BY 里用列别名的），而且只说四类： 不用到 LEFT JOIN 可空一侧的列（WHERE r.id IS NULL 是反连接，说的是「没关联上」；r.order_id IS NULL 和 ON 里的 r.order_id = o.id 一起已经蕴含 r.id IS NULL）；不是可能的时间窗口（_window_like：date_key BETWEEN 20260901 AND 20260930、stat_month = 202609——认不准是不是时间列，说成有效行过滤会被写进数据目录）。detail 给条件涉及的实表列。 +- 同一列上的条件改写（filter / time_range，a、b 两边都有）：两边 WHERE 里对同一张实表的同一列各写了一套条件、留下的 + 取值不同——等号对 IN（口径 status = 'paid'，这次 status IN ('paid', 'refunded')）、取值集合不同、范围的端点不同，说 + 「A 对 status 的条件是 …，B 是 …」，两边都只是取值集合、又有共同的取值时补「（B 多了 'refunded'）」。要两边用到这一列 + 的条件都只涉及这一列、比的就是那一列本身，按取值逐段算过两边留下的取值确实不同（_Range.compare：表结构里确定不为空 + 的列不拿空值比；x >= 100 AND x >= 50 对 x >= 100 这类冗余写法算出来一样，不说），这一列不在 LEFT JOIN 的可空一侧， + 两边在 WHERE 之外（ON、HAVING、CASE / FILTER、聚合的参数）都没用到这一列，不是认不准的时间窗口，没换表。两边都是 + 时间条件时 aspect 是 time_range。只改说明：走到这里时已经判了不等价（见 _Explainer._rewrites）。 - 聚合与去重（aggregate / group_by）：选出来的列只差在顶层的聚合函数上（同一列做参数，SUM 对 AVG；COUNT(*) 对 SUM(x)） 或者去重上（COUNT / SUM / AVG 去不去重、COUNT(*) 对 COUNT(DISTINCT x)、SELECT DISTINCT 对不去重），分组一样。参数 得是光一列、在每组里不是定死的（不被 WHERE 定成常量、不等于分组键、表结构里它那张表的键没有在组里定死——按 u.id 分组 @@ -144,7 +151,7 @@ WHERE / GROUP BY 里用列别名的），而且只说四类： 每一句都先确认这处不同真会影响结果：两边的 WHERE、HAVING、ON 里有可能恒为假的条件（常量条件，同一个对象上取不到值的 几个条件）时什么都不说；结果看不见行的（SELECT COUNT(*) > -1 这类只出一行、值又和行无关的）不说筛选、关联上的不同； 表结构里的约束（主键、唯一约束、外键、NOT NULL）说明不起作用的不说——约束只拿来少说，不拿来判等价。被 WHERE 用常量定死 -的列出现在排序、分组里的差异，恒为真的 HAVING，都不在四类里，不说。两边给人看的写法一样、和已经说过的一句正好反过来的 +的列出现在排序、分组里的差异，恒为真的 HAVING，都不在五类里，不说。两边给人看的写法一样、和已经说过的一句正好反过来的 不说。 **兜底。** 规范形式不一致、一条具体说法都没有时，给一条 aspect="other" 的「两条 SQL 的写法不同，未能归纳成具体差异， @@ -164,8 +171,9 @@ WHERE / GROUP BY 里用列别名的），而且只说四类： 关联先后、按 WHERE 把外连接改成内连接、换成相等的列；没给表结构（或者类型不一样）时按关联条件相等的两列换着分组、 排序；PostgreSQL、MySQL 没给表结构时去重、分组、求和的一段调换关联先后、并子查询、去掉没用到的 LEFT JOIN、把只取 可空一侧聚合的 LEFT JOIN 写成内连接。 -- 只给兜底的：不是简单形状的（CTE、子查询、UNION、窗口函数，规范化没把它们并掉的）；四类以外的不同（同一列上的时间 - 范围、同一列上换了筛选的值、关联条件、分组、HAVING、排序、取几行、选出来的列、聚合的参数）；几张表、几列交错的条件； +- 只给兜底的：不是简单形状的（CTE、子查询、UNION、窗口函数，规范化没把它们并掉的）；五类以外的不同（算出来的值上的 + 条件改写——LOWER(status) = 'paid' 对 LOWER(status) IN (…)、和别的列交错的同一列条件、关联条件、分组、HAVING、排序、 + 取几行、选出来的列、聚合的参数）；几张表、几列交错的条件； 自己关联自己、同名的表关联了几次的；换了表又有关联条件不同的；条件写在聚合里的一边对条件写在 WHERE 里的一边（SUM(status = 'paid') 对 COUNT(*) … WHERE status = 'paid'：真正的不同是计数为 0 的组、空集时是 0 还是 NULL，说不清）；反连接； 可能是时间窗口、又认不准是不是时间列的条件（业务日期键、统计月份、分区，第几天这类范围）。 @@ -5902,8 +5910,9 @@ class FacetDiff: set_op（几支查询怎么合并）/ group_by（分组键、去重）/ having / limit（取哪几行，含取行前的排序）/ order（只是最终 结果的排序不同，不影响取到哪些行）/ parse（解析不了）/ other（兜底：未能归纳成具体差异，见 explain_sql_diff）。a、b 是两边各自的写法（给人看，没有就是 None），text 是一句中文说明。detail 只在 time 上有：两边的时间列 - [{table, column}]（各是各的表），调用方据此对照数据目录（业务日期）；在 filter 上有 {"columns": [{table, column}]}： - 条件只涉及的实表列，对照有效行过滤用。""" + [{table, column}]（各是各的表），调用方据此对照数据目录（业务日期）；在 filter、time_range 上有 {"columns": + [{table, column}]}：条件只涉及的实表列，对照有效行过滤用。filter 一边是 None 的是「一边多出来的条件」；两边都有的是 + 同一列上的条件两边各写了一套（「A 对 X 的条件是 …，B 是 …」），不是一边多出来的有效行条件。""" aspect: str a: Any @@ -6360,8 +6369,10 @@ class _Range: subject = n.this return self._truth(n, assign[_fingerprint(subject)]) - def compare(self, conds_a: list[Any], conds_b: list[Any], grouped: bool = False) -> bool | None: - """两组条件（各自 AND 连，可以一边为空 = 不筛）留下的取值是不是一样：一样 True，不一样 False，认不出 None。""" + def compare(self, conds_a: list[Any], conds_b: list[Any], grouped: bool = False, + never_null: bool = False) -> bool | None: + """两组条件（各自 AND 连，可以一边为空 = 不筛）留下的取值是不是一样：一样 True，不一样 False，认不出 None。 + never_null：对象一定不为空（表结构里不为空的列），不拿空值比。""" parsed = [self.subject(c) for c in conds_a + conds_b] if any(p is None for p in parsed) or len({p[0] for p in parsed}) != 1: return None @@ -6373,7 +6384,7 @@ class _Range: from sqlglot import exp values = self._pieces(subject, literals, ranged, grouped) - if not isinstance(_unparen(subject), exp.Count): + if not isinstance(_unparen(subject), exp.Count) and not never_null: values.append(None) for v in values: ta = all(self._truth(c, v) == "T" for c in conds_a) @@ -6745,7 +6756,7 @@ def _alias_named(root: Any) -> bool: class _Explainer: """在两边的规范形式上找有把握、又确实影响结果的不同，写成具体的中文句子，同时记下是不是所有不同都说到了。 - 只在两边都是简单形状时说（见 _simple_block），也只说下面四类；别的不同（同一列上的时间范围、同一列上换了筛选的值、 + 只在两边都是简单形状时说（见 _simple_block），也只说下面五类；别的不同（算出来的值上换了筛选的值、 关联条件、分组、HAVING、排序、取几行、选出来的列、聚合的参数……）一律算没说到，交给兜底说明： - time：底层表上按哪一列筛时间不同（两边各在一列上按时间筛，列不同，另一边哪儿都没用这一列筛选，时间列不在 LEFT @@ -6757,6 +6768,8 @@ class _Explainer: 那一列（LOWER(x) <> 'PAID' 这类认不出取值范围、可能恒为真）；按取值逐段算过不是恒为真的（见 _Range.always，一定 不为空的列不算空值，见 _never_null）；不用到 LEFT JOIN 可空一侧的列（反连接，见 _on_nullable_side）；不是认不准 的时间窗口（见 _window_like）。 + - filter / time_range（a、b 两边都有）：同一列上的条件两边各写了一套、留下的取值确实不同（等号对 IN、取值集合不同、 + 范围的端点不同），说「A 对 X 的条件是 …，B 是 …」，见 _rewrites。 - aggregate / group_by：选出来的列只差在顶层的聚合函数（同一列做参数，SUM 对 AVG；COUNT(*) 对 SUM(x)）或者去重上 （COUNT / SUM / AVG 去不去重，COUNT(*) 对 COUNT(DISTINCT x)，SELECT DISTINCT 对不去重），分组一样。参数在组里 定死的不说（被 WHERE / ON 定成常量、等于分组键、本身是常量、它那张表的键在组里定死了，见 _fixed_in_group）；去重 @@ -7312,8 +7325,8 @@ class _Explainer: def conditions(self, A: _Block, B: _Block, mapping: Mapping[str, str], skip_a: set[str], skip_b: set[str]) -> bool: - """WHERE：两边没配上的条件里说得准的两类——按哪一列筛时间不同（_times）、一边多出来的筛选条件（_extra_filters）； - 剩下的算没说到。""" + """WHERE：两边没配上的条件里说得准的三类——按哪一列筛时间不同（_times）、同一列上的条件两边各写了一套（_rewrites）、 + 一边多出来的筛选条件（_extra_filters）；剩下的算没说到。""" xa = [c for c in A.where if self.fp(c) not in skip_a] xb = [c for c in B.where if self.fp(c) not in skip_b] only_a, only_b = self._unmatched(xa, xb, mapping) @@ -7322,6 +7335,7 @@ class _Explainer: if not self.visible: return False only_a, only_b, covered = self._times(A, B, only_a, only_b) + only_a, only_b = self._rewrites(A, B, xa, xb, only_a, only_b, mapping) covered &= self._extra_filters(A, B, xa, only_a, 0) covered &= self._extra_filters(B, A, xb, only_b, 1) return covered @@ -7361,6 +7375,100 @@ class _Explainer: return [c for c in only_a if all(c is not x for x in conds_a)], \ [c for c in only_b if all(c is not x for x in conds_b)], True + def _rewrites(self, A: _Block, B: _Block, xa: list[Any], xb: list[Any], only_a: list[Any], only_b: list[Any], + mapping: Mapping[str, str]) -> tuple[list[Any], list[Any]]: + """同一列上的条件两边各写了一套、留下的取值确实不同：说「A 对 X 的条件是 …，B 是 …」——等号对 IN（口径 + status = 'paid'，这次 status IN ('paid', 'refunded')）、取值集合不同、范围的端点不同。两边都只是「等于常量 / 在 + 常量列表里」、又有共同的取值时补一句各多了哪几个取值。说到了的从两边拿掉，说不准的原样还回去（交给 + _extra_filters，多半是兜底）。 + + 说得准要（每一条都是为了不说出不存在的不同）：两边对这一列都有条件、至少一边有没配上的；两边 WHERE 里用到这一列的 + 条件都只涉及这一列（不和别的列交错）、比的就是那一列本身（不是 LOWER(x)），按取值逐段算过两边留下的取值不一样 + （_Range.compare：表结构里确定不为空的列不拿空值比；认不出、算出来一样的不说——x >= 100 AND x >= 50 对 x >= 100 + 只是冗余的写法）；这一列不在 LEFT JOIN 的可空一侧（那是关联上的事）；两边在 WHERE 之外（ON、HAVING、CASE / FILTER、 + 条件里套着的聚合参数）都没用到这一列（SUM(CASE WHEN status = 'paid' …) 配上 WHERE status IN (…) 可能和只筛 + 'paid' 一样）；不是认不准的时间窗口（_window_like：date_key BETWEEN 20260901 AND 20260930）；没换表。两边的条件 + 都是时间条件（timed）时 aspect 是 time_range，否则是 filter（a、b 两边都有，dual_check_facts 不会把它当成「一边 + 多出来的有效行条件」写进数据目录）。 + + 只改说明，不碰等价判定：走到这里时两边已经判了不等价（见 explain_sql_diff）。""" + if self.swapped or self.usage_a is None or self.usage_b is None: + return only_a, only_b + refs_a = [self.refs(c, A) for c in xa] + refs_b = [self.refs(c, B) for c in xb] + if any(r is None for r in refs_a + refs_b): + # 有认不出用到哪些列的条件：它可能也用到了这一列，说「A 对 X 的条件是 …」就不全 + return only_a, only_b + keys: list[tuple[str, str]] = [] + for c, block in [(c, A) for c in only_a] + [(c, B) for c in only_b]: + found = self.refs(c, block) + if found is not None and len(found) == 1 and next(iter(found)) not in keys: + keys.append(next(iter(found))) + for key in keys: + on_a = [c for c, r in zip(xa, refs_a) if key in r] + on_b = [c for c, r in zip(xb, refs_b) if key in r] + pending_a = [c for c in only_a if any(c is x for x in on_a)] + pending_b = [c for c in only_b if any(c is x for x in on_b)] + if not on_a or not on_b or not (pending_a or pending_b): + continue + if any(r != {key} for r in refs_a if r & {key}) or any(r != {key} for r in refs_b if r & {key}): + continue + if self._on_nullable_side(on_a, A) or self._on_nullable_side(on_b, B): + continue + mapped_b = [self.mapped(c, mapping) for c in on_b] + if not self._plain_subjects(on_a + mapped_b): + continue + times = {self.timed(c, A) for c in on_a} | {self.timed(c, B) for c in on_b} + if times != {key} and any(_window_like(c) for c in on_a + on_b): + continue + kinds = _kinds_for(on_a + on_b) + if self.usage_a.touches({key}, skip=on_a, kinds=kinds) or self.usage_b.touches({key}, skip=on_b, kinds=kinds): + continue + never_null = _not_null_column(self.info, key[0], key[1], self.read) + if self.range.compare(on_a, mapped_b, never_null=never_null) is not False: + continue + shown_a, shown_b = self.show_all(_by_bound(on_a), A), self.show_all(_by_bound(on_b), B) + text = _say(self.la, "对", self.named(*key, qualified=len(A.all_rels()) > 1), "的条件是", shown_a, "，", + self.lb, "是", shown_b) + more = self._members(on_a), self._members(mapped_b) + # 两边都只是取值集合、又有共同的取值时补一句各多了哪几个（两边的取值毫不相干时前一句已经说全了） + if more[0] is not None and more[1] is not None and set(more[0]) & set(more[1]): + extra_a = [shown for value, shown in more[0].items() if value not in more[1]] + extra_b = [shown for value, shown in more[1].items() if value not in more[0]] + parts = [_say(label, "多了", "、".join(extra)) for label, extra in ((self.la, extra_a), + (self.lb, extra_b)) if extra] + if parts: + text = _say(text, "（", "，".join(parts), "）") + self.add("time_range" if times == {key} else "filter", shown_a, shown_b, text, + {"columns": [{"table": key[0], "column": key[1]}]}) + only_a = [c for c in only_a if all(c is not x for x in on_a)] + only_b = [c for c in only_b if all(c is not x for x in on_b)] + return only_a, only_b + + def _members(self, conds: list[Any]) -> dict[Any, str] | None: + """几个条件（AND 连）都是「一列等于常量 / 在常量列表里」时，留下的取值 {比较用的值（见 _Range._norm）: 给人看的 + 写法}；有别的写法（比大小、不等于、NOT、OR、列表里的 NULL）返回 None。""" + from sqlglot import exp + + kept: dict[Any, str] | None = None + for cond in conds: + cond = _unparen(cond) + if isinstance(cond, exp.EQ): + items = [cond.expression] if _literal_value(cond.expression) is not None else [cond.this] + elif isinstance(cond, exp.In) and cond.expressions and not cond.args.get("query") and \ + not cond.args.get("unnest") and not cond.args.get("field"): + items = list(cond.expressions) + else: + return None + values: dict[Any, str] = {} + for item in items: + value = _literal_value(item) + if value is None: + return None + values.setdefault(self.range._norm(value), self.sql(item)) + kept = values if kept is None else {v: shown for v, shown in kept.items() if v in values} + return kept + def _extra_filters(self, block: _Block, other: _Block, where: list[Any], rest: list[Any], side: int) -> bool: """block 这一边多出来的筛选条件（rest 是它没配上的 WHERE 项）。按用到的列连成一组（x > 5 和 x IS NOT NULL 是一组）一组一组看，说得准的说「A 只保留 … 的行，B 没有这个条件」「A 排除了 … 的行，B 没有」「A 按 X 筛时间， @@ -7943,9 +8051,9 @@ def explain_sql_diff(a: str, b: str, dialect: str | None = None, *, schema: Mapp 一元加号、0x10、换了名字的类型名，不知道是哪种库时套着的块注释、紧跟着字的 --、双引号括起来的名字……，见 _lost_in_parsing、_misread）不在语法树上判等价，只给兜底和原文的逐行对照。 - 不等价时怎么说：两边的规范形式都是简单形状时（一段 SELECT、实表、普通的连接，见 _simple_block），只把四类有把握、 - 真会影响结果的不同写成具体的中文句子（按哪一列筛时间、一边多出来的筛选条件、顶层的聚合函数和去重、关联的表和关联 - 类型，见 _Explainer）；具体说法一条都没有时给一条 aspect="other" 的兜底说明（「两条 SQL 的写法不同，未能归纳成具体 + 不等价时怎么说：两边的规范形式都是简单形状时（一段 SELECT、实表、普通的连接，见 _simple_block），只把五类有把握、 + 真会影响结果的不同写成具体的中文句子（按哪一列筛时间、一边多出来的筛选条件、同一列上的条件改写、顶层的聚合函数和 + 去重、关联的表和关联类型，见 _Explainer）；具体说法一条都没有时给一条 aspect="other" 的兜底说明（「两条 SQL 的写法不同，未能归纳成具体 差异，请对照原文」），只覆盖了一部分时追加一条「另有未能归纳的差异」，fallback 为真；text_diff 给出规范化之后不同 的行（没有规范形式可比时给原文的逐行对照）。 diff --git a/backend/tests/test_dual_check_facts.py b/backend/tests/test_dual_check_facts.py index 66fdf79b..c6bff52d 100644 --- a/backend/tests/test_dual_check_facts.py +++ b/backend/tests/test_dual_check_facts.py @@ -8,8 +8,9 @@ 配方里认作日期的列（候选的 upload_dates）：是的算日期列，不是的、读不出配方的说清日期是按文字保存的；数据源还没有 可用的表结构时说表结构不可用 - 差了不止一处、有兜底说明、两条 SQL 规范化后一致、差在别的方面、被采纳的一边没有可记的写法、只有一边按时间筛：不写， - 照实说原因。同一列上的时间范围不同、跨两张表的条件，差异说明只给兜底，同样不写；说明以后真把跨两张表的条件说成一条 - 筛选条件，derive 自己那道「只涉及一张表」的检查也拦得下 + 照实说原因。同一列上的时间范围不同（差异说明说成 time_range）、同一列上换了筛选的值（filter，两边都有条件）不是业务日期、 + 也不是一边多出来的有效记录条件，同样不写；跨两张表的条件差异说明只给兜底，说明以后真把它说成一条筛选条件，derive 自己 + 那道「只涉及一张表」的检查也拦得下 - 指标自己的筛选范围（只看华东、只看线上）不是有效记录条件：取值出现在指标名称、口径说明、所在分组、运行输入里的， 条件用到分组的列的，不是在排除退款、作废、测试这类无效记录的，都不写 - 写的是人工填写、已确认，备注写明运行、节点、指标；同样的说法已确认过就不再写（不升版本），确认过的不同说法不覆盖； @@ -234,10 +235,15 @@ def test_equivalent_sql_or_fallback_or_other_aspects_are_not_recorded(): assert [i["aspect"] for i in joined["diff"]["items"]] == ["aggregate"] assert "不只是去不去重" in derive(joined, SCHEMA)[1] - # 同一列上的时间范围不同：差异说明只给兜底（说不清差的是哪一段），不写 + # 同一列上的时间范围不同：差异说明说成 time_range（「原流程对 paid_date 的条件是 …，盲算是 …」），不是换了时间列，不写 ranged = candidate(RIGHT, RIGHT.replace("'2026-09-07'", "'2026-09-06'"), "b") - assert ranged["diff"]["fallback"] is True and [i["aspect"] for i in ranged["diff"]["items"]] == ["other"] - assert "未能归纳" in derive(ranged, SCHEMA)[1] + assert ranged["diff"]["fallback"] is False and [i["aspect"] for i in ranged["diff"]["items"]] == ["time_range"] + assert "两边差在时间范围" in derive(ranged, SCHEMA)[1] + # 同一列上换了筛选的值：两边都有条件（filter 两边都有），不是一边多出来的有效记录条件，不写 + valued = candidate(RIGHT, RIGHT.replace("status <> 'refunded'", "status NOT IN ('refunded', 'void')"), "b") + assert valued["diff"]["fallback"] is False and [(i["aspect"], i["a"] is not None, i["b"] is not None) + for i in valued["diff"]["items"]] == [("filter", True, True)] + assert "两边各有不同的筛选条件" in derive(valued, SCHEMA)[1] # 只差一个方面、可那个方面对不上目录里的三项（关联方式）：不写，原因说清差在哪 left = (f"SELECT SUM(o.amount) AS gmv FROM orders o LEFT JOIN customers c ON c.id = o.customer_id " diff --git a/backend/tests/test_dual_check_fixture.py b/backend/tests/test_dual_check_fixture.py index cbdea294..f10f100e 100644 --- a/backend/tests/test_dual_check_fixture.py +++ b/backend/tests/test_dual_check_fixture.py @@ -9,12 +9,13 @@ check-verify 在夹具上能过，真实数据上却一处都不高亮（2026-10 差异都没有」，delta_pct 是 (盲算值 − 原流程值) ÷ |原流程值| × 100 保留两位。夹具的数据源当 SQLite 比（方言 sqlite）， 两边的叫法是「原流程」「盲算」，同节点调用时一样。 -**变体。** variants 换掉「销售额」的那一对 SQL，挑的是收尾版差异说明（只说时间列、多出来的筛选、聚合与去重、关联 -四类，别的只给兜底）下各有代表性的几种：按哪一列筛时间不同（time_column，两列都声明为日期类型，预告写业务日期）、 +**变体。** variants 换掉「销售额」的那一对 SQL，挑的是收尾版差异说明（只说时间列、多出来的筛选、同一列上的条件改写、 +聚合与去重、关联五类，别的只给兜底）下各有代表性的几种：按哪一列筛时间不同（time_column，两列都声明为日期类型，预告写业务日期）、 只有一边按时间筛、一边多出来的筛选（排除无效记录的 filter_extra 预告写有效记录条件，指标自己范围的 filter_scope 不写）、 差两个筛选条件、关联方式、只差一个关联条件（join_condition：LEFT JOIN 的 ON 里多一个条件，只给兜底，逐行对照里那一行 -条件前面带着它所在的 JOIN）、聚合只差去重（dedup，预告写去重规则）、说不清的（fallback：同一列上的时间范围不同，只给 -兜底，审批卡默认展开逐行对照）、一边解析失败。 +条件前面带着它所在的 JOIN）、聚合只差去重（dedup，预告写去重规则）、说不清的（fallback：套着 DATE() 的时间范围不同—— +光一列上的范围不同如今说得出「对 paid_at 的条件是 …」，套着函数的认不出取值范围，只给兜底，审批卡默认展开逐行对照）、 +一边解析失败。 **写目录的预告。** 服务端在摘要里另带每个指标的 catalog（采用原流程、采用盲算各会往数据目录里写什么， dual_check_facts.preview，约定之外另加的），审批卡照它说；老服务端没有时卡上按差异的形状有条件地说，两种说法都按 diff --git a/backend/tests/test_sql_semantics.py b/backend/tests/test_sql_semantics.py index 7a9bbf93..0b9f35cf 100644 --- a/backend/tests/test_sql_semantics.py +++ b/backend/tests/test_sql_semantics.py @@ -616,10 +616,10 @@ def test_specific_differences_are_said_in_one_sentence_each(a, b, expected): @pytest.mark.parametrize("a,b", [ - # 同一列上的时间范围不同、同一列上换了筛选的值：不在能说的四类里 - (BASE, "SELECT SUM(amount) FROM orders WHERE paid_at BETWEEN '2026-09-01' AND '2026-09-30'"), - ("SELECT SUM(amount) FROM orders WHERE status = 'paid'", - "SELECT SUM(amount) FROM orders WHERE status IN ('paid', 'shipped')"), + # 算出来的值上换了筛选的值：认不出取值范围，不在能说的五类里（光一列上的条件改写见 + # test_rewritten_conditions_on_one_column_say_both_sides） + ("SELECT SUM(amount) FROM orders WHERE LOWER(status) = 'paid'", + "SELECT SUM(amount) FROM orders WHERE LOWER(status) IN ('paid', 'shipped')"), # 聚合的参数不同、选出来的列换了先后 ("SELECT SUM(paid_amount) FROM orders", "SELECT SUM(amount) FROM orders"), ("SELECT SUM(o.amount) FROM orders o JOIN refunds r ON r.order_id = o.id", @@ -653,8 +653,8 @@ def test_specific_differences_are_said_in_one_sentence_each(a, b, expected): "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT COUNT(*) FROM t WHERE total > 100"), ], ids=lambda v: None) def test_differences_outside_the_four_kinds_get_only_the_fallback(a, b): - """具体说法只说四类（按哪一列筛时间、一边多出来的筛选条件、顶层的聚合函数和去重、关联的表和关联类型），而且只在 - 两边都是简单形状时说。别的不同（同一列上的范围、换了筛选的值、聚合的参数、分组、HAVING、取行、排序、关联条件、 + """具体说法只说五类（按哪一列筛时间、一边多出来的筛选条件、同一列上的条件改写、顶层的聚合函数和去重、关联的表和 + 关联类型），而且只在两边都是简单形状时说。别的不同（算出来的值上换了筛选的值、聚合的参数、分组、HAVING、取行、排序、关联条件、 UNION、CTE 和子查询里的不同）以前说得出具体的，复验在这些地方一再抓到不存在的差异（被 WHERE 定死的排序键、恒为真的 HAVING、外层不在乎的子查询差异……），现在一律只给兜底说明，附规范化后不同的行。""" for dialect in DIALECTS: @@ -663,6 +663,94 @@ def test_differences_outside_the_four_kinds_get_only_the_fallback(a, b): assert not diff.equivalent and diff.items == [FacetDiff("other", None, None, FALLBACK)], (dialect, diff.items) +@pytest.mark.parametrize("a,b,aspect,text", [ + # 等号对 IN：口径 status = 'paid'，这次多收了 refunded（账本的叫法：A 是这次、B 是口径） + ("SELECT SUM(amount) FROM orders WHERE status IN ('paid', 'refunded')", + "SELECT SUM(amount) FROM orders WHERE status = 'paid'", "filter", + "这次对 status 的条件是 status IN ('paid', 'refunded')，口径是 status = 'paid'（这次多了 'refunded'）"), + # 取值集合不同：各多了几个 + ("SELECT SUM(o.amount) FROM orders o JOIN stores s ON s.id = o.store_id WHERE o.status IN ('paid', 'open', 'done')", + "SELECT SUM(o.amount) FROM orders o JOIN stores s ON s.id = o.store_id WHERE o.status IN ('paid', 'void')", "filter", + "这次对 orders.status 的条件是 orders.status IN ('done', 'open', 'paid')，口径是 orders.status IN ('paid', 'void')" + "（这次多了 'done'、'open'，口径多了 'void'）"), + # 取值毫不相干：前一句已经说全了，不补 + ("SELECT COUNT(*) FROM orders WHERE status = 'paid'", "SELECT COUNT(*) FROM orders WHERE status = 'done'", "filter", + "这次对 status 的条件是 status = 'paid'，口径是 status = 'done'"), + # 范围的端点不同 + ("SELECT COUNT(*) FROM orders WHERE amount >= 100", "SELECT COUNT(*) FROM orders WHERE amount > 100", "filter", + "这次对 amount 的条件是 amount >= 100，口径是 amount > 100"), + # 同一时间列上的范围不同：time_range + (BASE, "SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-01' AND paid_at < '2026-11-01'", "time_range", + "这次对 paid_at 的条件是 paid_at >= '2026-09-01' AND paid_at < '2026-10-01'，" + "口径是 paid_at >= '2026-09-01' AND paid_at < '2026-11-01'"), +], ids=lambda v: None) +def test_rewritten_conditions_on_one_column_say_both_sides(a, b, aspect, text): + """同一列上的条件两边各写了一套、留下的取值不同（下一轮验收：口径 status = 'paid'，这次 status IN ('paid', + 'refunded') 以前只给兜底）：说「这次对 X 的条件是 …，口径是 …」，aspect 两边都有（dual_check_facts 不会把它当成一边 + 多出来的有效行条件）。各方言、给不给表结构都一样。""" + for dialect in DIALECTS: + for schema in (None, sql_pairs.schema_cache()): + diff = explain_sql_diff(a, b, dialect, schema=schema, labels=("这次", "口径")) + check_shape(diff) + assert not diff.equivalent and not diff.fallback, (dialect, diff.items) + [item] = diff.items + assert (item.aspect, item.text) == (aspect, text), (dialect, item) + assert item.a is not None and item.b is not None + assert item.detail == {"columns": [{"table": "orders", "column": text.split("对 ")[1].split(" ")[0] + .split(".")[-1]}]} + + +@pytest.mark.parametrize("a,b,schema", [ + # 冗余的写法：算出来留下的取值一样，不说 + ("SELECT COUNT(*) FROM orders WHERE amount >= 100 AND amount >= 50", "SELECT COUNT(*) FROM orders WHERE amount >= 100", + None), + # 主键不为空：差的只是空值那一段，不说（没给表结构时照说，那时不知道它不为空） + ("SELECT COUNT(*) FROM orders WHERE id = 1 OR id IS NULL", "SELECT COUNT(*) FROM orders WHERE id = 1", "schema"), + # WHERE 之外还用到了这一列：SUM(CASE WHEN status = 'paid' …) 配上 IN (…) 和只筛 'paid' 结果一样 + ("SELECT SUM(CASE WHEN status = 'paid' THEN amount END) FROM orders WHERE status IN ('paid', 'refunded')", + "SELECT SUM(amount) FROM orders WHERE status = 'paid'", None), + # 和别的列交错的条件 + ("SELECT COUNT(*) FROM orders WHERE status = 'paid' OR amount > 100", + "SELECT COUNT(*) FROM orders WHERE status IN ('paid', 'done') OR amount > 100", None), +], ids=lambda v: None) +def test_rewritten_conditions_that_may_not_change_the_result_are_not_said(a, b, schema): + for dialect in DIALECTS: + diff = explain_sql_diff(a, b, dialect, schema=sql_pairs.schema_cache() if schema else None) + check_shape(diff) + assert not diff.equivalent and not any("的条件是" in d.text for d in diff.items), (dialect, diff.items) + + +def test_explaining_rewritten_conditions_never_changes_the_equivalence_verdict(monkeypatch): + """同一列上的条件改写只进说明：等价与否、在哪一级规范形式上判的，和不说这一类时一字不差（语料各对、两种先后、给不给 + 表结构）；规范化本来就认得的改写（IN 只有一个值、IN 列表去重排序）照样判等价、没有条目。""" + from app.data import sql_semantics + + pairs = [(p.a, p.b) for p in sql_pairs.PAIRS] + [ + ("SELECT SUM(amount) FROM orders WHERE status IN ('paid')", "SELECT SUM(amount) FROM orders WHERE status = 'paid'"), + ("SELECT COUNT(*) FROM orders WHERE status IN ('void', 'paid', 'paid')", + "SELECT COUNT(*) FROM orders WHERE status IN ('paid', 'void')"), + ("SELECT SUM(amount) FROM orders WHERE status IN ('paid', 'refunded')", + "SELECT SUM(amount) FROM orders WHERE status = 'paid'"), + ] + schema = sql_pairs.schema_cache() + + def verdicts() -> list[tuple[bool, str | None]]: + out = [] + for a, b in pairs: + for x, y in ((a, b), (b, a)): + for sc in (None, schema): + diff = explain_sql_diff(x, y, "sqlite", schema=sc) + out.append((diff.equivalent, diff.level)) + return out + + with_rewrites = verdicts() + monkeypatch.setattr(sql_semantics._Explainer, "_rewrites", lambda self, A, B, xa, xb, oa, ob, m: (oa, ob)) + assert verdicts() == with_rewrites + for a, b in pairs[-3:-1]: + diff = explain_sql_diff(a, b, "sqlite") + assert diff.equivalent and diff.items == [], (a, b) + + def test_a_difference_inside_a_cte_used_only_in_a_condition_subquery_is_never_missed(): """第 5 轮回退的例子：只在 IN 子查询里用到的 CTE 里的条件不同，以前整个漏报（返回空列表）。现在条件里有子查询的不是 简单形状，不说具体的，但一定不判等价、给兜底说明和规范化后不同的行。EXISTS、标量子查询一样。""" @@ -754,9 +842,9 @@ def test_nothing_specific_gives_one_fallback_item_and_the_lines_that_differ(): def test_specific_items_that_cover_only_part_end_with_the_rest_note(): - """按哪一列筛时间说得准，同一列 status 上换了筛选的值说不了：说时间那一句，补一句「另有未能归纳的差异」。""" - a = "SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-01' AND status = 'paid'" - b = "SELECT SUM(amount) FROM orders WHERE created_at >= '2026-09-01' AND status IN ('paid', 'done')" + """按哪一列筛时间说得准，LOWER(status) 上换了筛选的值说不了：说时间那一句，补一句「另有未能归纳的差异」。""" + a = "SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-01' AND LOWER(status) = 'paid'" + b = "SELECT SUM(amount) FROM orders WHERE created_at >= '2026-09-01' AND LOWER(status) IN ('paid', 'done')" diff = explain_sql_diff(a, b, labels=LABELS) check_shape(diff) assert diff.fallback and [d.text for d in diff.items] == ["原流程按 paid_at 筛时间，盲算按 created_at", FALLBACK_REST] diff --git a/backend/tests/test_sql_text_diff.py b/backend/tests/test_sql_text_diff.py index f49f670d..87a9680d 100644 --- a/backend/tests/test_sql_text_diff.py +++ b/backend/tests/test_sql_text_diff.py @@ -73,11 +73,11 @@ def test_an_extra_condition_sorted_first_does_not_drag_the_shared_ones_along(): def test_two_conditions_on_each_side_mark_only_the_one_that_differs(): - """同一列上的区间只差结束日期（这种只给兜底说明，审核人全靠逐行对照）：开始日期那一行两边一样，不标。""" + """同一列上的区间只差结束日期（差异说明说成 time_range，逐行对照照样给）：开始日期那一行两边一样，不标。""" a = f"SELECT SUM(fee) FROM parcels WHERE {WEEK}" b = "SELECT SUM(fee) FROM parcels WHERE sent_on BETWEEN '2031-03-03' AND '2031-03-10'" got = diff(a, b) - assert got.fallback + assert not got.fallback and [i.aspect for i in got.items] == ["time_range"] assert got.text_diff == (" WHERE", "- AND parcels.sent_on <= '2031-03-09'", "+ AND parcels.sent_on <= '2031-03-10'") diff --git a/frontend/src/dev/fixtures/dual_check.json b/frontend/src/dev/fixtures/dual_check.json index 6024462e..e2003b81 100644 --- a/frontend/src/dev/fixtures/dual_check.json +++ b/frontend/src/dev/fixtures/dual_check.json @@ -843,8 +843,8 @@ }, "fallback": { "sql": [ - "SELECT SUM(amount) AS gmv\nFROM orders\nWHERE paid_at >= '2026-09-01'\n AND paid_at < '2026-09-08'", - "SELECT SUM(amount) AS gmv\nFROM orders\nWHERE paid_at >= '2026-09-01'\n AND paid_at < '2026-09-09'" + "SELECT SUM(amount) AS gmv\nFROM orders\nWHERE paid_at >= '2026-09-01'\n AND DATE(paid_at) < '2026-09-08'", + "SELECT SUM(amount) AS gmv\nFROM orders\nWHERE paid_at >= '2026-09-01'\n AND DATE(paid_at) < '2026-09-09'" ], "diff": { "equivalent": false, @@ -857,8 +857,8 @@ "fallback": true, "text_diff": [ " WHERE", - "- AND orders.paid_at < '2026-09-08'", - "+ AND orders.paid_at < '2026-09-09'" + "- AND DATE(orders.paid_at) < '2026-09-08'", + "+ AND DATE(orders.paid_at) < '2026-09-09'" ] }, "catalog": { diff --git a/scripts/check-verify.mjs b/scripts/check-verify.mjs index 0ae030a4..59e96516 100644 --- a/scripts/check-verify.mjs +++ b/scripts/check-verify.mjs @@ -450,9 +450,9 @@ await section('审批卡：没有预告时，「记入数据目录」只说必 // 而避开本来对的一边。变体的 catalog 是服务端对这一对 SQL 的真实预告，按它核对 const { CATALOG_TABLE_FIELD_LABEL: LABEL } = await import(await load('/src/lib/terms.ts')) // [变体, 说明, 服务端不写的那一种卡上照实说的那句（只在卡上提数据目录时有）]。变体挑的是收尾版差异说明下各有代表性 - // 的几种（backend/tests/test_dual_check_fixture.py 的模块说明）：同一列上的时间范围不同只给兜底（fallback） + // 的几种（backend/tests/test_dual_check_fixture.py 的模块说明）：套着 DATE() 的时间范围不同只给兜底（fallback） const cases = [ - ['fallback', '说不清的（同一列上的时间范围不同，只给兜底）'], + ['fallback', '说不清的（套着 DATE() 的时间范围不同，只给兜底）'], ['two_filters', '差两个筛选条件'], ['join', '差在关联方式'], ['parse', '一边解析失败'], @@ -492,7 +492,7 @@ await section('审批卡：没有预告时，「记入数据目录」只说必 && note.includes('按不同的列筛时间') && note.includes('采用的一边所用的时间列'), note) } if (name === 'fallback') { - // 收尾版差异说明归纳不了同一列上的时间范围：卡上写那句固定的兜底说明，默认展开对照，审核人逐行对照两条 SQL + // 收尾版差异说明归纳不了套着函数的时间范围：卡上写那句固定的兜底说明，默认展开对照，审核人逐行对照两条 SQL check('……写那句固定的兜底说明，不当成一条具体差异', (await text(gmv.locator('[data-dual-fallback]'))) === '两条 SQL 的写法不同，未能归纳成具体差异' && await gmv.locator('[data-dual-diff-item]').count() === 0, await text(gmv.locator('[data-dual-fallback]'))) check('……默认展开对照', (await gmv.locator('[data-dual-compare-toggle]').getAttribute('aria-expanded')) === 'true') @@ -532,7 +532,7 @@ await section('审批卡：没有预告时，「记入数据目录」只说必 const marked = [...m.normDel, ...m.normAdd] check('……两边相同的条件（开始日期）不标，只标不同的那一行', m.normDel.length === 1 && m.normAdd.length === 1 && !marked.some((l) => l.includes(">= '2026-09-01'")) - && marked.every((l) => l.includes("orders.paid_at < '2026-09-")), marked.join(' ‖ ')) + && marked.every((l) => l.includes("DATE(orders.paid_at) < '2026-09-")), marked.join(' ‖ ')) } if (name === 'two_filters') { check('……一边多出的两个条件各占一行', m.normDel.includes("orders.channel = 'web'") From 7db976012134db19b63a6a1c691335f48db79a4f Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Sat, 17 Oct 2026 08:24:15 +0800 Subject: [PATCH 03/15] =?UTF-8?q?=E7=94=A8=E9=87=8F=E8=AE=A1=E9=87=8F?= =?UTF-8?q?=E6=8B=BF=E5=88=B0=E8=B0=83=E7=94=A8=E6=97=B6=E5=AE=9E=E9=99=85?= =?UTF-8?q?=E7=94=A8=E7=9A=84=E6=A8=A1=E5=9E=8B=E5=90=8D=EF=BC=9A=E8=AF=84?= =?UTF-8?q?=E6=B5=8B=E6=B1=87=E6=80=BB=E9=87=8C=E5=8A=A9=E6=89=8B=E7=94=A8?= =?UTF-8?q?=E9=87=8F=E4=B8=8D=E5=86=8D=E8=AE=B0=E6=88=90=20unknown?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 验收：评测汇总里助手用量的模型是 "unknown"，写作用量的模型却是对的。写作用量来自运行的 llm.end（节点配置里 就有模型名）；助手用量来自 providers/usage.Meter，它只从回调的 invocation_params 取 model / model_name，而演示 模型 MockChatModel 没有 _identifying_params，调用参数里只有 _type 和 stop，于是落到 UNKNOWN_MODEL。 改法两处： - Meter：调用参数里取不到时，看 LangChain 照模型对象给回调的 metadata 里的 ls_model_name（调用时传的 model， 否则对象上的 model / model_name），再取不到才看回复、才记 unknown。不认得调用参数的模型类也记得对。 - MockChatModel 补上 _identifying_params（model_name），和 OpenAI 兼容、Anthropic 一样在调用参数里报模型名。 测试（test_usage_meter）：调用参数里不报模型名的模型、演示模型都按对象上的名字记；三种接入（OpenAI 兼容、 Anthropic、演示模型）由 factory 建出来，照 LangChain 回调时给的参数喂给计量，记的都是接入里配的模型 id， 调用参数里本身也带着；评测作答（eval/answer.Driver）走演示模型时助手用量记在 mock-fast 名下。两处修复都 去掉回跑挂 3 条，只留计量那处时挂「调用参数里本身带着」那一条。 --- backend/app/providers/mock_model.py | 6 +++ backend/app/providers/usage.py | 10 +++- backend/tests/test_usage_meter.py | 79 +++++++++++++++++++++++++++++ 3 files changed, 93 insertions(+), 2 deletions(-) diff --git a/backend/app/providers/mock_model.py b/backend/app/providers/mock_model.py index 0ad18dd2..77870282 100644 --- a/backend/app/providers/mock_model.py +++ b/backend/app/providers/mock_model.py @@ -126,6 +126,12 @@ class MockChatModel(BaseChatModel): def _llm_type(self) -> str: return "mock" + @property + def _identifying_params(self) -> dict[str, Any]: + """调用参数里带上模型名（同 OpenAI 兼容、Anthropic 的 model）：用量计量（providers/usage）按它记在哪个模型名下， + 不报的话评测汇总里助手用量的模型就成了 unknown。""" + return {"model_name": self.model_name} + def bind_tools(self, tools: Sequence[Any], **kwargs: Any) -> "MockChatModel": normalized: list[dict[str, Any]] = [] for tool in tools: diff --git a/backend/app/providers/usage.py b/backend/app/providers/usage.py index 9404039a..56298a2c 100644 --- a/backend/app/providers/usage.py +++ b/backend/app/providers/usage.py @@ -18,8 +18,10 @@ async generator：with 块跨着 yield 的话，变量在两次 yield 之间留 流末尾那一段用量收得到；只有模型在那一行之后还往下写超过 DRAIN_CHARS 字的闲话时才当场关流（不为闲话付钱），连接断了也一样， 这一次只拿得到已经报了的部分：照样计次数、计已报的 token，记成估不出，不悄悄少算。 -模型名按发起调用时的参数取（invocation_params 的 model / model_name，就是接入里配的模型 id，价格目录按它查），取不到再 -看回复里的 model_name。 +模型名按发起调用时的参数取（invocation_params 的 model / model_name，就是接入里配的模型 id，价格目录按它查）；调用参数里 +没有的（以前的演示模型就没报，评测汇总里助手用量的模型因此记成 unknown），看 LangChain 照模型对象给回调的 metadata 里的 +ls_model_name；再取不到看回复里的 model_name，都没有才记 UNKNOWN_MODEL。OpenAI 兼容、Anthropic 的模型在调用参数里就带着 +model，演示模型（providers/mock_model）也把 model_name 报进调用参数。 """ from __future__ import annotations @@ -54,6 +56,10 @@ class Meter(BaseCallbackHandler): def _started(self, run_id: UUID, serialized: dict[str, Any] | None, kwargs: dict[str, Any]) -> None: params = kwargs.get("invocation_params") or {} model = params.get("model") or params.get("model_name") or params.get("model_id") + if not model: + # LangChain 照模型对象给回调的 metadata 里带着 ls_model_name（调用时传的 model，否则对象上的 model / + # model_name）：调用参数里不报模型名的模型类也认得出 + model = (kwargs.get("metadata") or {}).get("ls_model_name") if not model and isinstance(serialized, dict): model = (serialized.get("kwargs") or {}).get("model") or (serialized.get("kwargs") or {}).get("model_name") with self._lock: diff --git a/backend/tests/test_usage_meter.py b/backend/tests/test_usage_meter.py index ed0c35b5..828a8130 100644 --- a/backend/tests/test_usage_meter.py +++ b/backend/tests/test_usage_meter.py @@ -105,6 +105,85 @@ async def test_metering_counts_by_model_and_prices_from_the_catalog(): assert got["cost_usd"] == pytest.approx(estimate_cost(PRICED, 2000, 400)) +class Bare(Scripted): + """不报调用参数的模型（同以前的演示模型）：invocation_params 里没有 model，只在对象上有。""" + + @property + def _identifying_params(self) -> dict[str, Any]: + return {} + + +async def test_the_model_name_comes_from_the_model_object_when_the_call_parameters_lack_it(): + """评测汇总里助手用量的模型以前记成 unknown（下一轮验收）：演示模型的调用参数里没有模型名，计量也就认不出。现在 + 调用参数里没有时看 LangChain 照模型对象给回调的 ls_model_name；演示模型自己也把 model_name 报进调用参数。""" + from app.providers.mock_model import MockChatModel + + m = usage.meter() + with usage.metering(m): + await Bare(model=PRICED + "-bare").ainvoke("你好") + await MockChatModel(model_name="mock-wide", latency=0).ainvoke("你好") + got = m.summary()["by_model"] + assert set(got) == {PRICED + "-bare", "mock-wide"}, got + assert usage.UNKNOWN_MODEL not in got + + +@pytest.mark.parametrize("build", ["openai", "anthropic", "mock"]) +def test_the_call_parameters_of_each_provider_kind_carry_the_configured_model(build): + """三种接入（OpenAI 兼容、Anthropic、演示模型）由 providers/factory 建出来的模型：LangChain 回调时给的调用参数和 + ls_model_name（这里照它的写法直接取，不发请求）计量都认得出，记的就是接入里配的模型 id。""" + import uuid as _uuid + from types import SimpleNamespace + + from langchain_core.outputs import LLMResult + + from app.providers.factory import ModelSpec, build_chat_model + + provider = SimpleNamespace(kind=build, name="虚构接入", api_key=None, base_url=None, extra={}, + default_model=None) + model_id = {"openai": "gpt-made-up", "anthropic": "claude-made-up", "mock": "mock-wide"}[build] + import os + env = {"openai": "OPENAI_API_KEY", "anthropic": "ANTHROPIC_API_KEY"}.get(build) + old = os.environ.get(env) if env else None + if env: + os.environ[env] = "sk-made-up" + try: + model = build_chat_model(provider, ModelSpec(model=model_id)) + finally: + if env: + if old is None: + os.environ.pop(env, None) + else: + os.environ[env] = old + m = usage.meter() + run_id = _uuid.uuid4() + m.on_chat_model_start({}, [], run_id=run_id, invocation_params=model._get_invocation_params(), + metadata=dict(model._get_ls_params())) + m.on_llm_end(LLMResult(generations=[[ChatGeneration(message=AIMessage(content="好"))]]), run_id=run_id) + assert list(m.summary()["by_model"]) == [model_id] + # 调用参数里本身就带着（不靠 ls_model_name 兜着） + params = model._get_invocation_params() + assert model_id in (params.get("model"), params.get("model_name")), params + + +async def test_the_eval_driver_on_the_demo_model_records_its_name(monkeypatch): + """评测作答走演示模型时，助手用量记在演示模型的名下，不是 unknown（下一轮验收：写作用量的模型对、助手的记成 unknown）。""" + import app.api.copilot as copilot + from app.eval import answer + from app.providers.mock_model import MockChatModel + + async def _model(*_a, **_k): + return MockChatModel(model_name="mock-fast", latency=0), "mock-fast" + + monkeypatch.setattr(copilot, "get_chat_model", _model) + driver = answer.Driver("按天看销售额", datasource_ids=[], ctx=answer.TurnContext( + suite_id="s", suite_name="虚构题集", eval_run_id="e", case_id=None, case_seq=1, actor=None)) + ops, refused = await driver._generate("按天看销售额", None) + assert refused is None + driver._take(ops) + got = driver.turn.usage["assistant"]["by_model"] + assert set(got) == {"mock-fast"} and got["mock-fast"]["calls"] >= 1, got + + async def test_outside_the_meter_nothing_is_counted(): m = usage.meter() with usage.metering(m): From c64c7a106249d94897bb4f106ab75a4b2846515b Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Sat, 17 Oct 2026 08:28:54 +0800 Subject: [PATCH 04/15] =?UTF-8?q?=E7=9B=AE=E5=BD=95=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=E5=AF=B9=E6=8E=A2=E6=9F=A5=E8=8C=83=E5=9B=B4=E5=A4=96=E7=9A=84?= =?UTF-8?q?=E8=A1=A8=E7=85=A7=E5=AE=9E=E8=AF=B4=E3=80=8C=E5=9C=A8=E6=8E=A2?= =?UTF-8?q?=E6=9F=A5=E8=8C=83=E5=9B=B4=E5=A4=96=E3=80=8D=EF=BC=8C=E4=B8=8D?= =?UTF-8?q?=E5=86=8D=E8=AF=B4=E3=80=8C=E5=8F=AF=E8=83=BD=E5=B7=B2=E8=A2=AB?= =?UTF-8?q?=E5=88=A0=E9=99=A4=E6=88=96=E5=B0=9A=E6=9C=AA=E6=8E=A2=E6=9F=A5?= =?UTF-8?q?=E7=BB=93=E6=9E=84=E3=80=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 验收：GET /api/datasources/{id}/catalog/{表} 对「对象索引里有、但在探查范围外」的表回 404「数据源中没有表 …，可能 已被删除或尚未探查结构」，而 /schema 和 /catalog/search 都说对了。_table_key 一律写死这一句，没看 absent_reasons。 改法：source_objects 加 ABSENT_MESSAGE 和 absent_message()，给人看的五种「不在」（表已不在、全库对象清单里也没有、 在探查范围外、在范围里却读不到结构、老缓存说不清）一句一句，和界面的 CATALOG_ABSENT_TEXT、/schema 用的 schema_view.absent_detail 同一套原因。上传表格、记录表按版本走，没有对象索引又没截断过的老缓存认不出名字是不是 从来就没有过：这两种只说「数据源中没有表 …」，不说「已删除」。用到的三处都换成它：_table_key（表详情、审阅、 使用记录等十来个接口共用）、表助手开始前的 404、批量审阅逐条的 table_missing 错误。 测试：test_catalog_schema_truncated 新增一条（205 张表、规则只收 200 张的合成库）：范围外的表在表详情、表助手、 批量审阅里都说「探查范围外」，对象索引里也没有的说「没有表 …」。去掉修复回跑挂。 --- backend/app/api/catalog.py | 13 ++++++-- backend/app/data/catalog_review.py | 4 +-- backend/app/data/source_objects.py | 30 ++++++++++++++++++- .../tests/test_catalog_schema_truncated.py | 21 +++++++++++++ 4 files changed, 62 insertions(+), 6 deletions(-) diff --git a/backend/app/api/catalog.py b/backend/app/api/catalog.py index f5a85a6b..0526ca13 100644 --- a/backend/app/api/catalog.py +++ b/backend/app/api/catalog.py @@ -226,14 +226,21 @@ async def _detail(session: AsyncSession, source: Any, table: str, entry: catalog async def _table_key(session: AsyncSession, source: Any, table: str) -> tuple[str, catalog.CatalogEntry | None]: - """路径里的表名 → 目录的键。表结构里有的按表结构的写法；表结构里没有、但还留着目录的也认（便于清理）。""" + """路径里的表名 → 目录的键。表结构里有的按表结构的写法；表结构里没有、但还留着目录的也认（便于清理）。都没有时 404， + 说清是哪一种「不在」（_missing）。""" key = catalog.resolve_table_name(source.schema_cache, table) entry = await catalog.read_entry(session, source.id, key or table) if key is None and entry is None: - raise HTTPException(404, f"数据源中没有表 {table}，可能已被删除或尚未探查结构") + raise await _missing(session, source, table) return key or table, entry +async def _missing(session: AsyncSession, source: Any, table: str) -> HTTPException: + """表结构里没有这张表的 404：表已不在、数据源里没有、在探查范围外、读不到结构、说不清，和 /schema、/catalog/search、 + 目录清单的 absent_reason 同一套（source_objects.absent_message）。""" + return HTTPException(404, await source_objects.absent_message(session, source, table)) + + def _conflict(e: catalog.CatalogConflict) -> HTTPException: return HTTPException(409, str(e)) @@ -906,7 +913,7 @@ async def catalog_table_assistant(source_id: str, table: str, payload: Assistant row, source = await _resolved(session, source_id) key = catalog.resolve_table_name(source.schema_cache, table) if key is None: - raise HTTPException(404, f"数据源中没有表 {table}，可能已被删除或尚未探查结构") + raise await _missing(session, source, table) messages = [m.model_dump() for m in payload.messages] if problem := catalog_assistant.conversation_problem(messages): raise HTTPException(422, problem) diff --git a/backend/app/data/catalog_review.py b/backend/app/data/catalog_review.py index 43e27594..78eb3438 100644 --- a/backend/app/data/catalog_review.py +++ b/backend/app/data/catalog_review.py @@ -41,7 +41,7 @@ from typing import Any from sqlalchemy import func, select from sqlalchemy.ext.asyncio import AsyncSession -from app.data import catalog +from app.data import catalog, source_objects from app.db.models import Run #: 队列的筛选：全部、影响数字的项、名称和说明 @@ -346,7 +346,7 @@ async def review_batch(session: AsyncSession, source: Any, items: Iterable[Batch key = known[item.table_name] if key is None: results[i] = _result(item, ok=False, code=CODE_TABLE_MISSING, - error=f"数据源中没有表 {item.table_name}，可能已被删除或尚未探查结构") + error=await source_objects.absent_message(session, source, item.table_name)) continue groups.setdefault(key, []).append(i) at = catalog.now_iso() diff --git a/backend/app/data/source_objects.py b/backend/app/data/source_objects.py index 058fc989..f367045b 100644 --- a/backend/app/data/source_objects.py +++ b/backend/app/data/source_objects.py @@ -825,6 +825,33 @@ async def absent_reasons(session: AsyncSession, row: Any, names: Iterable[str]) return out +#: 表结构里没有的表给人看的一句（接口的 404、批量审阅逐条的错误）：和界面的 CATALOG_ABSENT_TEXT、/schema 的说法 +#: （schema_view.absent_detail，给模型看）同一套「不在」。{table} 是点名的表 +ABSENT_MESSAGE = { + ABSENT_REMOVED: "数据源中已经没有表 {table}（已删除或改名）", + ABSENT_MISSING: "数据源中没有表 {table}（全库的对象清单里也没有）", + ABSENT_OUT_OF_SCOPE: ("表 {table} 在探查范围外：对象索引里有它，但没有读取结构。在数据源卡片的「探查范围」里加入它，" + "或在数据目录的搜索框搜索它并「加入范围」"), + ABSENT_REFLECT_FAILED: "表 {table} 在探查范围内，但上次探查读不到它的结构（权限不够或对象有问题），修好后重新探查即可", + ABSENT_UNKNOWN: ("表 {table} 不在已探查的表里：升级前的探查只取了一部分表，无法判断它是否还在，请回到数据源卡片重新" + "「探查结构」"), +} +#: 说不出是哪一种「不在」时（上传表格、记录表按版本走；没有对象索引、又没截断过的老缓存认不出名字是不是从来就没有） +ABSENT_PLAIN = "数据源中没有表 {table}" + + +async def absent_message(session: AsyncSession, row: Any, table: str) -> str: + """点名的表不在表结构里时给人看的一句（ABSENT_MESSAGE）。手工接入的源按 absent_reasons 说是哪一种；上传表格、记录表 + 的结构随版本走，只说没有（目录清单把它们记成 removed，是说「那一版里没有」，点名一个从来没有过的名字时说不得「已删除」）； + 没有对象索引的老缓存同理：absent_reasons 对没截断的一律记 removed，名字写错了也会这样，只说没有。""" + if (getattr(row, "origin", None) or "manual") != "manual": + return ABSENT_PLAIN.format(table=table) + reason = (await absent_reasons(session, row, [table])).get(table) + if reason == ABSENT_REMOVED and not await has_index(session, row.id, index_schema(row)): + reason = None + return ABSENT_MESSAGE.get(reason or "", ABSENT_PLAIN).format(table=table) + + @dataclass(frozen=True) class OutsideObject: """对象索引里在、范围外的一个对象（助手检索、目录搜索用）。""" @@ -1039,7 +1066,8 @@ async def delete_index(session: AsyncSession, source_id: str) -> None: __all__ = [ - "ABSENT_MISSING", "ABSENT_OUT_OF_SCOPE", "ABSENT_REFLECT_FAILED", "ABSENT_REMOVED", "ABSENT_UNKNOWN", "ADD_LIMIT", "AddOutcome", + "ABSENT_MESSAGE", "ABSENT_MISSING", "ABSENT_OUT_OF_SCOPE", "ABSENT_PLAIN", "ABSENT_REFLECT_FAILED", "ABSENT_REMOVED", + "ABSENT_UNKNOWN", "ADD_LIMIT", "AddOutcome", "INDEX_LIMIT", "IndexResult", "IndexedObject", "OutsideObject", "REFLECT_BATCH", "RULES_MAX", "Reflected", "RefreshStats", "SCOPE_DEFAULT", "SCOPE_MAX", "SCOPE_OPTION", "SCOPE_REASONS", "ScopePlan", "ScopeRules", "Trail", "VIA_CATALOG", "VIA_LOOK", "VIA_SCHEMA_TOOL", "absent_reasons", "add_to_scope", "apply_scope", diff --git a/backend/tests/test_catalog_schema_truncated.py b/backend/tests/test_catalog_schema_truncated.py index f0c99435..e4a9ba70 100644 --- a/backend/tests/test_catalog_schema_truncated.py +++ b/backend/tests/test_catalog_schema_truncated.py @@ -157,3 +157,24 @@ def test_legacy_truncated_cache_still_does_not_deny(): "truncated": True, "total": 205}) text = describe_table(row, "t300") assert "中没有" not in text and "只探查了" in text and "information_schema" in text + + +async def test_catalog_detail_of_an_out_of_scope_table_says_so(client, wide): + """对象索引里有、在探查范围外的表：GET /catalog/{表} 的 404 说「在探查范围外」，和 /schema、/catalog/search 同一套「不在」 + （下一轮验收：以前说「可能已被删除或尚未探查结构」）；表助手、批量审阅的同一个错误也一样。对象索引里也没有的说「没有 + 这张表」。""" + sid, _ = wide + r = await client.get(f"/api/datasources/{sid}/catalog/t200") + assert r.status_code == 404 and "探查范围外" in r.json()["detail"], r.text + assert "可能已被删除" not in r.json()["detail"] + found = (await client.get(f"/api/datasources/{sid}/catalog/search", params={"q": "t200"})).json()["items"] + assert [i["absent_reason"] for i in found if i["name"] == "t200"] in (["out_of_scope"], [None]), found + r = await client.post(f"/api/datasources/{sid}/catalog/t201/assistant", json={"messages": [ + {"role": "user", "content": "这张表是干什么的"}]}) + assert r.status_code == 404 and "探查范围外" in r.json()["detail"], r.text + r = await client.post(f"/api/datasources/{sid}/catalog/review/batch", json={"items": [ + {"table_name": "t202", "path": "label", "action": "confirm", "if_version": 0}]}) + [result] = r.json()["results"] + assert result["ok"] is False and "探查范围外" in result["error"], result + r = await client.get(f"/api/datasources/{sid}/catalog/no_such_table") + assert r.status_code == 404 and "没有表 no_such_table" in r.json()["detail"] and "探查范围外" not in r.json()["detail"] From 3f75cb1d2ea75585c1ed448db549485c1b6f0bf6 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Sat, 17 Oct 2026 08:31:17 +0800 Subject: [PATCH 05/15] =?UTF-8?q?=E6=81=A2=E5=A4=8D=E8=A2=AB=E9=A9=B3?= =?UTF-8?q?=E5=9B=9E=E7=9A=84=E8=BF=90=E8=A1=8C=EF=BC=9A=E7=85=A7=20end=5F?= =?UTF-8?q?reason=20=E8=AF=B4=E3=80=8C=E8=A2=AB=E9=A9=B3=E5=9B=9E=E3=80=8D?= =?UTF-8?q?=EF=BC=8C=E5=92=8C=E7=BB=A7=E7=BB=AD=E8=BF=90=E8=A1=8C=E5=90=8C?= =?UTF-8?q?=E4=B8=80=E4=B8=AA=E9=94=99=E8=AF=AF=E7=A0=81=20run=5Frejected?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 验收：对被驳回的运行调 POST /api/runs/{id}/resume，回的是「这次运行已取消，没有待处理的审批，无法恢复。已取消的 运行请重新发起」，/continue 却是对的（run_rejected）。resume 只看 status，cancelled 一律说「已取消」；存量的 failed + 驳回还会被说成「失败的运行可点继续运行」，而继续运行其实会被拒。 改法：runner 加 _resume_refused：cancelled + end_reason=rejected、以及加 end_reason 之前记成 failed 的驳回 （_rejected_before，同 continue_failed 的判断）抛 ContinueRefused(REJECTED_RESUME, code=RUN_REJECTED)，别的照 _cannot_resume。resume 开头的状态检查、等审批凑齐时发现运行已经不在 interrupted 两处都用它。/resume 接口像 /continue 一样把带机器码的拒绝回成 CodedHTTPException(409, …, code)。ContinueRefused 是 ValueError 的子类， 别的调用方（待办服务）照旧按 ValueError 接。 测试：test_run_rejected 新增一条：驳回的运行 /resume 回 409 + run_rejected、说「被驳回」；手动停止的照旧说 已取消、不带这个码；存量 failed + 驳回也回 run_rejected，不再叫人点「继续运行」。去掉修复回跑挂。 --- backend/app/api/runs.py | 5 +++++ backend/app/engine/runner.py | 18 +++++++++++++++--- backend/tests/test_run_rejected.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 3 deletions(-) diff --git a/backend/app/api/runs.py b/backend/app/api/runs.py index f06d8b4a..606f82ad 100644 --- a/backend/app/api/runs.py +++ b/backend/app/api/runs.py @@ -418,6 +418,8 @@ async def resume_run( approval = await session.get(Approval, target) if approval is not None and approval.run_id == run_id and approval.status == "pending": return await _respond_to(target, payload.response, x_actor) + from app.engine.runner import ContinueRefused + try: # approval_id 以前定义了却从没传下去——一次运行里有多条待审批时， # 引擎无从知道这次回复的是哪一条 @@ -428,6 +430,9 @@ async def resume_run( ) except KeyError as e: raise _missing(e) from e + except ContinueRefused as e: + # 被驳回的（run_rejected）：和 /continue 同一个机器码 + raise CodedHTTPException(409, str(e), e.code) from e except ValueError as e: raise HTTPException(409, str(e)) from e diff --git a/backend/app/engine/runner.py b/backend/app/engine/runner.py index ac31b861..c3acfd8c 100644 --- a/backend/app/engine/runner.py +++ b/backend/app/engine/runner.py @@ -52,6 +52,8 @@ END_REJECTED = "rejected" END_STOPPED = "stopped" #: 被驳回的运行不能继续运行：从断点重跑会读到同一个驳回答复、再停在同一处 REJECTED_CONTINUE = "被驳回的运行不能继续运行：改好内容后重新发起" +#: 被驳回的运行也没有可恢复的审批（恢复、继续运行同一个错误码 RUN_REJECTED） +REJECTED_RESUME = "这次运行已被驳回，没有待处理的审批，无法恢复：改好内容后重新发起" #: 服务重启时还在排队的运行改成失败时写的话（run.error 和补发的 run.failed 是同一句） QUEUED_AT_RESTART = "服务重启时这次运行仍在排队，没有可恢复的进度，请重新发起运行" @@ -355,7 +357,7 @@ class RunManager: if not run: raise KeyError("运行记录不存在") if run.status not in ("interrupted",): - raise ValueError(_cannot_resume(run.status)) + raise await _resume_refused(session, run) spec = GraphSpec.model_validate(run.graph) workflow_id = run.workflow_id carried = _carried(run) @@ -458,8 +460,8 @@ class RunManager: await session.commit() run = await session.get(Run, run_id) if run.status != "interrupted": - # 审批没了是因为运行在这期间被放弃了：不能回一个「已记下」 - raise ValueError(_cannot_resume(run.status)) + # 审批没了是因为运行在这期间被放弃了（或被别人驳回了）：不能回一个「已记下」 + raise await _resume_refused(session, run) return run # 齐了：正式落 resolved，然后驱动引擎。谁批的跟着交给节点， @@ -1186,6 +1188,16 @@ _STATUS_TEXT = { } +async def _resume_refused(session: Any, run: Run) -> ValueError: + """恢复被拒时抛什么：被驳回的（cancelled + end_reason=rejected，以及加 end_reason 之前记成 failed 的驳回）说「被驳回」、 + 带 RUN_REJECTED，和继续运行同一个错误码（以前照状态说「已取消的运行请重新发起」、对存量的还叫人点「继续运行」， + 可继续运行也会被拒）；别的照状态说（_cannot_resume）。""" + if (run.status == "cancelled" and run.end_reason == END_REJECTED) or ( + run.status == "failed" and await _rejected_before(session, run)): + return ContinueRefused(REJECTED_RESUME, code=RUN_REJECTED) + return ValueError(_cannot_resume(run.status)) + + def _cannot_resume(status: str) -> str: hint = { "running": "运行仍在进行，无需恢复。", diff --git a/backend/tests/test_run_rejected.py b/backend/tests/test_run_rejected.py index 1da7e538..18f5cf35 100644 --- a/backend/tests/test_run_rejected.py +++ b/backend/tests/test_run_rejected.py @@ -186,6 +186,35 @@ async def test_the_api_carries_end_reason_and_refuses_to_continue_a_rejected_run assert stop.status_code == 409 and "被驳回" in stop.json()["detail"] +async def test_resume_on_a_rejected_run_says_rejected_with_the_same_code(client): + """对被驳回的运行调 /resume：照 end_reason 说「被驳回」，和 /continue 同一个错误码 run_rejected（下一轮验收：以前说 + 「这次运行已取消…已取消的运行请重新发起」）。手动停止的照旧说已取消；存量的 failed + 驳回也说被驳回，不再叫人点 + 「继续运行」。""" + run_id = await _rejected_run("approve") + await _wait(run_id, ("cancelled",)) + r = await client.post(f"/api/runs/{run_id}/resume", json={"response": {"approved": True}}) + assert r.status_code == 409 and r.json()["code"] == "run_rejected", r.text + assert "被驳回" in r.json()["detail"] and "已取消" not in r.json()["detail"] + async with SessionLocal() as session: + stopped = Run(workflow_name="停掉的", status="cancelled", end_reason="stopped", graph=chain(START, OUT), input={}) + old = Run(workflow_name="老的驳回", status="failed", graph=chain(START, GATES["approve"][0], OUT), input={}, + error="人工驳回：不对", error_node_id="gate") + session.add_all([stopped, old]) + await session.flush() + session.add_all([ + RunEvent(run_id=old.id, seq=1, type="node.failed", node_id="gate", ts=1.0, data={"error": "人工驳回：不对"}), + Task(run_id=old.id, node_id="gate", kind="approve", title="可以吗", status="resolved", + outcome={"decision": "reject"}), + ]) + await session.commit() + stopped_id, old_id = stopped.id, old.id + other = await client.post(f"/api/runs/{stopped_id}/resume", json={"response": {"approved": True}}) + assert other.status_code == 409 and other.json().get("code") != "run_rejected" and "已取消" in other.json()["detail"] + legacy = await client.post(f"/api/runs/{old_id}/resume", json={"response": {"approved": True}}) + assert legacy.status_code == 409 and legacy.json()["code"] == "run_rejected", legacy.text + assert "继续运行" not in legacy.json()["detail"].replace("不能继续运行", "") + + async def test_a_legacy_failed_rejection_cannot_be_continued_either(client): """加 end_reason 之前被驳回的运行记的是 failed（封存不改写）：失败的那一步上最近一条处理完的待办是驳回，也不许接着跑。 别的失败照旧能接着跑（这里断点不在，说的是没有断点，不是被驳回）。""" From 9040289e1f2e30d789cd33b47bbd4b11375d32e2 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Sat, 17 Oct 2026 08:35:57 +0800 Subject: [PATCH 06/15] =?UTF-8?q?=E5=9B=BA=E5=8C=96=E6=88=90=E5=8F=A3?= =?UTF-8?q?=E5=BE=84=E8=8D=89=E7=A8=BF=EF=BC=9A=E5=8F=A3=E5=BE=84=E5=8D=A1?= =?UTF-8?q?=E7=94=A8=E4=BC=A0=E5=85=A5=E7=9A=84=E5=90=8D=E5=AD=97=E3=80=81?= =?UTF-8?q?=E6=8A=A5=E5=91=8A=E6=8C=87=E4=BB=A4=E4=B8=8D=E5=86=99=E6=AD=BB?= =?UTF-8?q?=E4=BB=A3=E8=A1=A8=E9=82=A3=E4=B8=80=E8=BD=AE=E7=9A=84=E6=97=A5?= =?UTF-8?q?=E6=9C=9F=E3=80=81=E6=8C=87=E6=A0=87=E5=90=8D=E5=8F=96=E7=9B=AE?= =?UTF-8?q?=E5=BD=95=E4=B8=AD=E6=96=87=E5=90=8D=E3=80=81=E5=B7=B2=E5=9B=BA?= =?UTF-8?q?=E5=8C=96=E5=8E=BB=E9=87=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 验收报告里 certify.freeze 的四处： - 口径卡的 caliber 用的是代表问题的原文，没用传入的 name。改成 name（去首尾空白）优先，没传才用代表问题。 - 报告的 instructions 写死成「回答：<代表问题>」，问题里带着那一轮的日期（「上周（10月5日到11日）」「九月第一周」）， 参数化之后这句就错了。有入口参数时改写运行时渲染的参数（「这次的参数：p1 = {{ input.p1 }}，…；时间范围、筛选的 取值以参数为准」），不写问题原文；没有参数时取数的字面量都是口径的常量，照旧写问题原文。 - 指标名直接用列名。现在能从数据目录取到中文名的用中文名：先看查询快照冻结的目录（schema_artifact，和口径比对用的 同一份，ledger.match._frozen），老快照没冻结的看数据源现在的目录；只认列名本身就是目录里的一列（SELECT amount、 SUM(amount) AS amount），gmv 这类算出来起的别名取不到，照旧用列名。 - 认证队列里 cluster 项的 frozen_into 把同一个工作流列了好几次：固化时同组的几次运行都记进 origin.sources，一项里 的几次运行就各指一遍同一个工作流。shape 项同样会重。加 _frozen_of 按工作流去重（保持先后），三种项都用它。 测试（test_ledger_certify）：原来的固化用例改成断言指令里没有代表问题和日期、有 {{ input.p1 }} {{ input.p2 }}， 没传名字时口径卡照旧用代表问题，shape 和 cluster 两项的 frozen_into 都只有一个工作流；新增一条：传了名字的口径卡 用名字，目录里有中文名的列（amount → 金额）用中文名、取不到的照旧（n），没有参数时指令照旧写问题原文。去掉修复 回跑两条都挂。 --- backend/app/ledger/certify.py | 60 +++++++++++++++++++++++----- backend/tests/test_ledger_certify.py | 56 ++++++++++++++++++++++++-- 2 files changed, 102 insertions(+), 14 deletions(-) diff --git a/backend/app/ledger/certify.py b/backend/app/ledger/certify.py index cf57526b..f4384c2b 100644 --- a/backend/app/ledger/certify.py +++ b/backend/app/ledger/certify.py @@ -11,13 +11,16 @@ 个问题的类； - 被反复用的答案（reused）：导出过 ≥ 2 次（report.exported 审计，第 2 波）、做成过应用（蓝图的来源是这次运行）、在画布里 打开过或提取过（工作流的 origin.run_id 是这次运行）的轮次。 -每项给代表问题、轮次链接、（shape 的）SQL 形状和参数候选、有没有相关口径（可以对账的那份）、是不是已经固化过。点「不是口径」 +每项给代表问题、轮次链接、（shape 的）SQL 形状和参数候选、有没有相关口径（可以对账的那份）、是不是已经固化过（frozen_into， +同一个工作流只列一次）。点「不是口径」 写一条审计（ledger.certify.dismiss，subject 是这一项的 id），之后不再进队。门槛同 ledger/stats（E3 的重复度）。 **固化成口径草稿**（freeze，from-run 加 freeze 选项后转调它）：代表运行里报告引用了的查询，各变成一个调用工具节点——SQL 用 规范化后的文本，跨轮变化的字面量换成 '{{ input.p1 }}'（同组的别的运行 peers 用来认哪些位置变了），入口字段的缺省值取 -代表运行里的值；只有一行的结果每个数值列建一项口径卡指标（cell(nodes.取数, 0, '列')）；报告节点保留，出口契约写 -report_from + metrics_from（有多行结果的另加 cells: true），过 validate_graph。新工作流记 origin.sources： +代表运行里的值；只有一行的结果每个数值列建一项口径卡指标（cell(nodes.取数, 0, '列')，指标名能从数据目录取到中文名的 +用中文名，取不到用列名）；口径卡的名字用传入的 name，没传用代表问题；报告节点保留，指令在有入口参数时不写代表那一轮的问题 +原文（里面的日期参数化之后就不对了），改写运行时渲染的参数，没有参数时照旧写问题原文；出口契约写 report_from + +metrics_from（有多行结果的另加 cells: true），过 validate_graph。新工作流记 origin.sources： [{run_id, conversation_id, turn_id, question}]，这就是「口径 → 来源问题」的记录。之后由人在画布上审改、按受管级别发布—— 提取器是作者，不是发布者。 @@ -64,6 +67,9 @@ DRAFT_NAME = "{question}（固化的口径草稿）" DRAFT_DESCRIPTION = ("从问数据的回答固化：报告引用了的 {n} 条查询变成固定的取数，跨轮变化的 {p} 个字面量成为入口参数。" "经人工审核、按受管级别发布后，再问同样的问题就能认出等价、按口径出具") REPORT_INSTRUCTIONS = "回答：{question}" +#: 有入口参数时的报告指令：不写代表那一轮的问题原文——里面的日期（「上周（10月5日到11日）」「九月第一周」）参数化之后 +#: 就不对了；改写这次运行时渲染出来的参数。{params} 是「p1 = {{ input.p1 }}，…」 +REPORT_INSTRUCTIONS_PARAMS = "按取到的数回答这个口径的问题（这次的参数：{params}；时间范围、筛选的取值以参数为准）" class CertifyRefused(Exception): @@ -228,6 +234,16 @@ async def _frozen(session: Any) -> dict[str, list[dict[str, Any]]]: return out +def _frozen_of(run_ids: Sequence[str | None], frozen: dict[str, list[dict[str, Any]]]) -> list[dict[str, Any]]: + """这几次运行固化进了哪些工作流：同一个工作流只列一次（固化时同组的运行都记进 origin.sources，一项里的几次运行常常 + 指着同一个工作流），按先后。""" + out: dict[str, dict[str, Any]] = {} + for run_id in run_ids: + for w in frozen.get(run_id or "", []): + out.setdefault(str(w["workflow_id"]), w) + return list(out.values()) + + def _turn_view(member: Any) -> dict[str, Any]: return {"run_id": member.run_id, "conversation_id": member.conversation_id, "turn_id": member.turn_id, "question": member.question} @@ -308,7 +324,7 @@ async def queue(session: Any, *, principal: Any) -> dict[str, Any]: "constants": len(rep.literals) - len(params), "related": related, "representative": {"run_id": rep.run_id, "query": rep.artifact}, "peers": [m.run_id for m in once[1:]], - "frozen_into": [w for m in once for w in frozen.get(m.run_id, [])], + "frozen_into": _frozen_of([m.run_id for m in once], frozen), }) # 问题聚类 questions = await recent_questions(session, principal, limit=QUESTION_LIMIT) @@ -322,7 +338,7 @@ async def queue(session: Any, *, principal: Any) -> dict[str, Any]: items.append({"id": iid, "kind": "cluster", "question": seed.text, "count": cluster.count, "turns": [{"run_id": q.run_id, "turn_id": q.turn_id, "question": q.text} for q in cluster.members[:TURNS_LIMIT]], - "frozen_into": [w for q in cluster.members if q.run_id for w in frozen.get(q.run_id, [])]}) + "frozen_into": _frozen_of([q.run_id for q in cluster.members], frozen)}) items += await _reused(session, runs, turns, dismissed, frozen) return {"items": items, "considered": {"runs": len(runs), "matched": len(latest), "questions": len(questions)}, "threshold": {"runs": REPEAT_RUNS, "conversations": REPEAT_CONVERSATIONS, "exports": EXPORTS}, @@ -367,7 +383,7 @@ async def _reused(session: Any, runs: list[Run], turns: dict[str, dict[str, Any] out.append({"id": iid, "kind": "reused", "question": info.get("question") or "", "uses": uses, "turns": [{"run_id": run.id, "conversation_id": info.get("conversation_id"), "turn_id": info.get("turn_id"), "question": info.get("question") or ""}], - "representative": {"run_id": run.id}, "frozen_into": frozen.get(run.id, [])}) + "representative": {"run_id": run.id}, "frozen_into": _frozen_of([run.id], frozen)}) return out @@ -385,6 +401,24 @@ async def dismiss(session: Any, iid: str, *, kind: str | None = None, note: str # ========================================================================== +async def _labels(session: Any, snap: dict[str, Any], source: str) -> Any: + """结果列的中文名：先看查询当时冻结的数据目录（快照的 schema_artifact，同口径比对），没冻结的（老快照）看数据源现在 + 的目录。只认列名本身就是目录里的一列（SELECT amount、SUM(amount) AS amount）；算出来起的别名（gmv）取不到，调用方 + 照旧用列名。返回 _labeler：(表, 列) → 中文名或 None。""" + from app.data import catalog + from app.data.sql_semantics import _labeler, catalog_labels + from app.db.models import DataSource + from app.ledger.match import _frozen as frozen_schema + + _, names = frozen_schema(snap.get("schema_artifact")) + if not names: + source_id = (await session.execute(select(DataSource.id).where(DataSource.name == source))).scalar() + if source_id: + entries = await catalog.read_catalog(session, source_id) + names = catalog_labels({table: entry.notes for table, entry in entries.items()}) + return _labeler(names) + + async def freeze(session: Any, run_id: str, *, peers: Sequence[str] = (), name: str = "", conversation_id: str | None = None, turn_id: str | None = None) -> dict[str, Any]: """把一次问数据运行固化成口径草稿（见模块说明），落库成新的草稿工作流，返回 {workflow_id, name, nodes, edges, params, @@ -460,9 +494,10 @@ async def freeze(session: Any, run_id: str, *, peers: Sequence[str] = (), name: rows = snap.get("rows") if isinstance(snap.get("rows"), list) else [] columns = [str(c) for c in snap.get("columns") or []] if len(rows) == 1 and isinstance(rows[0], (list, tuple)): + label = await _labels(session, snap, shape.source) for col, value in zip(columns, rows[0]): if isinstance(value, (int, float)) and not isinstance(value, bool): - metrics.append({"id": f"m{len(metrics) + 1}", "name": col, + metrics.append({"id": f"m{len(metrics) + 1}", "name": label((None, col)) or col, "expression": f"cell(nodes.{nid(i)}, 0, {col!r})"}) else: multi = True @@ -473,15 +508,20 @@ async def freeze(session: Any, run_id: str, *, peers: Sequence[str] = (), name: found = next((n for n in (run.graph or {}).get("nodes") or [] if n.get("id") == reports[0].node_id), None) cfg = ((found or {}).get("data") or {}).get("config") or {} report_cfg = {k: cfg[k] for k in ("provider", "model", "entities") if k in cfg} - report_cfg["instructions"] = REPORT_INSTRUCTIONS.format(question=question or "（问题）") + if fields: + report_cfg["instructions"] = REPORT_INSTRUCTIONS_PARAMS.format( + params="，".join(f"{f['name']} = {{{{ input.{f['name']} }}}}" for f in fields)) + else: + # 没有入口参数：取数的字面量都是口径的常量，问题原文里的日期和它们对得上 + report_cfg["instructions"] = REPORT_INSTRUCTIONS.format(question=question or "（问题）") # 报告的核对规则一律按受管级别的要求写（engine/governance.REPORT_POLICY，和发布门禁同一份）：草稿要能按受管发布 report_cfg.update({key: value for key, value, _ in REPORT_POLICY}) contract: dict[str, Any] = {"report_from": "write"} tail: list[dict[str, Any]] = [] if metrics: tail.append({"id": "card", "type": "metrics", "position": {"x": 0, "y": 0}, - "data": {"label": "口径卡", "config": {"caliber": (question or "口径")[:40], "caliber_version": "v1", - "metrics": metrics}}}) + "data": {"label": "口径卡", "config": {"caliber": (name.strip() or question or "口径")[:40], + "caliber_version": "v1", "metrics": metrics}}}) report_cfg["metrics_from"] = ["card"] # 受管出具要写必需指标（不然「不予出具」这一档触发不了）：固化出的每个指标都是这个口径要给的数 contract.update({"metrics_from": "card", "required": [m["id"] for m in metrics]}) diff --git a/backend/tests/test_ledger_certify.py b/backend/tests/test_ledger_certify.py index e91dee59..4444064c 100644 --- a/backend/tests/test_ledger_certify.py +++ b/backend/tests/test_ledger_certify.py @@ -180,7 +180,11 @@ async def test_freezing_makes_a_fixed_sql_draft_that_passes_the_governed_gate(cl "required": ["m1"]} assert {k: nodes["write"]["data"]["config"].get(k) for k in ("numbers", "on_violation", "claims")} == { "numbers": "strict", "on_violation": "fail", "claims": "require_citation"} - assert nodes["write"]["data"]["config"]["instructions"] == f"回答：{QUESTION}" + # 有入口参数：不写代表那一轮的问题原文（「九月第一周」参数化之后就不对了），写运行时渲染的参数 + instructions = nodes["write"]["data"]["config"]["instructions"] + assert QUESTION not in instructions and "2026-09" not in instructions, instructions + assert "{{ input.p1 }}" in instructions and "{{ input.p2 }}" in instructions + assert nodes["card"]["data"]["config"]["caliber"] == QUESTION, "没传名字：口径卡照旧用代表问题" spec = GraphSpec.model_validate(wf.graph) assert not [i for i in validate_graph(spec).issues if i.level == "error"] assert not [i for i in publish_issues(spec, level="governed") if i.level == "error"], \ @@ -189,9 +193,12 @@ async def test_freezing_makes_a_fixed_sql_draft_that_passes_the_governed_gate(cl assert all(s["question"] == QUESTION for s in wf.origin["sources"]) assert wf.origin["run_id"] == rep_run.id and wf.origin["freeze"] is True - # 队列上标着已经固化过 - [again] = shape_items((await client.get("/api/ledger/certify-queue")).json()) - assert {w["workflow_id"] for w in again["frozen_into"]} == {wf.id} + # 队列上标着已经固化过：同一个工作流只列一次（三个来源运行都指着它） + body = (await client.get("/api/ledger/certify-queue")).json() + [again] = shape_items(body) + assert [w["workflow_id"] for w in again["frozen_into"]] == [wf.id] + [cluster] = [i for i in body["items"] if i["kind"] == "cluster" and i["question"] == QUESTION] + assert [w["workflow_id"] for w in cluster["frozen_into"]] == [wf.id], cluster["frozen_into"] # 按受管级别发布：来源运行在后台补比，变成等价已证明、出了队 res = await client.post(f"/api/workflows/{wf.id}/publish", json={"level": "governed"}) @@ -216,6 +223,47 @@ async def test_freezing_makes_a_fixed_sql_draft_that_passes_the_governed_gate(cl assert stats["frozen"] >= 1 and stats["published"] >= 1 +async def test_freezing_uses_the_given_name_and_catalog_labels_and_no_dates_without_parameters(client, monkeypatch): + """传了名字：口径卡用它（下一轮验收：以前用代表问题的原文）。指标名能从数据目录取到中文名的用中文名，取不到照旧用 + 列名。没有入口参数（字面量都没变过）时报告指令照旧写问题原文：那时问题里的日期就是口径的常量。""" + from app.data import catalog + from app.db.models import DataSource + from sqlalchemy import select + + async with SessionLocal() as session: + source = (await session.execute(select(DataSource).where(DataSource.name == S.CURRENT["source"]))).scalar_one() + await catalog.write_entry(session, source.id, "orders", {"columns": { + "amount": {"label": catalog.make_item("金额", "human")}}}, if_version=0, actor="测试员") + sql = "SELECT SUM(amount) AS amount, COUNT(*) AS n FROM orders WHERE status = 'paid' AND paid_at >= '2026-09-01'" + S.script(monkeypatch, "金额 [[v:Q1.r0.amount]]，[[v:Q1.r0.n]] 单。") + conv = await conversation() + async with SessionLocal() as session: + turn = ConversationTurn(conversation_id=conv, question=QUESTION) + session.add(turn) + await session.commit() + turn_id = turn.id + run = await run_manager.start(graph=S.explore_graph(sql), input_payload={"question": QUESTION}, origin_kind="chat", + origin={"conversation_id": conv, "turn_id": turn_id}) + row = await S.wait(run.id) + assert row.status == "succeeded", row.error + async with SessionLocal() as session: + (await session.get(ConversationTurn, turn_id)).run_id = row.id + await session.commit() + await ledger_match.drain() + res = await client.post("/api/copilot/from-run", json={"run_id": row.id, "freeze": True, "name": "九月销售额口径"}) + assert res.status_code == 200, res.text + made = res.json() + assert made["params"] == [] and made["name"] == "九月销售额口径" + async with SessionLocal() as session: + wf = await session.get(Workflow, made["workflow_id"]) + nodes = {n["id"]: n for n in wf.graph["nodes"]} + card = nodes["card"]["data"]["config"] + assert card["caliber"] == "九月销售额口径" + assert [(m["name"], m["expression"]) for m in card["metrics"]] == [ + ("金额", "cell(nodes.fetch, 0, 'amount')"), ("n", "cell(nodes.fetch, 0, 'n')")] + assert nodes["write"]["data"]["config"]["instructions"] == f"回答：{QUESTION}" + + async def test_freezing_needs_a_matched_successful_run(client, monkeypatch): res = await client.post("/api/copilot/from-run", json={"run_id": "nope", "freeze": True}) assert res.status_code == 404 From c3173909dbe07ee52c6797e965cedf66ba104c42 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Sat, 17 Oct 2026 08:42:26 +0800 Subject: [PATCH 07/15] =?UTF-8?q?=E6=95=B4=E7=90=86=EF=BC=9A=E5=89=8D?= =?UTF-8?q?=E4=B8=A4=E6=AC=A1=E6=8F=90=E4=BA=A4=E9=87=8C=E8=B6=85=E8=BF=87?= =?UTF-8?q?=20120=20=E5=AD=97=E7=9A=84=E4=B8=A4=E8=A1=8C=E6=8A=98=E8=A1=8C?= =?UTF-8?q?=EF=BC=8Cabsent=5Fmessage=20=E8=BF=9B=20source=5Fobjects=20?= =?UTF-8?q?=E7=9A=84=20=5F=5Fall=5F=5F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 只动排版和导出清单，不改行为。 --- backend/app/data/source_objects.py | 4 ++-- backend/app/data/sql_semantics.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/backend/app/data/source_objects.py b/backend/app/data/source_objects.py index f367045b..b938a7d4 100644 --- a/backend/app/data/source_objects.py +++ b/backend/app/data/source_objects.py @@ -1066,8 +1066,8 @@ async def delete_index(session: AsyncSession, source_id: str) -> None: __all__ = [ - "ABSENT_MESSAGE", "ABSENT_MISSING", "ABSENT_OUT_OF_SCOPE", "ABSENT_PLAIN", "ABSENT_REFLECT_FAILED", "ABSENT_REMOVED", - "ABSENT_UNKNOWN", "ADD_LIMIT", "AddOutcome", + "ABSENT_MESSAGE", "ABSENT_MISSING", "ABSENT_OUT_OF_SCOPE", "ABSENT_PLAIN", "ABSENT_REFLECT_FAILED", + "ABSENT_REMOVED", "ABSENT_UNKNOWN", "ADD_LIMIT", "AddOutcome", "absent_message", "INDEX_LIMIT", "IndexResult", "IndexedObject", "OutsideObject", "REFLECT_BATCH", "RULES_MAX", "Reflected", "RefreshStats", "SCOPE_DEFAULT", "SCOPE_MAX", "SCOPE_OPTION", "SCOPE_REASONS", "ScopePlan", "ScopeRules", "Trail", "VIA_CATALOG", "VIA_LOOK", "VIA_SCHEMA_TOOL", "absent_reasons", "add_to_scope", "apply_scope", diff --git a/backend/app/data/sql_semantics.py b/backend/app/data/sql_semantics.py index 8ec28094..ffb7ae58 100644 --- a/backend/app/data/sql_semantics.py +++ b/backend/app/data/sql_semantics.py @@ -7422,7 +7422,8 @@ class _Explainer: if times != {key} and any(_window_like(c) for c in on_a + on_b): continue kinds = _kinds_for(on_a + on_b) - if self.usage_a.touches({key}, skip=on_a, kinds=kinds) or self.usage_b.touches({key}, skip=on_b, kinds=kinds): + if self.usage_a.touches({key}, skip=on_a, kinds=kinds) or \ + self.usage_b.touches({key}, skip=on_b, kinds=kinds): continue never_null = _not_null_column(self.info, key[0], key[1], self.read) if self.range.compare(on_a, mapped_b, never_null=never_null) is not False: From f5c9f0c13c2905b06e27a6a1c0a5fc417f0d2fae Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Sat, 17 Oct 2026 08:51:51 +0800 Subject: [PATCH 08/15] =?UTF-8?q?=E5=89=96=E6=9E=90=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3=E6=94=B6=E5=8F=96=E5=80=BC=E8=A7=82=E5=AF=9F?= =?UTF-8?q?=E8=8C=83=E5=9B=B4=EF=BC=8C=E5=AF=B9=E8=AF=9D=E6=A1=86=E7=BB=99?= =?UTF-8?q?=E5=87=BA=E3=80=8C=E5=8F=96=E5=80=BC=E8=A7=82=E5=AF=9F=E3=80=8D?= =?UTF-8?q?=EF=BC=9B=E5=AE=9A=E6=97=B6=E7=BB=B4=E6=8A=A4=E6=8C=91=E4=B8=8D?= =?UTF-8?q?=E5=87=BA=E8=A6=81=E8=A7=82=E5=AF=9F=E7=9A=84=E8=A1=A8=E6=97=B6?= =?UTF-8?q?=E6=8A=8A=E5=8E=9F=E5=9B=A0=E8=AE=B0=E8=BF=9B=E8=A7=A6=E5=8F=91?= =?UTF-8?q?=E8=AE=B0=E5=BD=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 验收：取值观察只能经定时维护触发——POST /catalog/profile-jobs 的 scope 只收 needed、all，可剖析任务本来就支持 observe（只观察、不剖析）。另外 12 小时内观察过的表不再观察时，任务以 0 张表结束，原因只在剖析任务的 notice 里，从定时维护触发时触发记录（ScheduleFire.inputs）看不出来。 后端： - ProfileJobIn.scope 加 observe（maintenance 是定时维护用的，接口照旧不收）。挑不出表时照旧 409 profile_nothing， 说法是 NOTHING_TO_OBSERVE（「最近 12 小时内都观察过了」）。 - catalog_profile_jobs.nothing_reason：按此刻的表结构和目录一张表都挑不出来时交回 ProfileNothing 的原话；排在 起草后面时说不准，返回 None。 - source_maintenance._start：建出 0 张表的剖析任务、这次触发又没有探查（开始时表结构不会变）、也没排在起草后面 （目录不会变）时，判断得出任务会以 0 张表结束，把原因记进 inputs.skipped（同一套 NOT_RUN 说法进 reason）。 任务照建、触发照旧是 started：每小时一次的维护不至于一天二十几次「没能发起」。同一次触发里有探查、或者排在 起草后面时判断不了，保持现状，写进了模块说明。 前端：剖析对话框的范围里加「取值观察：有数据目录的 N 张表」（说法照维护设置里「取值观察」那一步：码值分布、 最新业务日期、按日行数和行数，和同口径的历史比出取值变化；最近 12 小时内观察过的这次不再观察），建任务带 scope: observe。张数按清单上有数据目录、不是视图的表算（isObservable，profile.ts），观察到期没有由服务端判， 所以是上限。挑不出表时说「没有要观察的表」和服务端原话，不给「改选全部表」（那是剖析，不是观察）。文案进 PROFILE_TEXT。 测试：test_catalog_observe 新增一条（接口建 observe 任务、只做观察检查，再建一次 409 profile_nothing，maintenance 422）；test_source_maintenance 新增一条（只选取值观察，12 小时内再触发：任务 0 张表，inputs.skipped 和 reason 写明 原因）。去掉修复回跑两条都挂。check-catalog-profile 跟上：范围多一项 observe:1，说法核对；新一节建取值观察任务 （请求体 {scope: observe}）、都观察过时的「没有要观察的表」且没有「改选全部表」。tsc、check-copy 通过， check-catalog-profile 204 项全过。 --- backend/app/api/catalog.py | 5 +- backend/app/data/catalog_profile_jobs.py | 17 ++++++- backend/app/data/source_maintenance.py | 10 +++- backend/tests/test_catalog_observe.py | 21 ++++++++ backend/tests/test_source_maintenance.py | 26 ++++++++++ frontend/src/api/client.ts | 2 +- frontend/src/lib/terms.ts | 7 +++ frontend/src/pages/catalog/ProfileDialog.tsx | 39 +++++++++------ frontend/src/pages/catalog/profile.ts | 16 ++++-- frontend/src/pages/catalog/profileJob.ts | 2 +- scripts/check-catalog-profile.mjs | 51 +++++++++++++++++++- 11 files changed, 170 insertions(+), 26 deletions(-) diff --git a/backend/app/api/catalog.py b/backend/app/api/catalog.py index 0526ca13..8810486c 100644 --- a/backend/app/api/catalog.py +++ b/backend/app/api/catalog.py @@ -114,8 +114,9 @@ GUIDANCE_NOT_TEXT = "起草要求要是一段文字（要清除起草要求请 class ProfileJobIn(BaseModel): #: 要剖析的表（范围 selected，按给的顺序去重）；最多 500 张（超了 422，说明是中文） tables: list[str] | None = None - #: 不指定表时的范围：needed 需要剖析的表（缺省）、all 全部表（都不含视图、没有可剖析的项的表） - scope: Literal["needed", "all"] | None = None + #: 不指定表时的范围：needed 需要剖析的表（缺省）、all 全部表（都不含视图、没有可剖析的项的表）、observe 取值观察（只观察 + #: 有数据目录、观察到期的表，不剖析；见 data/catalog_profile_observe）。maintenance 是定时维护用的，这里不收 + scope: Literal["needed", "all", "observe"] | None = None async def _resolved(session: AsyncSession, source_id: str) -> tuple[DataSource, Any]: diff --git a/backend/app/data/catalog_profile_jobs.py b/backend/app/data/catalog_profile_jobs.py index e24d2c19..37a4b097 100644 --- a/backend/app/data/catalog_profile_jobs.py +++ b/backend/app/data/catalog_profile_jobs.py @@ -692,6 +692,18 @@ async def _behind_draft(session: AsyncSession, source_id: str) -> bool: return catalog_jobs.sync_drafting(source_id) or await catalog_jobs.active_job(session, source_id) is not None +async def nothing_reason(session: AsyncSession, row: Any, source: Any, scope: str) -> str | None: + """按此刻的表结构和目录，这个范围一张表都挑不出来时为什么（ProfileNothing 的原话）；挑得出、或者排在起草后面（开始时按 + 起草后的目录再挑，现在说不准）返回 None。定时维护建出 0 张表的任务时用它把原因记进触发记录。""" + if await _behind_draft(session, row.id): + return None + try: + await _pick(session, row, source, tables=None, scope=scope) + except ProfileNothing as e: + return str(e) + return None + + async def create_job(session: AsyncSession, row: Any, source: Any, *, tables: list[str] | None, scope: str, settings: ProfileSettings, actor: str | None, retry_of: str | None = None, trigger: str = "manual", fire_at: str | None = None, schedule_id: str | None = None, @@ -1161,5 +1173,6 @@ __all__ = ["INTERRUPTED_ERROR", "KEEP_JOBS", "MAX_TABLES", "NOTHING_AT_ALL", "NO "NOTHING_TO_OBSERVE", "NothingToRetry", "OBSERVING_SCOPES", "SCOPES", "task_modes", "trigger_of", "ProfileBusy", "ProfileFinished", "ProfileJobs", "ProfileNothing", "THEN_PROFILE", "active_job", "busy_message", "cancel", "create_job", "due_reason", "ensure_free", "job_brief", "job_dict", "job_out", - "last_detail", "last_facts", "last_profiles", "outside_of", "latest_job", "load", "load_task", "manager", "prune", "public_last", - "retry_tables", "startup", "task_detail", "task_row", "task_rows", "unsaved_error", "waiting_for"] + "last_detail", "last_facts", "last_profiles", "outside_of", "latest_job", "load", "load_task", "manager", + "nothing_reason", "prune", "public_last", "retry_tables", "startup", "task_detail", "task_row", "task_rows", + "unsaved_error", "waiting_for"] diff --git a/backend/app/data/source_maintenance.py b/backend/app/data/source_maintenance.py index a2bba31e..aa8597bf 100644 --- a/backend/app/data/source_maintenance.py +++ b/backend/app/data/source_maintenance.py @@ -13,7 +13,10 @@ observe 取值观察，可多选，没设过是三步都做。 **到点**（fire，持着 scheduler 的 launch_lock）：按顺序建探查任务 → 剖析任务（范围 needed / observe / maintenance，按选的 步骤），任务上记 trigger=schedule 和 fire_at，靠 SourceLanes 串行：剖析排在同一次触发的探查后面（等探查进了同源队才建剖析 -任务），开始时按那时的表结构和目录挑表（挑出 0 张也照建，任务以 0 张表结束、提醒里写明为什么）。预算、止损照剖析设置。 +任务），开始时按那时的表结构和目录挑表（挑出 0 张也照建，任务以 0 张表结束、提醒里写明为什么）。建任务时就判断得出会是 0 张 +的（这次没有探查、也没排在起草后面，开始时表结构和目录都不会变——比如有数据目录的表都在 12 小时内观察过），原因另记进触发 +记录的 inputs.skipped 和 reason；同一次触发里有探查、或者排在起草后面时说不准（探查完、起草完可能就有了），保持现状，只在 +任务的提醒里写。预算、止损照剖析设置。 照实记没执行的：数据源不存在、已停用、是上传表格或记录表时整次不执行（failed_to_start）；剖析没开时剖析、观察两步不执行、 探查照做；上一次的维护（或别的探查、剖析）还没做完时这次跳过（skipped_overlap）。每次触发记下建了哪几个任务、哪几步没执行、 为什么（触发记录的 inputs：{steps, jobs, skipped}）。 @@ -221,6 +224,11 @@ async def _start(source_id: str, steps: list[str], *, actor: str, fire_at: str, table_versions.SnapshotMissing) as e: skipped += [{"step": s, "reason": str(e)} for s in wanted] return jobs, skipped + if not tasks and introspect_id is None and (why := await catalog_profile_jobs.nothing_reason( + session, row, view, scope)): + # 建任务时就挑不出表、这次又没有探查（开始时表结构不会变）、也没排在起草后面（目录不会变）：任务会以 0 张表 + # 结束，原因照实记进触发记录，不只在剖析任务的提醒里。任务照建（触发照旧是 started） + skipped += [{"step": s, "reason": why} for s in wanted] catalog_profile_jobs.manager.start(job.id) jobs.append({"step": "+".join(wanted), "kind": "profile", "id": job.id, "scope": scope, "tables": len(tasks)}) return jobs, skipped diff --git a/backend/tests/test_catalog_observe.py b/backend/tests/test_catalog_observe.py index 76abfad0..689cfe74 100644 --- a/backend/tests/test_catalog_observe.py +++ b/backend/tests/test_catalog_observe.py @@ -271,6 +271,27 @@ def test_observation_today_is_the_server_local_date(monkeypatch): time.tzset() +async def test_the_profile_jobs_api_takes_the_observe_scope(client, shop): + """取值观察以前只能经定时维护触发：POST /catalog/profile-jobs 的 scope 只收 needed、all（下一轮验收）。现在也收 observe： + 只观察、不剖析；12 小时内都观察过时 409 profile_nothing，说清为什么。""" + sid, _ = shop + r = await client.post(f"/api/datasources/{sid}/catalog/profile-jobs", json={"scope": "observe"}) + assert r.status_code == 202, r.text + job = r.json()["job"] + assert job["scope"] == "observe" + await catalog_profile_jobs.manager.wait(job["id"], timeout=60) + async with SessionLocal() as session: + [task] = (await session.execute(select(CatalogProfileTask).where( + CatalogProfileTask.job_id == job["id"]))).scalars() + assert {c["kind"] for c in task.checks} == set(observe.OBSERVE_KINDS), "只观察：不做关系、码值、日期的剖析" + assert {o.kind for o in await _obs(sid)} == set(observe.OBSERVE_KINDS) + again = await client.post(f"/api/datasources/{sid}/catalog/profile-jobs", json={"scope": "observe"}) + assert again.status_code == 409 and again.json()["code"] == "profile_nothing" + assert f"{observe.OBSERVE_EVERY_H} 小时内都观察过" in again.json()["detail"] + bad = await client.post(f"/api/datasources/{sid}/catalog/profile-jobs", json={"scope": "maintenance"}) + assert bad.status_code == 422, "maintenance 是定时维护用的，接口不收" + + class _FakeCx: """只看拼出来的 SQL：大表、业务日期没有索引。""" diff --git a/backend/tests/test_source_maintenance.py b/backend/tests/test_source_maintenance.py index 468c580c..73f76413 100644 --- a/backend/tests/test_source_maintenance.py +++ b/backend/tests/test_source_maintenance.py @@ -111,6 +111,32 @@ async def test_fire_builds_introspect_then_profile_in_order(client, tmp_path, ma await drop_source(sid) +async def test_observing_again_within_the_window_records_why_in_the_fire(client, tmp_path, made): + """只选取值观察：12 小时内观察过的表不再观察，任务以 0 张表结束。建任务时就判断得出（这次没有探查、也没排在起草后面， + 开始时表结构、目录不会变）：原因记进触发记录的 inputs.skipped 和 reason（下一轮验收：以前只在剖析任务的提醒里）。 + 任务照建（触发照旧是 started，不当成没能发起）。""" + sid = await _source(client, tmp_path / "c.db") + try: + async with SessionLocal() as session: + row = await session.get(DataSource, sid) + row.options = sm.with_steps(row.options, ["observe"], actor="测试员") + await session.commit() + schedule_id = await _schedule(sid) + made.append(schedule_id) + at = utcnow().replace(microsecond=0) + first = await service.fire_once(schedule_id, at, "manual") + assert first.status == "started" and not first.inputs.get("skipped"), first.inputs + await _wait_jobs(first) + second = await service.fire_once(schedule_id, at + timedelta(seconds=1), "manual") + await _wait_jobs(second) + assert second.status == "started" and [j["tables"] for j in second.inputs["jobs"]] == [0] + [skipped] = second.inputs["skipped"] + assert skipped["step"] == "observe" and "12 小时内都观察过" in skipped["reason"], skipped + assert "取值观察没有执行" in second.reason + finally: + await drop_source(sid) + + async def test_profile_off_records_steps_not_run_and_still_introspects(client, tmp_path, made): sid = await _source(client, tmp_path / "b.db", profile=False) try: diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index 63dc10ed..13c722d2 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -530,7 +530,7 @@ export const api = { * 409 带机读码：profile_disabled、datasource_inactive、schema_missing、snapshot_tampered、profile_busy（这个源已有排队、 * 进行中的剖析任务，detail 写明进行到第几张）、profile_nothing（按范围挑出 0 张表，detail 写明原因和建议） */ - create: (sourceId: string, body: { tables?: string[]; scope?: 'needed' | 'all' }) => + create: (sourceId: string, body: { tables?: string[]; scope?: 'needed' | 'all' | 'observe' }) => post<{ job: CatalogProfileJob }>(`${catalogBase(sourceId)}/profile-jobs`, body), /** 排队或进行中的那个剖析任务；没有为 null */ active: (sourceId: string, opts?: RequestOptions) => diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index a1143961..cd426a7a 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -3035,6 +3035,11 @@ export const PROFILE_TEXT = { scopeNeededNone: '没有需要剖析的表：都剖析过、上次都做完了，也没有新的待核实关联关系和新的码值、日期列', scopeAll: (n: number) => `全部 ${formatNumber(n)} 张表`, scopeAllHint: '有可剖析项的表，不含视图', + /** 取值观察（范围 observe）：只观察、不剖析。张数是有数据目录的表，其中最近观察过的服务端这次不观察 */ + scopeObserve: (n: number) => `取值观察：有数据目录的 ${formatNumber(n)} 张表`, + scopeObserveHint: (hours: number) => '只观察、不剖析：码值分布、最新业务日期、按日行数和行数，和同口径的历史比出取值变化，' + + `列在「变化」里。最近 ${formatNumber(hours)} 小时内观察过的表这次不再观察`, + scopeObserveNone: '还没有有数据目录的表：取值观察只看有数据目录的表，请先起草数据目录', /** 已选、筛选结果里有剖析不了的表：不交给服务端，写明有几张 */ scopeLeftOut: (n: number) => `另有 ${formatNumber(n)} 张不剖析（视图、表结构中已没有，或没有可剖析的项）`, scopeEmpty: '这个范围里没有可剖析的表', @@ -3230,6 +3235,7 @@ export const PROFILE_TEXT = { busy: '这个数据源已有剖析任务', tampered: '数据文件核对未通过', nothing: '没有可剖析的表', + nothingObserve: '没有要观察的表', other: '无法开始剖析', } as Record, blockedNext: { @@ -3239,6 +3245,7 @@ export const PROFILE_TEXT = { busy: '同一个数据源同一时间只有一个剖析任务：可以查看它的进度，等它结束后再开始新的剖析。', tampered: '请重新上传表格，或联系管理员核对数据文件。', nothing: '可以改选「全部表」，或在表清单中选定要剖析的表。', + nothingObserve: '观察过的表要过一段时间才再观察；要剖析，可以回到上一步改选「需要剖析的表」或「全部表」。', other: '请稍后重试。', } as Record, busyAt: (who: string | null, done: number, total: number) => diff --git a/frontend/src/pages/catalog/ProfileDialog.tsx b/frontend/src/pages/catalog/ProfileDialog.tsx index 24d253dc..5eba1092 100644 --- a/frontend/src/pages/catalog/ProfileDialog.tsx +++ b/frontend/src/pages/catalog/ProfileDialog.tsx @@ -20,8 +20,8 @@ import { } from '../../lib/terms' import { STATUS_TONE, StatusChip } from './parts' import { - PROFILE_MAX_TABLES, PROFILE_RETRYABLE, UNFINISHED_REASONS, coverageText, fillKey, isProfilable, profileBlockOf, profileScopeTables, - secondsText, sizeText, spanText, + PROFILE_MAX_TABLES, PROFILE_OBSERVE_EVERY_H, PROFILE_RETRYABLE, UNFINISHED_REASONS, coverageText, fillKey, isProfilable, profileBlockOf, + profileScopeTables, secondsText, sizeText, spanText, } from './profile' import type { ProfileBlock } from './profile' import { isJobActive } from './draftJob' @@ -32,16 +32,18 @@ import type { ProfileStartBody } from './profileJob' // 数据目录页的「数据剖析」。剖析是服务端的后台任务（profileJob.ts）：这里选范围、看预算、建任务；之后显示页面跟着的 // 那个任务（进行中的逐表进度、结束后的结果），任务本身的状态在页面上，关掉对话框照常进行。照起草对话框的结构： // -// 开始前：范围（已选 / 当前筛选 / 需要剖析的表（默认）/ 全部表，个数按清单上每行的 profile_due、profile_checks 算）、按表写的 -// 预算和这次最多约几条查询、醒目一句「只读查询、一条一条发、可随时停止」；同一个数据源有起草任务时按钮写「排队剖析」。 +// 开始前：范围（已选 / 当前筛选 / 需要剖析的表（默认）/ 全部表 / 取值观察，个数按清单上每行的 profile_due、profile_checks、 +// 有没有数据目录算；取值观察只观察不剖析，最近观察过的服务端这次不观察，张数是上限）、按表写的预算和这次最多约几条查询、 +// 醒目一句「只读查询、一条一条发、可随时停止」；同一个数据源有起草任务时按钮写「排队剖析」。 // 进行中：几张 / 共几张、已发几条查询、用时，按表数的真进度条，停止；逐表一行（状态、各类检查的进度、查询数、用时、正在做 // 哪一项），点一行展开这张表的检查清单、发现和发过的查询。 // 结束后：汇总（各状态几张、核实了什么）、止损和停止的原因照实写、按「有发现 / 有跳过 / 失败」筛、重试没做完的表；码值的 // 「填写含义」照旧（填完回到这里，接着下一列）。 -// 开始不了（409）：每种情况一句说法和下一步（未开启就地打开剖析设置；正忙写清是谁的、到哪了，可以去看它的进度）。 +// 开始不了（409）：每种情况一句说法和下一步（未开启就地打开剖析设置；正忙写清是谁的、到哪了，可以去看它的进度；取值观察 +// 挑不出表时说「没有要观察的表」，不给「改选全部表」）。 // =========================================================================== -type Scope = 'selected' | 'filtered' | 'needed' | 'all' +type Scope = 'selected' | 'filtered' | 'needed' | 'all' | 'observe' type Filter = 'all' | 'findings' | 'skipped' | 'failed' const FILTERS: readonly Filter[] = ['all', 'findings', 'skipped', 'failed'] @@ -114,6 +116,10 @@ export function ProfileDialog({ tables: needed, leftOut: 0, body: { scope: 'needed' } }) const all = profileScopeTables(rows, 'all') out.push({ key: 'all', label: PT.scopeAll(all.length), hint: PT.scopeAllHint, tables: all, leftOut: 0, body: { scope: 'all' } }) + const watched = profileScopeTables(rows, 'observe') + out.push({ key: 'observe', label: PT.scopeObserve(watched.length), + hint: watched.length ? PT.scopeObserveHint(PROFILE_OBSERVE_EVERY_H) : PT.scopeObserveNone, tables: watched, leftOut: 0, + body: { scope: 'observe' } }) return out }, [rows, visible, selected, filtered]) const [scope, setScope] = useState(() => { @@ -260,8 +266,9 @@ export function ProfileDialog({
{scopes.map((s) => { const over = !!s.body.tables && s.tables.length > PROFILE_MAX_TABLES - const note = over ? PT.scopeTooMany(PROFILE_MAX_TABLES) : !s.tables.length && s.key !== 'needed' ? PT.scopeEmpty - : s.leftOut ? PT.scopeLeftOut(s.leftOut) : s.hint + const note = over ? PT.scopeTooMany(PROFILE_MAX_TABLES) + : !s.tables.length && s.key !== 'needed' && s.key !== 'observe' ? PT.scopeEmpty + : s.leftOut ? PT.scopeLeftOut(s.leftOut) : s.hint return (

{distText(s.verdicts as Record, T.verdict)}

- {!!s.not_run &&

{O.notRun(s.not_run)}

} + {!!s.not_run &&

{O.notRun(s.not_run)}

} + {!!s.errors &&

{O.errors(s.errors)}

}

{O.outcomes}：{distText(s.outcomes as Record, T.outcome)}

{s.text && Object.keys(s.text).length > 0 &&

{O.text}：{distText(s.text as Record, T.textOutcome)}

}
diff --git a/frontend/src/typesEval.ts b/frontend/src/typesEval.ts index 4bb426fb..39d3e185 100644 --- a/frontend/src/typesEval.ts +++ b/frontend/src/typesEval.ts @@ -393,11 +393,14 @@ export interface EvalSpend { export type EvalStatus = 'queued' | 'running' | 'succeeded' | 'partial' | 'failed' | 'cancelled' | 'interrupted' export type EvalTrigger = 'manual' | 'schedule' | 'pair' +/** 一臂的计数（服务端 runner._counts）：done 是真的跑过的（评完、出错），not_run 是停止、中断、到了上限、止损没跑的， + * errors 是出错的（不含没跑的）；done + not_run + running + pending = total */ export interface EvalCounts { total?: number done?: number running?: number pending?: number + not_run?: number passed?: number failed?: number ungradable?: number @@ -428,7 +431,9 @@ export interface EvalSummary { done?: number cases?: number verdicts?: Partial> + /** 没跑的（停止、中断、到了上限、止损没开评）；出错的另在 errors */ not_run?: number + errors?: number accuracy?: EvalRate traced_accuracy?: EvalRate outcomes?: Partial> diff --git a/scripts/check-evals.mjs b/scripts/check-evals.mjs index 9e70dcdf..db3f53b1 100644 --- a/scripts/check-evals.mjs +++ b/scripts/check-evals.mjs @@ -362,7 +362,11 @@ function reply(state, method, path, url, body) { } if (method === 'POST' && rest === '/stop') { write() - Object.assign(ev, { status: 'cancelled', running: false, current: [], stop_reason: null, + // 服务端（runner._counts、_close）：停下的原因记 stop_reason、不当成出错；没跑的题单独计 not_run，不进 errors，done 不等于 total + Object.assign(ev, { status: 'cancelled', running: false, current: [], stop_reason: '评测已停止', error: null, + counts: { total: 3, done: 1, not_run: 2, running: 0, pending: 0, passed: 1, failed: 0, ungradable: 0, skipped: 0, errors: 0 }, + summary: { ...clone(SUMMARY), items: 3, done: 3, verdicts: { pass: 1, fail: 0, ungradable: 0, skipped: 0 }, not_run: 2, errors: 0, + accuracy: RATE(1, 1), traced_accuracy: RATE(1, 1) }, actions: { stop: false, retry: true, rescore: true, pin: false, unpin: false, open: 2, cleaned: false } }) return { json: ev } } @@ -743,6 +747,11 @@ await section('评测详情：进度、正在做什么、停止和重试、重 await page.locator('[data-eval-retry]').waitFor() check('停止之后：服务端说能重试，给「重试没做完的」，悬停写几项', (await page.locator('[data-eval-retry]').getAttribute('title')).includes('2 项') && writes(state, /\/stop$/).length === 1) + check('……停下的原因照服务端写「评测已停止」，不当成出错；没评的 2 项单列，不算出错', await until(async () => + (await page.locator('[data-eval-stop-reason]').count()) === 1) + && (await text(page, '[data-eval-stop-reason]')).includes('评测已停止') && await page.locator('[data-eval-error]').count() === 0 + && (await text(page, '[data-eval-not-run]')).startsWith('2 项没评') && await page.locator('[data-eval-errors]').count() === 0, + await text(page, '[data-eval-run-head]')) await page.locator('[data-eval-retry]').click() await until(() => writes(state, /\/retry$/).length === 1) check('重试：发了一次', writes(state, /\/retry$/).length === 1) From 8f79098b6f7e026f57a5a6960f4dbe0a9c00d834 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Sat, 17 Oct 2026 09:12:29 +0800 Subject: [PATCH 12/15] =?UTF-8?q?=E8=BF=90=E8=A1=8C=E6=97=B6=20db=5Fschema?= =?UTF-8?q?=20=E6=8C=89=E9=9C=80=E5=8A=A0=E5=85=A5=E6=8E=A2=E6=9F=A5?= =?UTF-8?q?=E8=8C=83=E5=9B=B4=E7=9A=84=E7=95=99=E7=97=95=E8=AE=B0=E4=B8=8A?= =?UTF-8?q?=E8=BF=90=E8=A1=8C=E7=9A=84=E5=8F=91=E8=B5=B7=E4=BA=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 验收：db_schema 点名探查范围外的表、当场加进范围时，留痕（source_objects 的 added_by，trail_dict 的 by）是 null， 只有 run_id；look 那条路记着对话里的发起人。tools/datasource._on_demand 建 Trail 时没传 actor。 改法：查不在的原因时顺手按 ctx.run_id 取 Run.started_by，作为 Trail.actor 传下去。没有运行（ctx 没给 run_id）时照旧 为空。 测试：test_catalog_schema_truncated 新增一条：建一次 started_by 为「分析员甲」的运行，用它的 ToolContext 让 db_schema 点名一张范围外的表，留痕的 via 是 db_schema、run_id、ref（节点 id）对得上，by 是发起人。去掉修复回跑挂。 --- backend/app/tools/datasource.py | 8 ++++- .../tests/test_catalog_schema_truncated.py | 29 +++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/backend/app/tools/datasource.py b/backend/app/tools/datasource.py index ca79a969..fe249549 100644 --- a/backend/app/tools/datasource.py +++ b/backend/app/tools/datasource.py @@ -968,12 +968,18 @@ async def _on_demand(source: Any, missing: list[str], ctx: ToolContext | None) - from app.db.base import SessionLocal from app.look.base import ON_DEMAND_PER_LOOK, ON_DEMAND_PER_RUN + from app.db.models import Run + out: dict[str, str] = {} lines: list[str] = [] + starter: str | None = None try: async with SessionLocal() as session: reasons = await so.absent_reasons(session, source, missing) used = await so.count_added_in_run(session, ctx.run_id) if ctx is not None and ctx.run_id else 0 + # 留痕记上运行的发起人（同 look 那条路记对话里的发起人）：不然只有 run_id，要回头去翻运行才知道是谁 + if ctx is not None and ctx.run_id: + starter = await session.scalar(select(Run.started_by).where(Run.id == ctx.run_id)) except Exception: # noqa: BLE001 - 查不了原因就照旧说 logger.exception("db_schema 查表不在的原因失败（数据源 %s）", getattr(source, "name", "?")) return out, lines @@ -992,7 +998,7 @@ async def _on_demand(source: Any, missing: list[str], ctx: ToolContext | None) - out[t] = text if take: got = await so.add_to_scope(getattr(source, "id", ""), take, trail=so.Trail( - via=so.VIA_SCHEMA_TOOL, run_id=(ctx.run_id or None) if ctx else None, + via=so.VIA_SCHEMA_TOOL, actor=starter, run_id=(ctx.run_id or None) if ctx else None, ref=(ctx.node_id or None) if ctx else None, need=f"运行时 {SCHEMA_PREFIX}{getattr(source, 'name', '')} 点名：{'、'.join(take)}")) if got.cache is not None and got.added: diff --git a/backend/tests/test_catalog_schema_truncated.py b/backend/tests/test_catalog_schema_truncated.py index e4a9ba70..beb9b500 100644 --- a/backend/tests/test_catalog_schema_truncated.py +++ b/backend/tests/test_catalog_schema_truncated.py @@ -147,6 +147,35 @@ async def test_schema_tool_adds_an_out_of_scope_table_instead_of_denying_it(wide assert "没有 nope" in await tool.coroutine(table="nope") +async def test_schema_tool_on_demand_trail_records_who_started_the_run(wide): + """运行时 db_schema 按需加入范围的留痕：记上运行的发起人（by），不只是 run_id（下一轮验收：以前 by 是 null）。""" + from sqlalchemy import select as _select + + from app.data import source_objects + from app.db.models import Run + from app.db.models_sources import SourceObject + from app.tools.datasource import build_datasource_tools + from app.tools.registry import ToolContext, Versions + + sid, name = wide + async with SessionLocal() as session: + run = Run(workflow_name="按需加入", status="running", graph={"nodes": [], "edges": []}, input={}, + started_by="分析员甲") + session.add(run) + await session.commit() + run_id = run.id + async with SessionLocal() as session: + (tool,) = await build_datasource_tools( + [f"db_schema__{name}"], ToolContext(run_id=run_id, node_id="ask", data_versions=Versions.CURRENT), session) + assert "按需加入了探查范围" in await tool.coroutine(table="t204") + async with SessionLocal() as session: + obj = (await session.execute(_select(SourceObject).where(SourceObject.source_id == sid, + SourceObject.name == "t204"))).scalar_one() + trail = source_objects.trail_dict(obj) + assert trail["via"] == source_objects.VIA_SCHEMA_TOOL and trail["run_id"] == run_id and trail["ref"] == "ask" + assert trail["by"] == "分析员甲", trail + + def test_legacy_truncated_cache_still_does_not_deny(): """升级前只探查了前 200 张的老缓存（没有对象索引）：查一张没探查到的表照旧说「只探查了」、请查数据字典。""" from types import SimpleNamespace From 590bfc9a59d586b1cdadb7e71ae1dcde1af23142 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Sat, 17 Oct 2026 09:15:07 +0800 Subject: [PATCH 13/15] =?UTF-8?q?=E5=8F=A3=E5=BE=84=E5=8F=AC=E5=9B=9E?= =?UTF-8?q?=E7=9A=84=E9=80=9A=E7=9F=A5=E6=8C=89=E7=A7=8D=E7=B1=BB=E5=88=86?= =?UTF-8?q?=E5=BC=80=E5=86=99=E7=AD=94=E6=A1=88=E7=9A=84=E6=9D=A1=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 验收：ledger.recall 消息里写「答案：2 个」，实际是 1 个问数回答加 1 个正式运行。notify_recall 拿 _uses_of 的长度直接 拼成「N 个」，不分种类。 改法：notify/service 加 _answers_text，按用法的种类数（ledger.uses.KINDS 的顺序、KIND_TEXT 的叫法，同版本用量和召回 批次的说法）写成「问数回答 1 个、正式运行 1 个」；全局渠道按这一批汇总、个人渠道只数他的，两处都用它。只写条数，照旧 不放旧值、新值、差值和原答案的值。 测试（test_ledger_recall）：新增一条：同一周一个问数回答、一次正式运行，升版召回之后全局渠道收到的「答案」一行是 「问数回答 1 个、正式运行 1 个」，消息里没有数；原来两条按「1 个」「2 个」断言的改成带种类的写法。去掉修复回跑挂 3 条。 --- backend/app/notify/service.py | 25 ++++++++++---- backend/tests/test_ledger_recall.py | 52 +++++++++++++++++++++++++++-- 2 files changed, 68 insertions(+), 9 deletions(-) diff --git a/backend/app/notify/service.py b/backend/app/notify/service.py index 6ec395f5..462f4643 100644 --- a/backend/app/notify/service.py +++ b/backend/app/notify/service.py @@ -342,6 +342,18 @@ async def notify_task(task_id: str, event: str, *, users: list[str], exclude: fr task_id=task_id) +def _answers_text(uses: list[dict[str, Any]]) -> str: + """召回消息里「答案」那一行：按种类分开写条数（「问数回答 1 个、正式运行 1 个」，种类的叫法同版本用量 ledger.uses.KIND_TEXT）， + 只写条数、不放数。以前合成一个「2 个」，一个问数回答加一次正式运行读起来像两个问数回答。一个都没有写「0 个」。""" + from collections import Counter + + from app.ledger.uses import KIND_TEXT, KINDS + + counts = Counter(str(u.get("kind") or "") for u in uses) + order = [k for k in KINDS if counts.get(k)] + sorted(k for k in counts if k not in KINDS) + return "、".join(f"{KIND_TEXT.get(k, k or '答案')} {counts[k]} 个" for k in order) or "0 个" + + async def notify_recall(task_id: str, users: list[str]) -> int: """口径召回的待办开出来了（app/ledger/recall）：口径升版、数真变了的那一种推 ledger.recall——订阅了它的全局渠道、答案主人 自己的个人渠道；然后照别的待办推 task.opened（已经推过 ledger.recall 的渠道不再推，同一个渠道只推一次）。目录、表结构变了 @@ -352,8 +364,9 @@ async def notify_recall(task_id: str, users: list[str]) -> int: 答案、待办按这一批汇总，链到待办箱（各人在那里看到派给自己的那条）；别的几条连 task.opened 也不再推全局渠道。个人渠道 照旧每个主人各一条，只数他的答案、链到他的那条待办。 - 消息照模块说明的规矩（X6）：口径名（搭建时起的工作流名）、新旧版本号、几个答案（几条待办）、召回编号、去处理的链接；不放 - 旧口径的值、新口径的值、差值和原答案的值。返回推了几个渠道。""" + 消息照模块说明的规矩（X6）：口径名（搭建时起的工作流名）、新旧版本号、几个答案（按种类分开写条数：问数回答几个、正式运行 + 几个……，见 _answers_text；几条待办）、召回编号、去处理的链接；不放旧口径的值、新口径的值、差值和原答案的值。返回推了几个 + 渠道。""" from app.ledger import recall try: @@ -387,13 +400,13 @@ async def notify_recall(task_id: str, users: list[str]) -> int: head = [("状态", "待处理"), ("口径", f"「{name[:80]}」"), ("版本", f"v{origin.get('from_version')} → v{origin.get('to_version')}")] tail = [("召回", f"#{task.run_id[:6]}")] - mine = len(recall._uses_of(origin, recall._owner_of_key(key))) - everyone = len(recall._uses_of(origin, ("anyone", ""))) + mine = _answers_text(recall._uses_of(origin, recall._owner_of_key(key))) + everyone = _answers_text(recall._uses_of(origin, ("anyone", ""))) - def message(answers: int, tasks: int, link: tuple[str, str] | None) -> dict[str, Any]: + def message(answers: str, tasks: int, link: tuple[str, str] | None) -> dict[str, Any]: return {"event": event, "status": "pending", "ok": False, "title": recall.CHANGED_TITLE.format(name=name[:80] or "口径", version=origin.get("to_version")), - "lines": [*head, ("答案", f"{answers} 个"), *([("待办", f"{tasks} 条")] if tasks > 1 else []), *tail], + "lines": [*head, ("答案", answers), *([("待办", f"{tasks} 条")] if tasks > 1 else []), *tail], "links": [link] if base and link else [], "subject": {"task_id": task_id, "run_id": task.run_id, "workflow_id": origin.get("workflow_id")}} diff --git a/backend/tests/test_ledger_recall.py b/backend/tests/test_ledger_recall.py index 0052f306..7e7232c7 100644 --- a/backend/tests/test_ledger_recall.py +++ b/backend/tests/test_ledger_recall.py @@ -464,11 +464,57 @@ async def test_recall_notification_has_no_numbers(monkeypatch): text = mine[0]["title"] + "".join(f"{k}{v}" for k, v in mine[0]["lines"]) for number in ("170.5", "208.5", "38.0", "38.00"): assert number not in text, f"消息里不放数：{number}" - assert dict(mine[0]["lines"]) == {"状态": "待处理", "口径": "「周销售」", "版本": "v1 → v2", "答案": "1 个", + assert dict(mine[0]["lines"]) == {"状态": "待处理", "口径": "「周销售」", "版本": "v1 → v2", "答案": "问数回答 1 个", "召回": dict(mine[0]["lines"])["召回"]} assert "周销售" in mine[0]["title"] and "v2" in mine[0]["title"] +async def test_recall_message_counts_answers_by_kind(monkeypatch): + """召回消息按种类分开写条数：一个问数回答、一次正式运行写「问数回答 1 个、正式运行 1 个」（下一轮验收：以前写「答案：2 个」， + 读的人以为是两个问数回答）。只写条数，照旧不放数。""" + from app.core.crypto import encrypt + from app.db.models import NotifyChannel + from app.engine.launch import launch_run + from app.notify import service + + sent: list[dict] = [] + + async def capture(channel, message): + sent.append({"channel": channel.name, **message}) + from app.notify.core import Outcome + + return Outcome(True, 200, 1, None) + + monkeypatch.setattr("app.notify.send", capture) + async with SessionLocal() as session: + row = NotifyChannel(name="召回-按种类", scope="global", url=encrypt("https://hooks.example.com/recall-kinds"), + events=["ledger.recall"]) + session.add(row) + await session.commit() + channel_id = row.id + try: + wf = await S.publish() + await answer(monkeypatch) + S.script(monkeypatch, "本周销售额 [[m:gmv]]。") + async with SessionLocal() as session: + formal = await launch_run(session, workflow_id=wf, run_class="formal", version=1, input_payload=WEEK1, + actor="analyst") + assert (await S.wait(formal.id)).status == "succeeded" + await publish_v2(wf) + await run_recall(wf) + await service.drain() + finally: + async with SessionLocal() as session: + await session.delete(await session.get(NotifyChannel, channel_id)) + await session.commit() + mine = [m for m in sent if m["channel"] == "召回-按种类"] + assert mine, sent + answers = [v for m in mine for k, v in m["lines"] if k == "答案"] + assert "问数回答 1 个、正式运行 1 个" in answers, answers + for number in ("170.5", "208.5", "38.0"): + assert all(number not in v for m in mine for _, v in m["lines"]), "消息里不放数" + + async def test_recall_pushes_the_shared_channel_once_per_recall_run(monkeypatch): """开登录时一组答案有几个主人就开几条待办，但全局渠道（团队群）按召回运行只收一条（答案、待办都按这次召回汇总，链到 待办箱）；订阅了 task.opened 的全局渠道也不再按每条待办各推一次。每个主人自己的个人渠道各收他那一条。""" @@ -522,12 +568,12 @@ async def test_recall_pushes_the_shared_channel_once_per_recall_run(monkeypatch) team = [m for m in sent if m["channel"] == "团队-召回"] assert [m["event"] for m in team] == ["ledger.recall"], [m["event"] for m in team] lines = dict(team[0]["lines"]) - assert lines["答案"] == "2 个" and lines["待办"] == "2 条", lines + assert lines["答案"] == "问数回答 2 个" and lines["待办"] == "2 条", lines assert [label for label, _ in team[0]["links"]] == ["去待办箱"] or team[0]["links"] == [] assert [m["event"] for m in sent if m["channel"] == "团队-待办"] == ["task.opened"], "只订阅了新待办的团队渠道也只收一条" for name in ("analyst_a", "analyst_b"): mine = [m for m in sent if m["channel"] == f"个人-{name}"] - assert [m["event"] for m in mine] == ["ledger.recall"] and dict(mine[0]["lines"])["答案"] == "1 个", mine + assert [m["event"] for m in mine] == ["ledger.recall"] and dict(mine[0]["lines"])["答案"] == "问数回答 1 个", mine # 账本接口里的召回待办只列看的这个人手上的：第一个答案的主人看得到自己那条，另一个主人看不到 from app.api.deps import Principal From d9eaf8b850d601327c768f32bfe8adca686c9174 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Sat, 17 Oct 2026 09:19:24 +0800 Subject: [PATCH 14/15] =?UTF-8?q?=E6=92=A4=E9=94=80=E5=88=86=E4=BA=AB?= =?UTF-8?q?=E9=93=BE=E6=8E=A5=E4=BA=A4=E5=9B=9E=E7=9A=84=20page=5Ftitle=20?= =?UTF-8?q?=E6=98=AF=E9=A1=B5=E9=9D=A2=E6=A0=87=E9=A2=98=EF=BC=8C=E5=92=8C?= =?UTF-8?q?=E5=BB=BA=E3=80=81=E5=88=97=E7=9A=84=E4=B8=80=E6=A0=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 验收：DELETE /api/apps/{app_id}/shares/{share_id} 的返回里 page_title 是页面 id（"home"），建分享时返回的是页面标题。 revoke_share 调 share_out(row) 时没给标题表，share_out 就退回页面 id。 改法：撤销照列表的取法（应用当前的版本 _previous 的 spec）给 share_out 标题表；撤销过的再撤销也一样。 测试：test_shares 的撤销用例加断言：建出来的 page_title 是「首页」，撤销、再撤销交回的都是「首页」。去掉修复回跑挂。 --- backend/app/api/shares.py | 7 +++++-- backend/tests/test_shares.py | 4 ++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/backend/app/api/shares.py b/backend/app/api/shares.py index 4bcc1b94..27006a5a 100644 --- a/backend/app/api/shares.py +++ b/backend/app/api/shares.py @@ -216,7 +216,9 @@ async def list_shares(app_id: str, session: AsyncSession = Depends(get_session), @_manage.delete("/{app_id}/shares/{share_id}") async def revoke_share(app_id: str, share_id: str, session: AsyncSession = Depends(get_session), x_actor: str | None = Header(default=None)) -> dict[str, Any]: - """撤销：之后打开回 410「链接已撤销」。撤销过的再撤销照旧交回它（不重复记审计）。管理员关了开关也能撤销。""" + """撤销：之后打开回 410「链接已撤销」。撤销过的再撤销照旧交回它（不重复记审计）。管理员关了开关也能撤销。交回的形状和 + 建、列的同一个（page_title 是页面标题，取法同列表：应用当前的版本）。""" + from app.api.apps import _previous from app.apps.audit import record app, _ = await _managed(session, app_id, x_actor) @@ -228,7 +230,8 @@ async def revoke_share(app_id: str, share_id: str, session: AsyncSession = Depen await record(session, action="app.share_revoke", subject_kind="view_share", subject_id=row.id, actor=actor_of(x_actor), data={"app_id": app.id, "page_id": row.page_id, "prefix": row.prefix}) await session.commit() - return share_out(row) + version = await _previous(session, app) + return share_out(row, _titles(version.spec if version is not None else None)) # -------------------------------------------------------------------------- diff --git a/backend/tests/test_shares.py b/backend/tests/test_shares.py index 28d110d0..49738185 100644 --- a/backend/tests/test_shares.py +++ b/backend/tests/test_shares.py @@ -326,6 +326,10 @@ async def test_revoked_and_expired_links_say_so_with_410(world): # noqa: F811 admin = await auth_env.login("root") r = await admin.delete(f"/api/apps/{app_id}/shares/{gone['id']}") assert r.status_code == 200 and r.json()["state"] == "revoked" and r.json()["revoked_by"] == "root" + # 撤销交回的那一条和建、列的同一个形状：page_title 是页面标题，不是页面 id（下一轮验收：以前撤销回的是「home」） + assert gone["page_title"] == "首页" and r.json()["page_title"] == gone["page_title"], r.json()["page_title"] + again = await admin.delete(f"/api/apps/{app_id}/shares/{gone['id']}") + assert again.json()["page_title"] == "首页", "撤销过的再撤销照旧交回它，标题一样" async with SessionLocal() as s: (await s.get(ViewShare, old["id"])).expires_at = utcnow() - timedelta(minutes=5) await s.commit() From 77b4c9c68ac15aad16676a52c287bea97118f94e Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Sat, 17 Oct 2026 09:19:24 +0800 Subject: [PATCH 15/15] =?UTF-8?q?E4=20=E7=9A=84=E7=89=88=E6=9C=AC=E7=94=A8?= =?UTF-8?q?=E9=87=8F=E5=8F=AA=E7=AE=97=E6=9C=89=E5=8F=A3=E5=BE=84=E5=8D=A1?= =?UTF-8?q?=E7=9A=84=E5=B7=A5=E4=BD=9C=E6=B5=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 验收：E4 的用量把没有口径卡的普通发布工作流也当成口径计数。_e4 按（工作流, 版本）数所有 run_class=formal 的运行， 不看那一版是不是口径。 改法：数完之后按 WorkflowVersion 的图过一遍，只留图上有源头口径卡的版本（recall.source_cards：metrics 节点、不是 转抄别处口径的 caliber_from，和召回认口径卡的同一个判断）。比对里等价已证明的运行数（proven_runs）本来就只来自口径， 不动。写进了模块说明。 测试：test_ledger_api 新增一条：一个有口径卡、一个没有的已发布工作流各跑两次正式运行，E4 用量里只有前者、formal_runs 为 2。去掉修复回跑挂。 --- backend/app/ledger/stats.py | 27 ++++++++++++++++++++++++--- backend/tests/test_ledger_api.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 3 deletions(-) diff --git a/backend/app/ledger/stats.py b/backend/app/ledger/stats.py index 180d984a..64f37605 100644 --- a/backend/app/ledger/stats.py +++ b/backend/app/ledger/stats.py @@ -17,7 +17,8 @@ 段对话的算重复的形状，给出这样的形状几个、有重复形状的运行占几成；问题聚类（app/chat/questions：全站唯一的聚类，只取问题 原文、不含 history、排除系统派生的运行）里 ≥ REPEAT_RUNS 个问题的类占几成。认证队列的转化（第 3 波，ledger/certify）：固化成 口径草稿几份（origin.freeze 的工作流）、其中发布了几份、点了「不是口径」几项。 -- E4 召回：每个口径版本被用了多少次（比对里等价已证明的运行数、这一版的正式运行数）；每次发布（口径升到某一版）召回了几组、 +- E4 召回：每个口径版本被用了多少次（比对里等价已证明的运行数、这一版的正式运行数——只算图上有源头口径卡的版本，没有口径卡 + 的普通发布工作流的正式运行不算）；每次发布（口径升到某一版）召回了几组、 收过尾的几组、其中数变了的几组和比例、重算失败的几组（ledger/recall：召回运行和它收尾时记的 ledger.recalled，第 4 波）。 按召回运行的发起时间筛窗口；表结构变了之后的口径试算不算召回。 """ @@ -289,8 +290,10 @@ async def _e4(session: Any, contents: list[dict[str, Any]], **scope: Any) -> dic stmt = stmt.where(Run.created_at >= scope["since"]) if scope.get("until") is not None: stmt = stmt.where(Run.created_at < scope["until"]) - formal = {ledger_labels.caliber_key(w, v): int(n) - for w, v, n in (await session.execute(stmt.group_by(Run.workflow_id, Run.version))).all()} + counted = {(w, v): int(n) for w, v, n in (await session.execute(stmt.group_by(Run.workflow_id, Run.version))).all()} + # 只算有口径卡的那几版：没有口径卡的普通发布工作流的正式运行不是「这个口径版本被用了」 + calibers = await _caliber_versions(session, counted) + formal = {ledger_labels.caliber_key(w, v): n for (w, v), n in counted.items() if (w, v) in calibers} recalls = int(await session.scalar(select(func.count()).select_from(Run).where(Run.origin_kind == "recall")) or 0) usage = [{"caliber": key, "workflow_id": key.rsplit("@", 1)[0], "version": int(key.rsplit("@", 1)[1]), "proven_runs": proven.get(key, 0), "formal_runs": formal.get(key, 0)} @@ -298,6 +301,24 @@ async def _e4(session: Any, contents: list[dict[str, Any]], **scope: Any) -> dic return {"usage": usage, "recall_runs": recalls, "per_publish": await _per_publish(session, **scope)} +async def _caliber_versions(session: Any, keys: Collection[tuple[str, int]]) -> set[tuple[str, int]]: + """这几个（工作流, 版本）里图上有源头口径卡的那些（recall.source_cards：metrics 节点、不是转抄别处的 caliber_from）。""" + from sqlalchemy import or_ + + from app.db.models import WorkflowVersion + from app.ledger.recall import source_cards + + keys = list(keys) + out: set[tuple[str, int]] = set() + for i in range(0, len(keys), 200): + chunk = keys[i:i + 200] + rows = (await session.execute(select(WorkflowVersion.workflow_id, WorkflowVersion.version, WorkflowVersion.graph) + .where(or_(*[(WorkflowVersion.workflow_id == w) & (WorkflowVersion.version == v) + for w, v in chunk])))).all() + out |= {(w, v) for w, v, graph in rows if source_cards(graph)} + return out + + async def _per_publish(session: Any, **scope: Any) -> dict[str, Any]: """每次发布召回了几组、其中数变了的比例（E4 下半）：按（工作流、新版本）分，读召回运行收尾时记的结论。""" from app.core.events import EventType diff --git a/backend/tests/test_ledger_api.py b/backend/tests/test_ledger_api.py index 2e5eeff2..3814e4e5 100644 --- a/backend/tests/test_ledger_api.py +++ b/backend/tests/test_ledger_api.py @@ -295,6 +295,35 @@ async def test_stats_count_what_the_fixtures_did(client, monkeypatch): assert body["e4"]["per_publish"]["available"] is True and body["e4"]["per_publish"]["groups"] == 0 +async def test_e4_usage_counts_formal_runs_only_of_workflows_with_a_caliber_card(client): + """E4 的版本用量只算有口径卡的工作流（下一轮验收：没有口径卡的普通发布工作流的正式运行也被当成口径计数）。卡是转抄别处 + 口径（caliber_from）的不算源头口径，同 recall.source_cards。""" + from datetime import datetime, timezone + + from app.db.models import Run, Workflow, WorkflowVersion + + since = datetime.now(timezone.utc) + card = {"id": "card", "type": "metrics", "position": {"x": 0, "y": 0}, + "data": {"label": "口径卡", "config": {"caliber": "周销售", "caliber_version": "v1", "metrics": []}}} + plain = {"id": "out", "type": "output", "position": {"x": 0, "y": 0}, "data": {"label": "出口", "config": {}}} + async with SessionLocal() as session: + made = {} + for name, nodes in (("有口径卡", [card, plain]), ("普通工作流", [plain])): + wf = Workflow(name=name, graph={"nodes": nodes, "edges": []}, status="published", published_version=1) + session.add(wf) + await session.flush() + session.add(WorkflowVersion(workflow_id=wf.id, version=1, graph={"nodes": nodes, "edges": []})) + for _ in range(2): + session.add(Run(workflow_id=wf.id, version=1, run_class="formal", status="succeeded", + graph={"nodes": nodes, "edges": []}, input={})) + made[name] = wf.id + await session.commit() + body = (await client.get("/api/ledger/stats", params={"since": since.isoformat()})).json() + usage = {u["caliber"]: u for u in body["e4"]["usage"]} + assert usage[f"{made['有口径卡']}@1"]["formal_runs"] == 2 + assert f"{made['普通工作流']}@1" not in usage, usage + + async def test_stats_take_the_run_ids_the_caller_names(monkeypatch): """评测（第 3 波）直接调 compute_stats，点名自己的运行：只算这几次。""" from app.ledger.stats import compute_stats