"""数据源的变化记录：接口、处理动作、给召回的拉取、通知（next-round 03 B3）。 SQLite 夹具（虚构的店铺库）探查建基线、写目录，再删一列、重新探查得到真实的结构变化；取值变化（新码值）直接经 source_changes.record 记一条。守的是： - GET /changes 缺省列开着的和知道了的，带各级别计数；状态、种类筛选；不认识的状态 422； - 处理：误报必须写理由（422），处理过的再处理 409；清理目录把影响面里的项从目录里去掉、记目录版本；加入码值走 apply_patch（含义待填写、依据写变化编号）、记目录版本；知道了只改状态；剖析没开时「重新剖析」照实 422； - 同一个变化再次测到只更新 last_seen_at、seen_count； - list_changes 按 seq 游标拉、affecting 按表反查、triggers_recall 只认结构类和触及规则的新码值； - source.changed 的推送：没有渠道订阅时推不出去；全局渠道订阅了，消息里只有源名、条数和链接（第 4 波打开了订阅）。 """ from __future__ import annotations import json import sqlite3 import uuid import httpx import pytest from httpx import ASGITransport, AsyncClient from sqlalchemy import select from app.data import catalog, source_changes as sc from app.db.base import SessionLocal from app.db.models import DataSource from app.db.models_sources import SourceChange from app.main import app from app.notify import core, service from tests.fixtures.sources import drop_source, introspect_now @pytest.fixture async def client(): async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: yield c @pytest.fixture async def shop(client, tmp_path): path = tmp_path / "shop.db" db = sqlite3.connect(path) db.executescript(""" CREATE TABLE orders (id INTEGER PRIMARY KEY, status VARCHAR(10), channel VARCHAR(8), amount NUMERIC(10, 2)); INSERT INTO orders VALUES (1, 'paid', 'web', 3), (2, 'void', 'shop', 4); """) db.commit() db.close() name = f"chg_{uuid.uuid4().hex[:8]}" r = await client.post("/api/datasources", json={"name": name, "kind": "sqlite", "database": str(path), "readonly": True}) sid = r.json()["id"] await introspect_now(client, sid) async with SessionLocal() as session: await catalog.write_entry(session, sid, "orders", { "label": catalog.make_item("订单", "human"), "columns": {"status": {"codes": catalog.make_item({"paid": "已支付", "void": "作废"}, "human"), "label": catalog.make_item("状态", "human")}, "channel": {"codes": catalog.make_item({"web": "线上"}, "profile", "proposed")}}, }, if_version=0, actor="测试员") db = sqlite3.connect(path) db.execute("ALTER TABLE orders DROP COLUMN status") db.commit() db.close() await introspect_now(client, sid) yield {"id": sid, "name": name} await drop_source(sid) async def _change(sid: str, kind: str) -> SourceChange: async with SessionLocal() as session: return (await session.execute(select(SourceChange).where( SourceChange.source_id == sid, SourceChange.kind == kind))).scalars().first() async def _codes_new(sid: str, *, rules: bool = False) -> SourceChange: async with SessionLocal() as session: row = await session.get(DataSource, sid) change, new = await sc.record(session, row, sc.NewChange( kind=sc.CODES_NEW, table="orders", column="channel", severity="warning", after={"codes": ["app"], "shares": {"app": 0.4}}, basis={"method": "full", "key": "full"}, affects={"catalog": [], "templates": [], "rules_touched": rules}, key=("full", ("app",))), sc.Detected(job_kind="profile", job_id="job-x", trigger="schedule")) await session.commit() assert new return change async def test_list_counts_and_filters(client, shop): sid = shop["id"] r = await client.get(f"/api/datasources/{sid}/changes") assert r.status_code == 200, r.text body = r.json() [removed] = [c for c in body["items"] if c["kind"] == "column_removed"] assert removed["schema"] == "agentlab-change/1" and removed["column"] == "status" and removed["status"] == "open" assert removed["severity"] == "action" and "删除了列" in removed["summary"] assert body["counts"]["pending"] == body["total"] and body["counts"]["by_severity"]["action"] >= 1 r = await client.get(f"/api/datasources/{sid}/changes", params={"kind": ["column_added"]}) assert r.json()["items"] == [] assert (await client.get(f"/api/datasources/{sid}/changes", params={"status": "nope"})).status_code == 422 one = (await client.get(f"/api/datasources/{sid}/changes/{removed['id']}")).json()["change"] assert one["id"] == removed["id"] async def test_dismiss_needs_a_reason_and_finished_changes_are_final(client, shop): sid = shop["id"] change = await _change(sid, "column_removed") url = f"/api/datasources/{sid}/changes/{change.id}/resolve" r = await client.post(url, json={"action": "dismiss"}) assert r.status_code == 422 and "理由" in r.json()["detail"] r = await client.post(url, json={"action": "dismiss", "note": "测试库里故意删的列"}, headers={"X-Actor": "tester"}) assert r.status_code == 200, r.text got = r.json()["change"] assert got["status"] == "dismissed" and got["resolution"]["note"] == "测试库里故意删的列" assert got["resolution"]["action"] == "dismiss" and got["resolution"]["by"] == "tester" assert (await client.post(url, json={"action": "acknowledge"})).status_code == 409 counts = (await client.get(f"/api/datasources/{sid}/changes")).json()["counts"] assert counts["by_severity"]["action"] == 0, "误报不再算要处理" async def test_clean_catalog_removes_the_items_and_records_the_catalog_version(client, shop): sid = shop["id"] change = await _change(sid, "column_removed") r = await client.post(f"/api/datasources/{sid}/changes/{change.id}/resolve", json={"action": "clean_catalog"}) assert r.status_code == 200, r.text body = r.json() async with SessionLocal() as session: entry = await catalog.read_entry(session, sid, "orders") assert "status" not in (entry.notes.get("columns") or {}), "说到被删的列的项都清掉了" assert entry.notes["label"]["value"] == "订单", "别的项不动" assert body["catalog_version"] == entry.version == body["change"]["resolution"]["catalog_version"] assert body["change"]["status"] == "resolved" async def test_add_codes_goes_through_apply_patch(client, shop): sid = shop["id"] change = await _codes_new(sid) r = await client.post(f"/api/datasources/{sid}/changes/{change.id}/resolve", json={"action": "add_codes"}) assert r.status_code == 200, r.text async with SessionLocal() as session: entry = await catalog.read_entry(session, sid, "orders") item = entry.notes["columns"]["channel"]["codes"] assert item["value"] == {"web": "线上", "app": ""}, "码值是补充：原来的留着，新码值含义待填写" assert (item["source"], item["status"]) == ("human", "confirmed") assert change.id[:6] in item["reason"], "确认时的依据写这条变化的编号" assert r.json()["catalog_version"] == entry.version removed = await _change(sid, "column_removed") r = await client.post(f"/api/datasources/{sid}/changes/{removed.id}/resolve", json={"action": "add_codes"}) assert r.status_code == 422, "只有新码值能加入码值" async def test_acknowledge_and_reprofile_when_profiling_is_off(client, shop): sid = shop["id"] change = await _codes_new(sid) url = f"/api/datasources/{sid}/changes/{change.id}/resolve" r = await client.post(url, json={"action": "reprofile"}) assert r.status_code == 422 and "剖析" in r.json()["detail"], "剖析没开：照实说做不了" r = await client.post(url, json={"action": "acknowledge"}) assert r.status_code == 200 and r.json()["change"]["status"] == "acknowledged" counts = (await client.get(f"/api/datasources/{sid}/changes")).json()["counts"] assert counts["pending"] > counts["open"], "知道了还没处理的照样算要人看" async def test_same_change_detected_again_only_bumps_seen_count(shop): sid = shop["id"] first = await _codes_new(sid) async with SessionLocal() as session: row = await session.get(DataSource, sid) again, new = await sc.record(session, row, sc.NewChange( kind=sc.CODES_NEW, table="orders", column="channel", severity="action", key=("full", ("app",))), sc.Detected(job_kind="profile")) await session.commit() assert not new and again.id == first.id and again.seen_count == 2 and again.severity == "action" async def test_cursor_pull_affecting_and_recall_triggers(shop): sid, name = shop["id"], shop["name"] codes = await _codes_new(sid, rules=True) async with SessionLocal() as session: everything = await sc.list_changes(session, source_id=sid) seqs = [c["seq"] for c in everything] assert seqs == sorted(seqs) and len(seqs) >= 2 rest = await sc.list_changes(session, source_id=sid, since_seq=seqs[0]) assert [c["seq"] for c in rest] == seqs[1:], "按 seq 游标：大于 since_seq 的" hit = await sc.affecting(session, name, ["ORDERS"]) assert {c["id"] for c in hit} == {c["id"] for c in everything}, "表名不分大小写" assert await sc.affecting(session, name, ["refunds"]) == [] assert await sc.affecting(session, name, ["orders"], since=seqs[-1]) == [] by = {c["kind"]: c for c in everything} assert sc.triggers_recall(by["column_removed"]), "结构类触发召回" assert sc.triggers_recall(next(c for c in everything if c["id"] == codes.id)), "触及有效记录条件的新码值触发召回" assert not sc.triggers_recall({"kind": "volume_drop", "affects": {}}), "纯取值漂移只算数据质量提醒" async def test_source_changed_message_has_only_name_counts_and_link(shop, monkeypatch): sid, name = shop["id"], shop["name"] assert await service.notify_source_changed(sid) == 0, "没有渠道订阅它：推不出去" got: list[httpx.Request] = [] def handler(request: httpx.Request) -> httpx.Response: got.append(request) return httpx.Response(200, json={"ok": True}) monkeypatch.setattr(core, "_transport", httpx.MockTransport(handler)) monkeypatch.setattr(core, "RETRY_DELAYS", (0, 0, 0)) async with SessionLocal() as session: from app.core.crypto import encrypt from app.db.models import NotifyChannel channel = NotifyChannel(name=f"变化-{uuid.uuid4().hex[:6]}", scope="global", format="generic", url=encrypt("https://hooks.example.com/agentlab/changes-7a1c"), events=["source.changed"]) session.add(channel) await session.commit() channel_id = channel.id try: assert await service.notify_source_changed(sid) == 1 [request] = got body = json.loads(request.content) text = body["title"] + body["text"] assert body["event"] == "source.changed" and name in body["title"] assert "要处理" in text and "orders" not in text and "status" not in text and "app" not in text, \ "不放表名、列名、取值" assert body["subject"] == {"source_id": sid} finally: async with SessionLocal() as session: from app.db.models import NotifyChannel row = await session.get(NotifyChannel, channel_id) await session.delete(row) await session.commit() def test_maintenance_option_is_hidden_from_the_datasource_form(): from app.api import datasources from app.data import source_maintenance assert source_maintenance.MAINTENANCE_OPTION in datasources._MANAGED_OPTIONS