mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-21 00:42:02 +00:00
补齐剖析任务的测试(设计文档 §7 后端那一串),顺手改掉几处过时的说明
新增 tests/test_catalog_profile_jobs.py,用很小的 SQLite 夹具和假的查询层让超时、失败、停在哪一条都可控: - 同源排队:起草在跑时剖析建成 queued、waiting_for = draft,起草结束后自动开始;剖析在跑时建起草任务不 409、排在后面(waiting_for = profile),同种任务照旧 409;停掉排队中的剖析任务(在队里等着的一起停掉,检查项不留「排队」);导入目录、同步起草在剖析时 409 profile_busy;同步起草、导入期间建的剖析任务排在它后面,导入期间再导入一次 409 - 停止:停在 visits 的码值上,做完的 4 项照常写进目录、没做的记 cancelled 且目录不动,后面的表 cancelled,正在等的那条查询留痕写明没等结果;已结束的再停 409 catalog_profile_finished - 关停标中断、启动标中断不续跑(进行中那张表的检查项转成跳过);重试只带没做完的表、设置取当前的,没有可重试的 422,正忙 409 - 任务级止损:连续 3 条失败(error 带驱动报错第一行)、最近 10 条 7 条超时(停下、后面的表带同一句原因);提醒在好转后清掉,成功一条重置连续失败数 - 每张表开始前重读数据源:停用、关掉剖析就停;遮罩按最新的设置 - 范围:全部 / 需要剖析 / 指定(视图、表结构里没有、没有可剖析的项建成 skipped),有待核实关系的在前;profile_due 的 never / unfinished / new_relations;挑不出表 409 profile_nothing;超 500 张 422;空表一条查询记 skipped - 留痕:IN 列表里抽到的键值(文字键)不出现在库里的 query_log;表清单、表详情带上次剖析 - 设置迁移:启动时删掉旧键,老页面带旧键保存只摘掉;保留:超过保留数删旧的,不删某张表最近一次剖析所在的任务,建任务时顺手清 剖析停在一张表中途时,那张表的 error 改说「已做完的 N 项照常写进了数据目录」;engine.py、catalog_transfer.py、catalog_jobs.py 里还写着查询次数上限、只看起草任务的几处说明跟上。
This commit is contained in:
@@ -0,0 +1,671 @@
|
||||
"""数据剖析的后台任务(data/catalog_profile_jobs.py、/api/datasources/{id}/catalog/profile-jobs)。
|
||||
|
||||
剖析的结论怎么进目录、安全约束(只读、守卫、遮罩、大表)在 test_catalog_profile.py;这里钉住任务本身:
|
||||
|
||||
- 同源排队:起草在跑时剖析排队(waiting_for = "draft"),起草结束自动开始;反过来起草排在剖析后面(不 409);停掉排队的;
|
||||
导入目录、同步起草在剖析时 409 profile_busy;同步起草期间建的剖析任务排在它后面
|
||||
- 停止:做完的项写进目录,没做的保留旧结论、记 cancelled,后面的表 cancelled;已结束的再停 409
|
||||
- 服务重启标中断、不续跑;关停时同样;重试只带没做完的表
|
||||
- 任务级止损:连续 3 条失败、最近 10 条 7 条超时(停下、后面的表带同一句原因),最近 6 条 4 条超时提醒、好转后清掉
|
||||
- 每张表开始前重读数据源:停用、关掉剖析就停;遮罩按最新的设置
|
||||
- 范围:需要剖析 / 全部 / 指定;空表一条查询记 skipped;视图跳过;没有可剖析的项;挑不出表 409;profile_due 的三种情形
|
||||
- 留痕:IN 列表里抽到的键值不进 AgentLab 的库
|
||||
- 设置迁移:旧键启动时删掉,老页面带着旧键保存只摘掉
|
||||
- 保留:超过保留数删旧的,但不删某张表最近一次剖析所在的任务
|
||||
|
||||
表名、列名、数据都是虚构的。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
import sqlite3
|
||||
import uuid
|
||||
from datetime import timedelta
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.data import catalog, catalog_jobs, catalog_profile, catalog_profile_jobs, catalog_transfer
|
||||
from app.data import engine as data_engine
|
||||
from app.data.catalog_profile import PROFILE_OPTION, ProfileSettings, SourceRunner, SqlDialect
|
||||
from app.data.guard import SqlRejected
|
||||
from app.db.base import SessionLocal, utcnow
|
||||
from app.db.models import CatalogProfileJob, CatalogProfileTask, DataSource
|
||||
from tests.test_catalog_profile import ( # noqa: F401
|
||||
ACTOR, _detail, _finished, _job, _notes, _on, _profile, _relation, _start, _table, _url, client, make_source,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
async def _stop_jobs():
|
||||
yield
|
||||
await catalog_jobs.manager.shutdown(grace=5)
|
||||
await catalog_profile_jobs.manager.shutdown(grace=5)
|
||||
|
||||
|
||||
async def until(predicate, timeout: float = 10.0) -> None:
|
||||
for _ in range(int(timeout / 0.01)):
|
||||
if await predicate():
|
||||
return
|
||||
await asyncio.sleep(0.01)
|
||||
raise AssertionError("等不到")
|
||||
|
||||
|
||||
def _base(sid: str) -> str:
|
||||
return f"/api/datasources/{sid}/catalog"
|
||||
|
||||
|
||||
async def _rows(client, sid: str) -> dict[str, dict]:
|
||||
r = await client.get(_base(sid))
|
||||
assert r.status_code == 200, r.text
|
||||
return {row["table_name"]: row for row in r.json()["tables"]}
|
||||
|
||||
|
||||
class Gate:
|
||||
"""把一个异步函数换成「等放行才执行」:entered 置位说明有调用在等了。"""
|
||||
|
||||
def __init__(self, real, when=lambda *a, **k: True) -> None:
|
||||
self.real, self.when = real, when
|
||||
self.entered, self.release = asyncio.Event(), asyncio.Event()
|
||||
|
||||
async def __call__(self, *args, **kwargs):
|
||||
if self.when(*args, **kwargs):
|
||||
self.entered.set()
|
||||
await self.release.wait()
|
||||
return await self.real(*args, **kwargs)
|
||||
|
||||
|
||||
def _hold_queries(monkeypatch, when=lambda source, sql, **kw: True) -> Gate:
|
||||
gate = Gate(data_engine.run_query, when)
|
||||
monkeypatch.setattr(data_engine, "run_query", gate)
|
||||
return gate
|
||||
|
||||
|
||||
def _db(path, ddl: str, rows: dict[str, list[tuple]] | None = None) -> str:
|
||||
db = sqlite3.connect(path)
|
||||
try:
|
||||
db.executescript(ddl)
|
||||
for table, values in (rows or {}).items():
|
||||
marks = ", ".join("?" for _ in values[0])
|
||||
db.executemany(f"INSERT INTO {table} VALUES ({marks})", values)
|
||||
db.commit()
|
||||
finally:
|
||||
db.close()
|
||||
return str(path)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 同源排队
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_a_profile_job_waits_for_the_draft_job_and_then_starts(client, make_source, scenic_db, monkeypatch):
|
||||
sid = await make_source(scenic_db, options=_on())
|
||||
gate = Gate(catalog.write_draft)
|
||||
monkeypatch.setattr(catalog, "write_draft", gate)
|
||||
draft = (await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["gates"]})).json()["job"]
|
||||
await asyncio.wait_for(gate.entered.wait(), 10)
|
||||
|
||||
r = await _start(client, sid, ["channels"])
|
||||
assert r.status_code == 202, r.text # 不报错:排在起草后面
|
||||
job = r.json()["job"]
|
||||
assert job["status"] == "queued" and job["waiting_for"] == "draft"
|
||||
listed = (await client.get(_base(sid))).json()
|
||||
assert listed["active_profile_job"]["waiting_for"] == "draft"
|
||||
assert listed["active_job"]["waiting_for"] is None
|
||||
await asyncio.sleep(0.1)
|
||||
assert (await _job(client, sid, job["id"]))["status"] == "queued", "起草没结束,剖析不开始"
|
||||
|
||||
gate.release.set()
|
||||
await catalog_jobs.manager.wait(draft["id"], timeout=10)
|
||||
done = await _finished(client, sid, job["id"])
|
||||
assert done["status"] == "succeeded" and done["waiting_for"] is None
|
||||
draft_done = (await client.get(f"{_base(sid)}/draft-jobs/{draft['id']}")).json()["job"]
|
||||
assert done["started_at"] >= draft_done["finished_at"], "起草结束后才开始"
|
||||
|
||||
|
||||
async def test_a_draft_job_queues_behind_a_running_profile_job(client, make_source, scenic_db, monkeypatch):
|
||||
"""同种任务互斥、不同种排队:剖析在跑时建起草任务不 409,排在后面(waiting_for = "profile")。"""
|
||||
sid = await make_source(scenic_db, options=_on())
|
||||
gate = _hold_queries(monkeypatch)
|
||||
profile = (await _start(client, sid, ["channels"])).json()["job"]
|
||||
await asyncio.wait_for(gate.entered.wait(), 10)
|
||||
|
||||
r = await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["gates"]})
|
||||
assert r.status_code == 202, r.text
|
||||
draft = r.json()["job"]
|
||||
assert draft["status"] == "queued" and draft["waiting_for"] == "profile"
|
||||
got = (await client.get(f"{_base(sid)}/draft-jobs/{draft['id']}")).json()["job"]
|
||||
assert got["waiting_for"] == "profile"
|
||||
assert (await client.get(_base(sid))).json()["active_job"]["waiting_for"] == "profile"
|
||||
# 同种任务照旧互斥
|
||||
again = await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["gates"]})
|
||||
assert again.status_code == 409 and again.json()["code"] == "catalog_draft_busy"
|
||||
|
||||
gate.release.set()
|
||||
await catalog_profile_jobs.manager.wait(profile["id"], timeout=10)
|
||||
await catalog_jobs.manager.wait(draft["id"], timeout=10)
|
||||
got = (await client.get(f"{_base(sid)}/draft-jobs/{draft['id']}")).json()["job"]
|
||||
profile_done = await _job(client, sid, profile["id"])
|
||||
assert got["status"] == "succeeded" and got["started_at"] >= profile_done["finished_at"]
|
||||
|
||||
|
||||
async def test_a_queued_profile_job_can_be_cancelled(client, make_source, scenic_db, monkeypatch):
|
||||
sid = await make_source(scenic_db, options=_on())
|
||||
gate = Gate(catalog.write_draft)
|
||||
monkeypatch.setattr(catalog, "write_draft", gate)
|
||||
draft = (await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["gates"]})).json()["job"]
|
||||
await asyncio.wait_for(gate.entered.wait(), 10)
|
||||
job = (await _start(client, sid, ["channels", "visits"])).json()["job"]
|
||||
assert job["tasks"][0]["checks_total"] > 0, "排队的表也先列好了检查项"
|
||||
|
||||
r = await client.post(f"{_url(sid)}/{job['id']}/cancel")
|
||||
assert r.status_code == 200, r.text
|
||||
stopped = r.json()["job"]
|
||||
assert stopped["status"] == "cancelled" and stopped["finished_at"] and stopped["queries"] == 0
|
||||
assert {t["status"] for t in stopped["tasks"]} == {"cancelled"}
|
||||
detail = await _detail(client, sid, job["id"], 0)
|
||||
assert {c["state"] for c in detail["checks"]} == {"skipped"}, "结束了的表上不留排着的检查项"
|
||||
assert {c["reason"] for c in detail["checks"]} == {"cancelled"} and detail["unfinished"] == len(detail["checks"])
|
||||
assert not catalog_profile_jobs.manager.running(job["id"]), "在队里等着的也停掉了"
|
||||
again = await client.post(f"{_url(sid)}/{job['id']}/cancel")
|
||||
assert again.status_code == 409 and again.json()["code"] == "catalog_profile_finished"
|
||||
|
||||
gate.release.set()
|
||||
await catalog_jobs.manager.wait(draft["id"], timeout=10)
|
||||
assert (await client.get(f"{_base(sid)}/draft-jobs/{draft['id']}")).json()["job"]["status"] == "succeeded"
|
||||
|
||||
|
||||
async def test_import_and_sync_draft_are_409_while_profiling(client, make_source, scenic_db, monkeypatch):
|
||||
sid = await make_source(scenic_db, options=_on(), draft=["gates"])
|
||||
exported = (await client.get(f"{_base(sid)}/export")).json()
|
||||
gate = _hold_queries(monkeypatch)
|
||||
job = (await _start(client, sid, ["channels"])).json()["job"]
|
||||
await asyncio.wait_for(gate.entered.wait(), 10)
|
||||
|
||||
r = await client.post(f"{_base(sid)}/import", json={"content": exported})
|
||||
assert r.status_code == 409 and r.json()["code"] == "profile_busy", r.text
|
||||
assert "正在剖析" in r.json()["detail"]
|
||||
r = await client.post(f"{_base(sid)}/draft", json={"tables": ["gates"]})
|
||||
assert r.status_code == 409 and r.json()["code"] == "profile_busy", r.text
|
||||
|
||||
gate.release.set()
|
||||
await catalog_profile_jobs.manager.wait(job["id"], timeout=10)
|
||||
r = await client.post(f"{_base(sid)}/import", json={"content": exported})
|
||||
assert r.status_code == 200, r.text
|
||||
|
||||
|
||||
async def test_a_profile_job_created_during_a_sync_write_runs_after_it(client, make_source, scenic_db, monkeypatch):
|
||||
"""同步起草、导入目录执行期间占着这个源的队:期间新建的剖析任务排在后面,做完才开始。"""
|
||||
sid = await make_source(scenic_db, options=_on())
|
||||
gate = Gate(catalog.draft_catalog)
|
||||
monkeypatch.setattr(catalog, "draft_catalog", gate)
|
||||
sync = asyncio.create_task(client.post(f"{_base(sid)}/draft", json={"tables": ["gates"]}))
|
||||
await asyncio.wait_for(gate.entered.wait(), 10)
|
||||
job = (await _start(client, sid, ["channels"])).json()["job"]
|
||||
assert job["status"] == "queued" and job["waiting_for"] == "draft"
|
||||
await asyncio.sleep(0.1)
|
||||
assert (await _job(client, sid, job["id"]))["status"] == "queued"
|
||||
gate.release.set()
|
||||
assert (await sync).status_code == 200
|
||||
assert (await _finished(client, sid, job["id"]))["status"] == "succeeded"
|
||||
|
||||
importing = Gate(catalog_transfer.run_import)
|
||||
monkeypatch.setattr(catalog_transfer, "run_import", importing)
|
||||
exported = (await client.get(f"{_base(sid)}/export")).json()
|
||||
imported = asyncio.create_task(client.post(f"{_base(sid)}/import", json={"content": exported}))
|
||||
await asyncio.wait_for(importing.entered.wait(), 10)
|
||||
job = (await _start(client, sid, ["channels"])).json()["job"]
|
||||
assert job["status"] == "queued" and job["waiting_for"] is None
|
||||
# 导入期间再导入一次:同步写操作一次一个
|
||||
busy = await client.post(f"{_base(sid)}/import", json={"content": exported})
|
||||
assert busy.status_code == 409, busy.text
|
||||
importing.release.set()
|
||||
assert (await imported).status_code == 200
|
||||
assert (await _finished(client, sid, job["id"]))["status"] == "succeeded"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 停止、服务重启、重试
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _at_visits_codes(source, sql, **kw) -> bool:
|
||||
return "GROUP BY status" in sql and "FROM visits" in sql
|
||||
|
||||
|
||||
async def test_stopping_writes_what_is_done_and_keeps_the_rest(client, make_source, scenic_db, monkeypatch):
|
||||
"""停在 visits 的码值上:关系都做完了,照常写进目录;码值、日期没做,记停止、目录不动;后面的 orders 记停止。"""
|
||||
sid = await make_source(scenic_db, options=_on(), draft=["visits", "channels"])
|
||||
gate = _hold_queries(monkeypatch, _at_visits_codes)
|
||||
job = (await _start(client, sid, ["channels", "visits", "orders"])).json()["job"]
|
||||
await asyncio.wait_for(gate.entered.wait(), 10)
|
||||
running = await _job(client, sid, job["id"])
|
||||
row = next(t for t in running["tasks"] if t["table_name"] == "visits")
|
||||
assert row["status"] == "running" and row["current"] == "正在取 visits.status 的取值分布"
|
||||
assert row["checks_by_kind"]["relation"] == [4, 4] and row["checks_by_kind"]["codes"] == [0, 1]
|
||||
|
||||
r = await client.post(f"{_url(sid)}/{job['id']}/cancel")
|
||||
assert r.status_code == 200, r.text
|
||||
stopped = r.json()["job"]
|
||||
by = {t["table_name"]: t for t in stopped["tasks"]}
|
||||
assert stopped["status"] == "cancelled"
|
||||
assert by["channels"]["status"] == "ok"
|
||||
assert by["visits"]["status"] == "partial" and by["visits"]["current"] is None
|
||||
assert by["visits"]["error"] == "剖析已停止:已做完的 4 项照常写进了数据目录,剩下 2 项没查"
|
||||
assert by["orders"]["status"] == "cancelled" and by["orders"]["error"] is None
|
||||
|
||||
detail = await _detail(client, sid, job["id"], 1)
|
||||
states = {c["kind"]: (c["state"], c["reason"]) for c in detail["checks"]}
|
||||
assert states["codes"] == ("skipped", "cancelled") and states["date"] == ("skipped", "cancelled")
|
||||
assert detail["query_log"][-1]["outcome"] == "error" and "没有等这条查询的结果" in detail["query_log"][-1]["detail"]
|
||||
notes = (await _notes(client, sid, "visits"))["notes"]
|
||||
assert _relation(notes, "gate_id")["status"] == "verified", "做完的项写进了目录"
|
||||
assert "status" not in (notes.get("columns") or {}) and "business_date" not in notes, "没做的不动"
|
||||
again = await client.post(f"{_url(sid)}/{job['id']}/cancel")
|
||||
assert again.status_code == 409 and again.json()["code"] == "catalog_profile_finished"
|
||||
|
||||
|
||||
async def test_shutdown_marks_the_job_interrupted(client, make_source, scenic_db, monkeypatch):
|
||||
sid = await make_source(scenic_db, options=_on(), draft=["visits"])
|
||||
gate = _hold_queries(monkeypatch, _at_visits_codes)
|
||||
job = (await _start(client, sid, ["visits", "orders"])).json()["job"]
|
||||
await asyncio.wait_for(gate.entered.wait(), 10)
|
||||
await catalog_profile_jobs.manager.shutdown(grace=5)
|
||||
got = await _job(client, sid, job["id"])
|
||||
by = {t["table_name"]: t for t in got["tasks"]}
|
||||
assert got["status"] == "interrupted" and got["error"] == "服务重启,剖析中断"
|
||||
assert by["visits"]["status"] == "partial" and by["visits"]["error"].startswith("服务重启,剖析中断:已做完的 4 项")
|
||||
assert by["orders"]["status"] == "cancelled" and by["orders"]["error"] == "服务重启,剖析中断"
|
||||
assert _relation((await _notes(client, sid, "visits"))["notes"], "gate_id")["status"] == "verified"
|
||||
|
||||
|
||||
async def test_startup_marks_unfinished_jobs_interrupted_and_does_not_resume(client, make_source, scenic_db):
|
||||
sid = await make_source(scenic_db, options=_on())
|
||||
checks = [{"kind": "codes", "target": "status", "path": "columns.status.codes", "state": "done", "reason": None,
|
||||
"detail": None, "queries": 1},
|
||||
{"kind": "date", "target": "visit_time", "path": "business_date", "state": "running", "reason": None,
|
||||
"detail": None, "queries": 0}]
|
||||
async with SessionLocal() as session:
|
||||
job = CatalogProfileJob(source_id=sid, status="running", scope="selected", settings={"enabled": True})
|
||||
session.add(job)
|
||||
await session.flush()
|
||||
session.add(CatalogProfileTask(job_id=job.id, seq=0, table_name="channels", status="ok"))
|
||||
session.add(CatalogProfileTask(job_id=job.id, seq=1, table_name="visits", status="running", checks=checks))
|
||||
session.add(CatalogProfileTask(job_id=job.id, seq=2, table_name="gates", status="pending"))
|
||||
await session.commit()
|
||||
job_id = job.id
|
||||
assert await catalog_profile_jobs.startup() >= 1
|
||||
got = await _job(client, sid, job_id)
|
||||
by = {t["table_name"]: t for t in got["tasks"]}
|
||||
assert got["status"] == "interrupted" and got["error"] == "服务重启,剖析中断" and got["finished_at"]
|
||||
assert by["channels"]["status"] == "ok" and by["channels"]["error"] is None
|
||||
assert by["visits"]["status"] == by["gates"]["status"] == "cancelled"
|
||||
assert by["visits"]["error"] == by["gates"]["error"] == "服务重启,剖析中断"
|
||||
visits = await _detail(client, sid, job_id, 1)
|
||||
assert [c["state"] for c in visits["checks"]] == ["done", "skipped"]
|
||||
assert visits["checks"][1]["reason"] == "cancelled" and visits["unfinished"] == 1
|
||||
assert not catalog_profile_jobs.manager.running(job_id), "不自动续跑"
|
||||
|
||||
|
||||
async def test_retry_takes_only_unfinished_tables(client, make_source, scenic_db, monkeypatch):
|
||||
sid = await make_source(scenic_db, options=_on(table_max_queries=2), draft=["visits"])
|
||||
first = await _profile(client, sid, ["visits", "channels", "nope"])
|
||||
by = {t["table_name"]: t["status"] for t in first["tasks"]}
|
||||
assert by == {"visits": "partial", "channels": "ok", "nope": "skipped"} and first["status"] == "partial"
|
||||
|
||||
r = await client.patch(f"/api/datasources/{sid}", json={"options": _on()})
|
||||
assert r.status_code == 200, r.text
|
||||
r = await client.post(f"{_url(sid)}/{first['id']}/retry", headers=ACTOR)
|
||||
assert r.status_code == 202, r.text
|
||||
retry = r.json()["job"]
|
||||
assert retry["retry_of"] == first["id"] and retry["scope"] == "selected"
|
||||
assert [t["table_name"] for t in retry["tasks"]] == ["visits"]
|
||||
assert retry["settings"]["table_max_queries"] == 40, "设置取数据源当前的"
|
||||
done = await _finished(client, sid, retry["id"])
|
||||
assert done["status"] == "succeeded"
|
||||
r = await client.post(f"{_url(sid)}/{retry['id']}/retry")
|
||||
assert r.status_code == 422 and "不需要重试" in r.json()["detail"]
|
||||
|
||||
monkeypatch.setattr(catalog_profile_jobs.manager, "start", lambda job_id: None) # 排着、不开始
|
||||
queued = (await _start(client, sid, ["gates"])).json()["job"]
|
||||
r = await client.post(f"{_url(sid)}/{first['id']}/retry")
|
||||
assert r.status_code == 409 and r.json()["code"] == "profile_busy"
|
||||
await client.post(f"{_url(sid)}/{queued['id']}/cancel")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 任务级止损、每张表开始前重读数据源
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
_MANY = ["visits", "orders", "members", "channels", "ticket_types", "gates", "parks", "stores", "products",
|
||||
"payments", "refunds", "reviews"]
|
||||
|
||||
|
||||
async def test_three_failures_in_a_row_stop_the_job(client, make_source, scenic_db, monkeypatch):
|
||||
sid = await make_source(scenic_db, options=_on(), draft=["visits"])
|
||||
|
||||
async def broken(source, sql, *, limits=None):
|
||||
raise RuntimeError("connection refused by the server\ndetails follow")
|
||||
|
||||
monkeypatch.setattr(data_engine, "run_query", broken)
|
||||
job = await _profile(client, sid, ["visits", "channels", "orders"])
|
||||
assert job["status"] == "partial" and job["stop_reason"] == "failed"
|
||||
assert job["error"] == "连续 3 条查询失败,已停止剖析:connection refused by the server"
|
||||
assert (job["queries"], job["errors"], job["timeouts"]) == (3, 3, 0)
|
||||
visits, channels, orders = (_table(job, n) for n in ("visits", "channels", "orders"))
|
||||
assert visits["status"] == "partial" and visits["error"] == job["error"]
|
||||
assert "failed" in {s["reason"] for s in visits["skipped"]}
|
||||
assert channels["status"] == orders["status"] == "cancelled"
|
||||
assert channels["error"] == orders["error"] == job["error"] and channels["queries"] == 0
|
||||
|
||||
|
||||
async def test_mostly_timeouts_stop_the_job_as_slow(client, make_source, scenic_db, monkeypatch):
|
||||
sid = await make_source(scenic_db, options=_on())
|
||||
|
||||
async def slow(source, sql, *, limits=None):
|
||||
raise SqlRejected("查询超过 1 秒被中断。请添加 WHERE 条件或 LIMIT 缩小范围。")
|
||||
|
||||
monkeypatch.setattr(data_engine, "run_query", slow)
|
||||
job = await _profile(client, sid, _MANY)
|
||||
assert job["stop_reason"] == "slow" and job["status"] == "partial"
|
||||
assert job["error"] == "数据库响应慢:最近 10 条查询 7 条超时,已停止,避免给业务库加压"
|
||||
assert job["timeouts"] == job["queries"] == 7
|
||||
assert job["notice"] and job["notice"].startswith("数据库响应慢")
|
||||
cancelled = [t for t in job["tasks"] if t["status"] == "cancelled"]
|
||||
assert cancelled and all(t["error"] == job["error"] for t in cancelled)
|
||||
|
||||
|
||||
def _runner(monkeypatch, outcomes: list[str]) -> SourceRunner:
|
||||
"""SourceRunner 接一个假的查询层:按 outcomes 依次超时(t)、失败(e)或成功(o)。"""
|
||||
seq = iter(outcomes)
|
||||
|
||||
async def fake(source, sql, *, limits=None):
|
||||
step = next(seq)
|
||||
if step == "t":
|
||||
raise SqlRejected("查询超过 1 秒被中断")
|
||||
if step == "e":
|
||||
raise RuntimeError("no such table: orders")
|
||||
return data_engine.QueryResult(columns=["n"], rows=[[1]], row_count=1, truncated=False, elapsed_ms=1,
|
||||
sql=sql)
|
||||
|
||||
monkeypatch.setattr(data_engine, "run_query", fake)
|
||||
source = SimpleNamespace(id="s", name="shop", kind="sqlite", options={})
|
||||
return SourceRunner(source, ProfileSettings(enabled=True), SqlDialect("sqlite"))
|
||||
|
||||
|
||||
async def _feed(runner: SourceRunner, n: int) -> None:
|
||||
for _ in range(n):
|
||||
try:
|
||||
await runner.run("SELECT 1")
|
||||
except catalog_profile._Skip:
|
||||
pass
|
||||
|
||||
|
||||
async def test_the_slow_notice_clears_when_queries_recover(monkeypatch):
|
||||
runner = _runner(monkeypatch, ["t"] * 4 + ["o"] * 3 + ["t"] * 3)
|
||||
await _feed(runner, 4)
|
||||
assert runner.notice == "数据库响应慢:最近 4 条查询有 4 条超时。超时再多会自动停止,也可以先停止、换个时间再剖析"
|
||||
assert runner.stopped is None
|
||||
await _feed(runner, 3)
|
||||
assert runner.notice is None, "好转后清掉"
|
||||
await _feed(runner, 3)
|
||||
# 最近 10 条里 7 条超时:停下,之后再发直接停
|
||||
assert runner.stopped == "slow" and runner.timeouts == 7 and runner.queries == 10
|
||||
assert runner.error == "数据库响应慢:最近 10 条查询 7 条超时,已停止,避免给业务库加压"
|
||||
with pytest.raises(catalog_profile._Stop):
|
||||
await runner.run("SELECT 1")
|
||||
|
||||
|
||||
async def test_a_success_resets_the_failure_count(monkeypatch):
|
||||
runner = _runner(monkeypatch, ["e", "e", "o", "e", "e", "e"])
|
||||
await _feed(runner, 5)
|
||||
assert runner.stopped is None and runner.errors == 4
|
||||
await _feed(runner, 1)
|
||||
assert runner.stopped == "failed" and runner.error == "连续 3 条查询失败,已停止剖析:no such table: orders"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("change, said", [({"enabled": False}, "已停用"),
|
||||
({"options": _on(enabled=False)}, "已关闭数据剖析")])
|
||||
async def test_disabling_the_source_stops_before_the_next_table(client, make_source, scenic_db, monkeypatch,
|
||||
change, said):
|
||||
sid = await make_source(scenic_db, options=_on())
|
||||
gate = _hold_queries(monkeypatch, lambda source, sql, **kw: "FROM channels" in sql)
|
||||
job = (await _start(client, sid, ["channels", "visits", "orders"])).json()["job"]
|
||||
await asyncio.wait_for(gate.entered.wait(), 10)
|
||||
async with SessionLocal() as session:
|
||||
row = await session.get(DataSource, sid)
|
||||
for key, value in change.items():
|
||||
setattr(row, key, value)
|
||||
await session.commit()
|
||||
gate.release.set()
|
||||
done = await _finished(client, sid, job["id"])
|
||||
by = {t["table_name"]: t for t in done["tasks"]}
|
||||
assert by["channels"]["status"] == "ok", "正在做的那张做完"
|
||||
assert by["visits"]["status"] == by["orders"]["status"] == "cancelled"
|
||||
assert said in by["visits"]["error"] and by["visits"]["error"] == done["error"]
|
||||
assert done["status"] == "partial" and done["stop_reason"] is None
|
||||
|
||||
|
||||
async def test_masks_follow_the_latest_settings_table_by_table(client, make_source, scenic_db, monkeypatch):
|
||||
sid = await make_source(scenic_db, options=_on())
|
||||
gate = _hold_queries(monkeypatch, lambda source, sql, **kw: "FROM orders" in sql)
|
||||
job = (await _start(client, sid, ["orders", "channels"])).json()["job"]
|
||||
await asyncio.wait_for(gate.entered.wait(), 10)
|
||||
async with SessionLocal() as session:
|
||||
row = await session.get(DataSource, sid)
|
||||
row.options = {**_on(), "mask_columns": "channel_type"}
|
||||
await session.commit()
|
||||
gate.release.set()
|
||||
done = await _finished(client, sid, job["id"])
|
||||
channels = _table(done, "channels")
|
||||
assert {(s["target"], s["reason"]) for s in channels["skipped"]} >= {("channel_type", "masked")}
|
||||
assert not any("channel_type" in q["sql"].lower() for q in channels["query_log"])
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 范围
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_scopes_needed_all_and_selected(client, make_source, scenic_db, monkeypatch):
|
||||
sid = await make_source(scenic_db, options=_on(), draft=["visits", "channels", "gates", "suppliers"])
|
||||
rows = await _rows(client, sid)
|
||||
assert rows["v_daily_visits"]["is_view"] and rows["v_daily_visits"]["profile_checks"] == 0
|
||||
assert not rows["v_daily_visits"]["profile_due"]
|
||||
assert rows["suppliers"]["profile_checks"] == 0 and not rows["suppliers"]["profile_due"]
|
||||
assert (rows["visits"]["profile_due"], rows["visits"]["profile_due_reason"]) == (True, "never")
|
||||
assert rows["visits"]["last_profile"] is None and rows["visits"]["profile_checks"] == 6
|
||||
expected = [name for name, row in rows.items() if row["profile_checks"] > 0 and not row["is_view"]]
|
||||
|
||||
monkeypatch.setattr(catalog_profile_jobs.manager, "start", lambda job_id: None) # 只看挑了哪些表
|
||||
everything = (await _start(client, sid, scope="all")).json()["job"]
|
||||
names = [t["table_name"] for t in everything["tasks"]]
|
||||
assert everything["scope"] == "all" and sorted(names) == sorted(expected)
|
||||
# 有待核实关系的表在前(剖析首先是来核实它们的),其余按表结构顺序
|
||||
async with SessionLocal() as session:
|
||||
entries = await catalog.read_catalog(session, sid)
|
||||
pending = [catalog_profile.has_pending_relations(entries[n].notes if n in entries else {}) for n in names]
|
||||
assert pending == sorted(pending, reverse=True) and pending[0] is True
|
||||
await client.post(f"{_url(sid)}/{everything['id']}/cancel")
|
||||
needed = (await _start(client, sid)).json()["job"]
|
||||
assert needed["scope"] == "needed" and [t["table_name"] for t in needed["tasks"]] == names, "都没剖析过"
|
||||
await client.post(f"{_url(sid)}/{needed['id']}/cancel")
|
||||
|
||||
picked = (await _start(client, sid, ["v_daily_visits", "nope", "visits", "VISITS", "suppliers"])).json()["job"]
|
||||
tasks = {t["table_name"]: t for t in picked["tasks"]}
|
||||
assert picked["scope"] == "selected" and list(tasks) == ["v_daily_visits", "nope", "visits", "suppliers"]
|
||||
assert tasks["v_daily_visits"]["status"] == "skipped" and "视图" in tasks["v_daily_visits"]["error"]
|
||||
assert tasks["nope"]["status"] == "skipped" and "表结构里没有 nope" in tasks["nope"]["error"]
|
||||
assert tasks["suppliers"]["status"] == "skipped" and tasks["suppliers"]["error"].startswith("没有可剖析的项")
|
||||
assert tasks["visits"]["status"] == "pending" and tasks["visits"]["checks_total"] == 6
|
||||
await client.post(f"{_url(sid)}/{picked['id']}/cancel")
|
||||
|
||||
|
||||
async def test_profile_due_never_unfinished_and_new_relations(client, make_source, scenic_db):
|
||||
sid = await make_source(scenic_db, options=_on(table_max_queries=2), draft=["visits", "gates"])
|
||||
await _profile(client, sid, ["visits"])
|
||||
rows = await _rows(client, sid)
|
||||
assert (rows["visits"]["profile_due"], rows["visits"]["profile_due_reason"]) == (True, "unfinished")
|
||||
assert rows["visits"]["last_profile"]["status"] == "partial" and rows["visits"]["last_profile"]["unfinished"] >= 1
|
||||
assert "paths" not in rows["visits"]["last_profile"]
|
||||
|
||||
r = await client.patch(f"/api/datasources/{sid}", json={"options": _on()})
|
||||
assert r.status_code == 200, r.text
|
||||
await _profile(client, sid, ["visits", "gates"])
|
||||
rows = await _rows(client, sid)
|
||||
assert (rows["visits"]["profile_due"], rows["visits"]["profile_due_reason"]) == (False, None)
|
||||
assert rows["gates"]["last_profile"]["status"] == "ok" and not rows["gates"]["profile_due"]
|
||||
needed = await _start(client, sid, ["visits"])
|
||||
assert needed.status_code == 202, "指定了表照样能剖析"
|
||||
await catalog_profile_jobs.manager.wait(needed.json()["job"]["id"], timeout=30)
|
||||
|
||||
# 剖析之后目录里冒出了新的待核实关系:这张表又要剖析了
|
||||
rel = {"id": catalog.relation_id("gates", ["gate_code"], "channels", ["channel_code"]), "columns": ["gate_code"],
|
||||
"to_table": "channels", "to_columns": ["channel_code"], "cardinality": "many_to_one", "coverage": None,
|
||||
"source": "name", "status": "proposed"}
|
||||
async with SessionLocal() as session:
|
||||
entry = await catalog.read_entry(session, sid, "gates")
|
||||
notes = {**entry.notes, "relations": [*entry.notes.get("relations", []), rel]}
|
||||
await catalog.write_entry(session, sid, "gates", notes, if_version=entry.version, actor=None)
|
||||
rows = await _rows(client, sid)
|
||||
assert (rows["gates"]["profile_due"], rows["gates"]["profile_due_reason"]) == (True, "new_relations")
|
||||
job = (await _start(client, sid)).json()["job"]
|
||||
assert "gates" in [t["table_name"] for t in job["tasks"]] and "visits" not in [t["table_name"] for t in job["tasks"]]
|
||||
await catalog_profile_jobs.manager.wait(job["id"], timeout=30)
|
||||
|
||||
|
||||
async def test_nothing_to_profile_is_409(client, make_source, tmp_path):
|
||||
path = _db(tmp_path / "plain.db", "CREATE TABLE suppliers (id INTEGER PRIMARY KEY, name TEXT);",
|
||||
{"suppliers": [(1, "甲供应商")]})
|
||||
sid = await make_source(path, options=_on())
|
||||
r = await _start(client, sid)
|
||||
assert r.status_code == 409 and r.json()["code"] == "profile_nothing"
|
||||
assert "全部表" in r.json()["detail"] and "指定" in r.json()["detail"]
|
||||
r = await _start(client, sid, scope="all")
|
||||
assert r.status_code == 409 and r.json()["code"] == "profile_nothing" and "指定" in r.json()["detail"]
|
||||
r = await _start(client, sid, [f"t{i}" for i in range(501)])
|
||||
assert r.status_code == 422 and "500" in r.json()["detail"]
|
||||
assert (await client.get(f"{_url(sid)}/latest")).json()["job"] is None
|
||||
|
||||
|
||||
async def test_an_empty_table_costs_one_query(client, make_source, tmp_path):
|
||||
path = _db(tmp_path / "empty.db", "CREATE TABLE orders (id INTEGER PRIMARY KEY, status INTEGER, created_at TEXT);")
|
||||
sid = await make_source(path, options=_on())
|
||||
job = await _profile(client, sid, ["orders"])
|
||||
orders = _table(job, "orders")
|
||||
assert orders["status"] == "skipped" and orders["error"] == "空表,没有可剖析的数据"
|
||||
assert orders["queries"] == 1 == job["queries"] and orders["query_log"][0]["kind"] == "row_estimate"
|
||||
assert {c["reason"] for c in orders["checks"]} == {"no_data"} and orders["unfinished"] == 0
|
||||
assert job["status"] == "succeeded", "空表不算没做完"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 留痕、单表接口
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_sampled_keys_never_reach_the_query_log(client, make_source, tmp_path):
|
||||
"""抽样到的键值拼进了 IN 列表,留痕里换成「?」:AgentLab 自己的库里不留这些值。"""
|
||||
customers = [(f"CUST-{i:04d}", "active" if i % 3 else "closed") for i in range(1, 31)]
|
||||
orders = [(i, f"CUST-{1 + i % 30:04d}", "paid") for i in range(1, 61)]
|
||||
path = _db(tmp_path / "shop.db",
|
||||
"CREATE TABLE customers (code TEXT PRIMARY KEY, status TEXT);"
|
||||
"CREATE TABLE orders (id INTEGER PRIMARY KEY, customer_code TEXT, status TEXT);",
|
||||
{"customers": customers, "orders": orders})
|
||||
sid = await make_source(path, options=_on())
|
||||
detail = await _notes(client, sid, "orders")
|
||||
rel = {"columns": ["customer_code"], "to_table": "customers", "to_columns": ["code"]}
|
||||
r = await client.post(f"{_base(sid)}/orders/patch",
|
||||
json={"changes": [{"path": "relations.new", "value": rel, "reason": "订单的客户编码"}],
|
||||
"if_version": detail["version"]})
|
||||
assert r.status_code == 200, r.text
|
||||
|
||||
job = await _profile(client, sid, ["orders"])
|
||||
orders_task = _table(job, "orders")
|
||||
matched = [q for q in orders_task["query_log"] if " IN (" in q["sql"]]
|
||||
assert matched and all(re.search(r"IN \((\?(, )?)+\)", q["sql"]) for q in matched), matched
|
||||
finding = next(f for f in orders_task["findings"] if f["kind"] == "relation")
|
||||
assert finding["sample"] == 30 and finding["coverage"] == 1.0
|
||||
async with SessionLocal() as session:
|
||||
stored = (await session.execute(select(CatalogProfileTask.query_log, CatalogProfileTask.findings)
|
||||
.where(CatalogProfileTask.job_id == job["id"]))).all()
|
||||
assert stored and "CUST-" not in json.dumps([row[0] for row in stored], ensure_ascii=False)
|
||||
|
||||
|
||||
async def test_the_table_detail_and_list_carry_the_last_profile(client, make_source, scenic_db, monkeypatch):
|
||||
sid = await make_source(scenic_db, options=_on(), draft=["visits"])
|
||||
gate = _hold_queries(monkeypatch, _at_visits_codes)
|
||||
job = (await _start(client, sid, ["visits"])).json()["job"]
|
||||
await asyncio.wait_for(gate.entered.wait(), 10)
|
||||
detail = await _notes(client, sid, "visits")
|
||||
assert detail["last_profile"]["status"] == "running" and detail["last_profile"]["job_id"] == job["id"]
|
||||
assert (await _rows(client, sid))["visits"]["last_profile"]["status"] == "running"
|
||||
assert not (await _rows(client, sid))["visits"]["profile_due"], "正在剖析的不算要剖析"
|
||||
gate.release.set()
|
||||
await catalog_profile_jobs.manager.wait(job["id"], timeout=30)
|
||||
|
||||
detail = await _notes(client, sid, "visits")
|
||||
last = detail["last_profile"]
|
||||
assert last["status"] == "ok" and last["created_by"] == "周宁" and last["job_status"] == "succeeded"
|
||||
assert last["query_log"] and last["findings"] and last["seq"] == 0
|
||||
row = (await _rows(client, sid))["visits"]["last_profile"]
|
||||
assert set(row) == {"job_id", "seq", "status", "started_at", "finished_at", "summary", "skipped_count",
|
||||
"unfinished", "error", "row_estimate"}
|
||||
assert row["summary"] == last["summary"] and row["row_estimate"] == {"rows": 1500, "method": "count",
|
||||
"at_least": None}
|
||||
assert (await _notes(client, sid, "gates"))["last_profile"] is None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 设置迁移、保留
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_old_whole_run_budgets_are_removed_on_startup_and_on_save(client, make_source, scenic_db):
|
||||
sid = await make_source(scenic_db, options={PROFILE_OPTION: {"enabled": True, "max_queries": 60,
|
||||
"max_total_s": 120, "sample_size": 500},
|
||||
"query_timeout_s": "20"})
|
||||
assert await catalog_profile_jobs.startup() >= 0
|
||||
async with SessionLocal() as session:
|
||||
row = await session.get(DataSource, sid)
|
||||
assert row.options == {PROFILE_OPTION: {"enabled": True, "sample_size": 500}, "query_timeout_s": "20"}
|
||||
# 老页面还带着旧键:只摘掉,不报错
|
||||
r = await client.patch(f"/api/datasources/{sid}",
|
||||
json={"options": _on(max_queries=30, max_total_s=90, table_max_queries=25)})
|
||||
assert r.status_code == 200, r.text
|
||||
assert r.json()["options"][PROFILE_OPTION] == {"enabled": True, "table_max_queries": 25}
|
||||
|
||||
|
||||
async def test_retention_keeps_recent_jobs_and_each_tables_last_profile(client, make_source, scenic_db, monkeypatch):
|
||||
sid = await make_source(scenic_db, options=_on())
|
||||
start = utcnow() - timedelta(hours=1)
|
||||
ids: list[str] = []
|
||||
async with SessionLocal() as session:
|
||||
for i in range(5):
|
||||
job = CatalogProfileJob(id=uuid.uuid4().hex, source_id=sid, status="succeeded", scope="selected",
|
||||
settings={"enabled": True}, created_at=start + timedelta(minutes=i))
|
||||
session.add(job)
|
||||
await session.flush()
|
||||
session.add(CatalogProfileTask(job_id=job.id, seq=0, table_name="visits", status="ok"))
|
||||
if i == 0:
|
||||
session.add(CatalogProfileTask(job_id=job.id, seq=1, table_name="gates", status="ok"))
|
||||
ids.append(job.id)
|
||||
await session.commit()
|
||||
async with SessionLocal() as session:
|
||||
assert await catalog_profile_jobs.prune(session, sid, keep=2) == 2
|
||||
left = set((await session.execute(select(CatalogProfileJob.id)
|
||||
.where(CatalogProfileJob.source_id == sid))).scalars())
|
||||
assert left == {ids[4], ids[3], ids[0]}, "最早那个是 gates 最近一次剖析所在的任务,不删"
|
||||
assert (await _notes(client, sid, "gates"))["last_profile"]["job_id"] == ids[0]
|
||||
|
||||
# 建任务时顺手清
|
||||
monkeypatch.setattr(catalog_profile_jobs, "KEEP_JOBS", 1)
|
||||
job = await _profile(client, sid, ["visits"])
|
||||
async with SessionLocal() as session:
|
||||
left = set((await session.execute(select(CatalogProfileJob.id)
|
||||
.where(CatalogProfileJob.source_id == sid))).scalars())
|
||||
assert ids[0] in left and job["id"] in left and ids[3] not in left
|
||||
Reference in New Issue
Block a user