Files
AgentLab/backend/tests/test_source_maintenance.py
T
ThearchyHelios f5c94cf12d W3-a 变化记录的接口与处理、定时维护(next-round 03 B3、B4)
变化没有统一的记录:进不了审阅,发不了通知,口径账本的召回也没有可以订阅的接口;目录只能手动探查、手动剖析。现在:

- api/source_changes.py(骨架已在 datasources 通配之前登记,搭建者可用,访问密钥不放):GET /api/datasources/{id}/changes
  (缺省开着的和知道了的,按「使用次数权重 × 严重程度」排,带各级别计数)、GET /changes/{change_id}、POST
  /changes/{change_id}/resolve。处理动作落到目录已有的写入接口:清理目录里说到被删的列的项(整份写、乐观锁)、加入码值
  (apply_patch 配 _merge_codes,含义待填写,依据写变化编号)、重新起草这张表(模型用不了时只按表结构起草)、重新剖析、
  知道了、误报(必须写理由);每个动作写 resolution,动了目录的记目录版本,建了任务的记任务编号。
- data/source_changes.py 给召回:list_changes(按 seq 游标拉)、affecting(按源和表反查)、resolve、triggers_recall(只认
  结构类、处理时改了目录的、触及有效记录条件的新码值)。
- 通知 source.changed 的发送函数 notify/service.notify_source_changed:只推还要人看的「要处理」级变化,按源汇总,消息里
  只有源名、各级别条数和链接(D9);探查、剖析任务结束时有新的要处理变化就调。这一行的订阅范围第 4 波有了界面才打开,
  现在推不出去(照实)。
- 定时维护(data/source_maintenance.py,X4:就是一条 target_kind=source_maintenance 的定时,不另建表):fire 按顺序建
  探查任务 → 剖析任务(范围 needed / observe / maintenance 按选的步骤),任务上记 trigger=schedule、fire_at、schedule_id;
  剖析等探查进了同源队才建,靠 SourceLanes 串行;剖析没开、数据源停用 / 不存在 / 是上传表格时照实记没执行,同源还有
  探查或剖析在做时记跳过;触发记录的 inputs 写明建了哪几个任务、哪几步没执行。check_target 核对目标。维护步骤存在
  options.catalog_maintenance,经 GET / PUT /api/datasources/{id}/maintenance 改,数据源表单不显示它(api/datasources.py
  的 _MANAGED_OPTIONS 加了这一个键)。

测试:test_source_changes(列表与计数、误报要理由、处理过的 409、清理目录与加入码值记目录版本、剖析没开照实 422、
去重、游标拉取与反查、召回触发、通知只有源名条数链接)、test_source_maintenance(按顺序串行、trigger 与 fire_at、
剖析没开记未执行、停用 / 不存在 / 忙、调度循环只补最近一次且积压只记一条、步骤接口与核对目标)。
2026-10-16 16:48:40 +08:00

211 lines
11 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""数据源的定时维护(app/data/source_maintenance.py,next-round 03 B4、总方案 X4)。
定时只有一套:维护就是一条 target_kind=source_maintenance 的 Schedule,认领、补跑、积压只记一条都是 scheduler 那一套。
这里守:
- 到点按顺序建探查任务 → 剖析任务(范围按选的步骤),任务上记 trigger=schedule 和 fire_at;剖析排在同一次触发的探查后面
(同源队串行:探查做完剖析才开始);触发记录写明建了哪几个任务;
- 照实记没执行的:剖析没开时剖析、观察两步不执行(探查照做);数据源停用、不存在、是上传表格时整次不执行;上一次的探查
还没做完时这次跳过;
- 调度循环的认领:错过的只补最近一次、积压只记一条;
- 维护步骤的接口:不合规 422,合规存进 options、数据源表单不显示它;核对目标(check_target)。
"""
from __future__ import annotations
import sqlite3
import uuid
from datetime import timedelta
import pytest
from httpx import ASGITransport, AsyncClient
from sqlalchemy import delete, select
from app.data import catalog, catalog_profile_jobs, introspect_jobs, source_maintenance as sm
from app.db.base import SessionLocal, utcnow
from app.db.models import CatalogProfileJob, DataSource, Schedule, ScheduleFire
from app.db.models_sources import IntrospectJob
from app.main import app
from app.scheduler import loop, service
from tests.fixtures.sources import drop_source, introspect_now
@pytest.fixture
async def client():
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
yield c
async def _source(client, path, *, profile: bool = True) -> str:
db = sqlite3.connect(path)
db.executescript("""
CREATE TABLE orders (id INTEGER PRIMARY KEY, status VARCHAR(10), paid_on DATE);
INSERT INTO orders VALUES (1, 'paid', '2026-10-01'), (2, 'void', '2026-10-02');
""")
db.commit()
db.close()
r = await client.post("/api/datasources", json={"name": f"mnt_{uuid.uuid4().hex[:8]}", "kind": "sqlite",
"database": str(path), "readonly": True})
sid = r.json()["id"]
await introspect_now(client, sid)
async with SessionLocal() as session:
row = await session.get(DataSource, sid)
if profile:
row.options = {**(row.options or {}), "catalog_profile": {"enabled": True}}
await session.commit()
await catalog.write_entry(session, sid, "orders", {
"columns": {"status": {"codes": catalog.make_item({"paid": "已支付", "void": "作废"}, "human")}},
}, if_version=0, actor="测试员")
return sid
async def _schedule(sid: str, **over) -> str:
async with SessionLocal() as session:
fields = {"name": f"维护-{uuid.uuid4().hex[:6]}", "cron": "0 3 * * *", "timezone": "Asia/Shanghai",
**service.target_columns("source_maintenance", sid), **over}
row = Schedule(**fields)
session.add(row)
await session.commit()
return row.id
@pytest.fixture
async def made():
ids: list[str] = []
yield ids
async with SessionLocal() as session:
await session.execute(delete(Schedule).where(Schedule.id.in_(ids)))
await session.commit()
async def _wait_jobs(fire: ScheduleFire) -> None:
for job in (fire.inputs or {}).get("jobs") or []:
manager = introspect_jobs.manager if job["kind"] == "introspect" else catalog_profile_jobs.manager
await manager.wait(job["id"], timeout=60)
async def test_fire_builds_introspect_then_profile_in_order(client, tmp_path, made):
sid = await _source(client, tmp_path / "a.db")
try:
schedule_id = await _schedule(sid)
made.append(schedule_id)
at = utcnow().replace(microsecond=0)
fire = await service.fire_once(schedule_id, at, "manual", actor="测试员")
assert fire.status == "started" and fire.reason is None, fire.reason
jobs = fire.inputs["jobs"]
assert [j["kind"] for j in jobs] == ["introspect", "profile"] and jobs[1]["scope"] == "maintenance"
assert fire.inputs["steps"] == list(sm.STEPS)
await _wait_jobs(fire)
async with SessionLocal() as session:
intro = await session.get(IntrospectJob, jobs[0]["id"])
prof = await session.get(CatalogProfileJob, jobs[1]["id"])
assert intro.trigger == "schedule" and intro.scope["fire_at"] == at.isoformat()
assert intro.scope["schedule_id"] == schedule_id and intro.created_by == "测试员"
assert catalog_profile_jobs.trigger_of(prof) == {"trigger": "schedule", "fire_at": at.isoformat(),
"schedule_id": schedule_id}
assert prof.started_at >= intro.finished_at, "同源串行:探查做完,剖析才开始"
assert prof.status in ("succeeded", "partial"), prof.error
# 同一次触发再来一遍:交回已有的那一条,不再建任务
again = await service.fire_once(schedule_id, at, "manual", actor="测试员")
assert again.id == fire.id
finally:
await drop_source(sid)
async def test_profile_off_records_steps_not_run_and_still_introspects(client, tmp_path, made):
sid = await _source(client, tmp_path / "b.db", profile=False)
try:
schedule_id = await _schedule(sid)
made.append(schedule_id)
fire = await service.fire_once(schedule_id, utcnow().replace(microsecond=0), "manual")
assert fire.status == "started"
assert [j["kind"] for j in fire.inputs["jobs"]] == ["introspect"]
assert {s["step"] for s in fire.inputs["skipped"]} == {"profile_needed", "observe"}
assert "剖析需要的表没有执行" in fire.reason and "取值观察没有执行" in fire.reason and "数据剖析" in fire.reason
await _wait_jobs(fire)
# 只选观察一步:剖析没开就整次没执行
async with SessionLocal() as session:
row = await session.get(DataSource, sid)
row.options = sm.with_steps(row.options, ["observe"], actor="测试员")
await session.commit()
fire = await service.fire_once(schedule_id, utcnow().replace(microsecond=0) + timedelta(seconds=1), "manual")
assert fire.status == "failed_to_start" and "取值观察没有执行" in fire.reason
finally:
await drop_source(sid)
async def test_disabled_missing_and_busy_sources(client, tmp_path, made):
sid = await _source(client, tmp_path / "c.db")
try:
schedule_id = await _schedule(sid)
made.append(schedule_id)
async with SessionLocal() as session:
row = await session.get(DataSource, sid)
row.enabled = False
await session.commit()
fire = await service.fire_once(schedule_id, utcnow().replace(microsecond=0), "manual")
assert fire.status == "failed_to_start" and "已停用" in fire.reason
async with SessionLocal() as session:
row = await session.get(DataSource, sid)
row.enabled = True
session.add(IntrospectJob(source_id=sid, status="queued", scope={}))
await session.commit()
fire = await service.fire_once(schedule_id, utcnow().replace(microsecond=0) + timedelta(seconds=2), "manual")
assert fire.status == "skipped_overlap" and "正在探查结构" in fire.reason
async with SessionLocal() as session:
await session.execute(delete(IntrospectJob).where(IntrospectJob.source_id == sid,
IntrospectJob.status == "queued"))
await session.commit()
finally:
await drop_source(sid)
fire = await service.fire_once(schedule_id, utcnow().replace(microsecond=0) + timedelta(seconds=3), "manual")
assert fire.status == "failed_to_start" and fire.reason == sm.SOURCE_GONE
async def test_loop_claims_and_only_the_latest_missed_fire_runs(client, tmp_path, made):
sid = await _source(client, tmp_path / "d.db")
try:
async with SessionLocal() as session:
row = await session.get(DataSource, sid)
row.options = sm.with_steps(row.options, ["introspect"], actor="测试员")
await session.commit()
now = utcnow().replace(second=0, microsecond=0)
# 每 10 分钟一次,停机错过了好几次:只补最近一次,更早的积压只记一条
schedule_id = await _schedule(sid, cron="*/10 * * * *", timezone="UTC",
next_run_at=now - timedelta(minutes=40), misfire_grace_s=3600)
made.append(schedule_id)
fires = await loop.fire_due(now + timedelta(minutes=1))
assert sorted(f.status for f in fires) == ["skipped_misfire", "started"], [(f.kind, f.status) for f in fires]
started = next(f for f in fires if f.status == "started")
assert [j["kind"] for j in started.inputs["jobs"]] == ["introspect"]
await _wait_jobs(started)
async with SessionLocal() as session:
sched = await session.get(Schedule, schedule_id)
count = len(list((await session.execute(select(ScheduleFire).where(
ScheduleFire.schedule_id == schedule_id))).scalars()))
assert sched.next_run_at > now and count == 2
finally:
await drop_source(sid)
async def test_steps_api_and_check_target(client, tmp_path):
sid = await _source(client, tmp_path / "e.db")
try:
url = f"/api/datasources/{sid}/maintenance"
body = (await client.get(url)).json()
assert body["steps"] == list(sm.STEPS) and body["profile_enabled"] is True and body["schedules"] == []
assert (await client.put(url, json={"steps": ["nope"]})).status_code == 422
assert (await client.put(url, json={"steps": []})).status_code == 422
r = await client.put(url, json={"steps": ["observe", "introspect"]})
assert r.status_code == 200 and r.json()["steps"] == ["introspect", "observe"], "按固定顺序存"
[shown] = [s["options"] for s in (await client.get("/api/datasources")).json() if s["id"] == sid]
assert sm.MAINTENANCE_OPTION not in shown, "数据源表单不显示、不提交它"
async with SessionLocal() as session:
assert await sm.check_target(session, sid) is None
assert await sm.check_target(session, "missing") == sm.SOURCE_GONE
row = await session.get(DataSource, sid)
row.origin = "upload"
await session.commit()
assert await sm.check_target(session, sid) == sm.NOT_MAINTAINED
finally:
await drop_source(sid)