mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-16 12:02:00 +00:00
f5c94cf12d
变化没有统一的记录:进不了审阅,发不了通知,口径账本的召回也没有可以订阅的接口;目录只能手动探查、手动剖析。现在:
- api/source_changes.py(骨架已在 datasources 通配之前登记,搭建者可用,访问密钥不放):GET /api/datasources/{id}/changes
(缺省开着的和知道了的,按「使用次数权重 × 严重程度」排,带各级别计数)、GET /changes/{change_id}、POST
/changes/{change_id}/resolve。处理动作落到目录已有的写入接口:清理目录里说到被删的列的项(整份写、乐观锁)、加入码值
(apply_patch 配 _merge_codes,含义待填写,依据写变化编号)、重新起草这张表(模型用不了时只按表结构起草)、重新剖析、
知道了、误报(必须写理由);每个动作写 resolution,动了目录的记目录版本,建了任务的记任务编号。
- data/source_changes.py 给召回:list_changes(按 seq 游标拉)、affecting(按源和表反查)、resolve、triggers_recall(只认
结构类、处理时改了目录的、触及有效记录条件的新码值)。
- 通知 source.changed 的发送函数 notify/service.notify_source_changed:只推还要人看的「要处理」级变化,按源汇总,消息里
只有源名、各级别条数和链接(D9);探查、剖析任务结束时有新的要处理变化就调。这一行的订阅范围第 4 波有了界面才打开,
现在推不出去(照实)。
- 定时维护(data/source_maintenance.py,X4:就是一条 target_kind=source_maintenance 的定时,不另建表):fire 按顺序建
探查任务 → 剖析任务(范围 needed / observe / maintenance 按选的步骤),任务上记 trigger=schedule、fire_at、schedule_id;
剖析等探查进了同源队才建,靠 SourceLanes 串行;剖析没开、数据源停用 / 不存在 / 是上传表格时照实记没执行,同源还有
探查或剖析在做时记跳过;触发记录的 inputs 写明建了哪几个任务、哪几步没执行。check_target 核对目标。维护步骤存在
options.catalog_maintenance,经 GET / PUT /api/datasources/{id}/maintenance 改,数据源表单不显示它(api/datasources.py
的 _MANAGED_OPTIONS 加了这一个键)。
测试:test_source_changes(列表与计数、误报要理由、处理过的 409、清理目录与加入码值记目录版本、剖析没开照实 422、
去重、游标拉取与反查、召回触发、通知只有源名条数链接)、test_source_maintenance(按顺序串行、trigger 与 fire_at、
剖析没开记未执行、停用 / 不存在 / 忙、调度循环只补最近一次且积压只记一条、步骤接口与核对目标)。
211 lines
11 KiB
Python
211 lines
11 KiB
Python
"""数据源的定时维护(app/data/source_maintenance.py,next-round 03 B4、总方案 X4)。
|
||
|
||
定时只有一套:维护就是一条 target_kind=source_maintenance 的 Schedule,认领、补跑、积压只记一条都是 scheduler 那一套。
|
||
这里守:
|
||
|
||
- 到点按顺序建探查任务 → 剖析任务(范围按选的步骤),任务上记 trigger=schedule 和 fire_at;剖析排在同一次触发的探查后面
|
||
(同源队串行:探查做完剖析才开始);触发记录写明建了哪几个任务;
|
||
- 照实记没执行的:剖析没开时剖析、观察两步不执行(探查照做);数据源停用、不存在、是上传表格时整次不执行;上一次的探查
|
||
还没做完时这次跳过;
|
||
- 调度循环的认领:错过的只补最近一次、积压只记一条;
|
||
- 维护步骤的接口:不合规 422,合规存进 options、数据源表单不显示它;核对目标(check_target)。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import sqlite3
|
||
import uuid
|
||
from datetime import timedelta
|
||
|
||
import pytest
|
||
from httpx import ASGITransport, AsyncClient
|
||
from sqlalchemy import delete, select
|
||
|
||
from app.data import catalog, catalog_profile_jobs, introspect_jobs, source_maintenance as sm
|
||
from app.db.base import SessionLocal, utcnow
|
||
from app.db.models import CatalogProfileJob, DataSource, Schedule, ScheduleFire
|
||
from app.db.models_sources import IntrospectJob
|
||
from app.main import app
|
||
from app.scheduler import loop, service
|
||
from tests.fixtures.sources import drop_source, introspect_now
|
||
|
||
|
||
@pytest.fixture
|
||
async def client():
|
||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
|
||
yield c
|
||
|
||
|
||
async def _source(client, path, *, profile: bool = True) -> str:
|
||
db = sqlite3.connect(path)
|
||
db.executescript("""
|
||
CREATE TABLE orders (id INTEGER PRIMARY KEY, status VARCHAR(10), paid_on DATE);
|
||
INSERT INTO orders VALUES (1, 'paid', '2026-10-01'), (2, 'void', '2026-10-02');
|
||
""")
|
||
db.commit()
|
||
db.close()
|
||
r = await client.post("/api/datasources", json={"name": f"mnt_{uuid.uuid4().hex[:8]}", "kind": "sqlite",
|
||
"database": str(path), "readonly": True})
|
||
sid = r.json()["id"]
|
||
await introspect_now(client, sid)
|
||
async with SessionLocal() as session:
|
||
row = await session.get(DataSource, sid)
|
||
if profile:
|
||
row.options = {**(row.options or {}), "catalog_profile": {"enabled": True}}
|
||
await session.commit()
|
||
await catalog.write_entry(session, sid, "orders", {
|
||
"columns": {"status": {"codes": catalog.make_item({"paid": "已支付", "void": "作废"}, "human")}},
|
||
}, if_version=0, actor="测试员")
|
||
return sid
|
||
|
||
|
||
async def _schedule(sid: str, **over) -> str:
|
||
async with SessionLocal() as session:
|
||
fields = {"name": f"维护-{uuid.uuid4().hex[:6]}", "cron": "0 3 * * *", "timezone": "Asia/Shanghai",
|
||
**service.target_columns("source_maintenance", sid), **over}
|
||
row = Schedule(**fields)
|
||
session.add(row)
|
||
await session.commit()
|
||
return row.id
|
||
|
||
|
||
@pytest.fixture
|
||
async def made():
|
||
ids: list[str] = []
|
||
yield ids
|
||
async with SessionLocal() as session:
|
||
await session.execute(delete(Schedule).where(Schedule.id.in_(ids)))
|
||
await session.commit()
|
||
|
||
|
||
async def _wait_jobs(fire: ScheduleFire) -> None:
|
||
for job in (fire.inputs or {}).get("jobs") or []:
|
||
manager = introspect_jobs.manager if job["kind"] == "introspect" else catalog_profile_jobs.manager
|
||
await manager.wait(job["id"], timeout=60)
|
||
|
||
|
||
async def test_fire_builds_introspect_then_profile_in_order(client, tmp_path, made):
|
||
sid = await _source(client, tmp_path / "a.db")
|
||
try:
|
||
schedule_id = await _schedule(sid)
|
||
made.append(schedule_id)
|
||
at = utcnow().replace(microsecond=0)
|
||
fire = await service.fire_once(schedule_id, at, "manual", actor="测试员")
|
||
assert fire.status == "started" and fire.reason is None, fire.reason
|
||
jobs = fire.inputs["jobs"]
|
||
assert [j["kind"] for j in jobs] == ["introspect", "profile"] and jobs[1]["scope"] == "maintenance"
|
||
assert fire.inputs["steps"] == list(sm.STEPS)
|
||
await _wait_jobs(fire)
|
||
async with SessionLocal() as session:
|
||
intro = await session.get(IntrospectJob, jobs[0]["id"])
|
||
prof = await session.get(CatalogProfileJob, jobs[1]["id"])
|
||
assert intro.trigger == "schedule" and intro.scope["fire_at"] == at.isoformat()
|
||
assert intro.scope["schedule_id"] == schedule_id and intro.created_by == "测试员"
|
||
assert catalog_profile_jobs.trigger_of(prof) == {"trigger": "schedule", "fire_at": at.isoformat(),
|
||
"schedule_id": schedule_id}
|
||
assert prof.started_at >= intro.finished_at, "同源串行:探查做完,剖析才开始"
|
||
assert prof.status in ("succeeded", "partial"), prof.error
|
||
# 同一次触发再来一遍:交回已有的那一条,不再建任务
|
||
again = await service.fire_once(schedule_id, at, "manual", actor="测试员")
|
||
assert again.id == fire.id
|
||
finally:
|
||
await drop_source(sid)
|
||
|
||
|
||
async def test_profile_off_records_steps_not_run_and_still_introspects(client, tmp_path, made):
|
||
sid = await _source(client, tmp_path / "b.db", profile=False)
|
||
try:
|
||
schedule_id = await _schedule(sid)
|
||
made.append(schedule_id)
|
||
fire = await service.fire_once(schedule_id, utcnow().replace(microsecond=0), "manual")
|
||
assert fire.status == "started"
|
||
assert [j["kind"] for j in fire.inputs["jobs"]] == ["introspect"]
|
||
assert {s["step"] for s in fire.inputs["skipped"]} == {"profile_needed", "observe"}
|
||
assert "剖析需要的表没有执行" in fire.reason and "取值观察没有执行" in fire.reason and "数据剖析" in fire.reason
|
||
await _wait_jobs(fire)
|
||
# 只选观察一步:剖析没开就整次没执行
|
||
async with SessionLocal() as session:
|
||
row = await session.get(DataSource, sid)
|
||
row.options = sm.with_steps(row.options, ["observe"], actor="测试员")
|
||
await session.commit()
|
||
fire = await service.fire_once(schedule_id, utcnow().replace(microsecond=0) + timedelta(seconds=1), "manual")
|
||
assert fire.status == "failed_to_start" and "取值观察没有执行" in fire.reason
|
||
finally:
|
||
await drop_source(sid)
|
||
|
||
|
||
async def test_disabled_missing_and_busy_sources(client, tmp_path, made):
|
||
sid = await _source(client, tmp_path / "c.db")
|
||
try:
|
||
schedule_id = await _schedule(sid)
|
||
made.append(schedule_id)
|
||
async with SessionLocal() as session:
|
||
row = await session.get(DataSource, sid)
|
||
row.enabled = False
|
||
await session.commit()
|
||
fire = await service.fire_once(schedule_id, utcnow().replace(microsecond=0), "manual")
|
||
assert fire.status == "failed_to_start" and "已停用" in fire.reason
|
||
async with SessionLocal() as session:
|
||
row = await session.get(DataSource, sid)
|
||
row.enabled = True
|
||
session.add(IntrospectJob(source_id=sid, status="queued", scope={}))
|
||
await session.commit()
|
||
fire = await service.fire_once(schedule_id, utcnow().replace(microsecond=0) + timedelta(seconds=2), "manual")
|
||
assert fire.status == "skipped_overlap" and "正在探查结构" in fire.reason
|
||
async with SessionLocal() as session:
|
||
await session.execute(delete(IntrospectJob).where(IntrospectJob.source_id == sid,
|
||
IntrospectJob.status == "queued"))
|
||
await session.commit()
|
||
finally:
|
||
await drop_source(sid)
|
||
fire = await service.fire_once(schedule_id, utcnow().replace(microsecond=0) + timedelta(seconds=3), "manual")
|
||
assert fire.status == "failed_to_start" and fire.reason == sm.SOURCE_GONE
|
||
|
||
|
||
async def test_loop_claims_and_only_the_latest_missed_fire_runs(client, tmp_path, made):
|
||
sid = await _source(client, tmp_path / "d.db")
|
||
try:
|
||
async with SessionLocal() as session:
|
||
row = await session.get(DataSource, sid)
|
||
row.options = sm.with_steps(row.options, ["introspect"], actor="测试员")
|
||
await session.commit()
|
||
now = utcnow().replace(second=0, microsecond=0)
|
||
# 每 10 分钟一次,停机错过了好几次:只补最近一次,更早的积压只记一条
|
||
schedule_id = await _schedule(sid, cron="*/10 * * * *", timezone="UTC",
|
||
next_run_at=now - timedelta(minutes=40), misfire_grace_s=3600)
|
||
made.append(schedule_id)
|
||
fires = await loop.fire_due(now + timedelta(minutes=1))
|
||
assert sorted(f.status for f in fires) == ["skipped_misfire", "started"], [(f.kind, f.status) for f in fires]
|
||
started = next(f for f in fires if f.status == "started")
|
||
assert [j["kind"] for j in started.inputs["jobs"]] == ["introspect"]
|
||
await _wait_jobs(started)
|
||
async with SessionLocal() as session:
|
||
sched = await session.get(Schedule, schedule_id)
|
||
count = len(list((await session.execute(select(ScheduleFire).where(
|
||
ScheduleFire.schedule_id == schedule_id))).scalars()))
|
||
assert sched.next_run_at > now and count == 2
|
||
finally:
|
||
await drop_source(sid)
|
||
|
||
|
||
async def test_steps_api_and_check_target(client, tmp_path):
|
||
sid = await _source(client, tmp_path / "e.db")
|
||
try:
|
||
url = f"/api/datasources/{sid}/maintenance"
|
||
body = (await client.get(url)).json()
|
||
assert body["steps"] == list(sm.STEPS) and body["profile_enabled"] is True and body["schedules"] == []
|
||
assert (await client.put(url, json={"steps": ["nope"]})).status_code == 422
|
||
assert (await client.put(url, json={"steps": []})).status_code == 422
|
||
r = await client.put(url, json={"steps": ["observe", "introspect"]})
|
||
assert r.status_code == 200 and r.json()["steps"] == ["introspect", "observe"], "按固定顺序存"
|
||
[shown] = [s["options"] for s in (await client.get("/api/datasources")).json() if s["id"] == sid]
|
||
assert sm.MAINTENANCE_OPTION not in shown, "数据源表单不显示、不提交它"
|
||
async with SessionLocal() as session:
|
||
assert await sm.check_target(session, sid) is None
|
||
assert await sm.check_target(session, "missing") == sm.SOURCE_GONE
|
||
row = await session.get(DataSource, sid)
|
||
row.origin = "upload"
|
||
await session.commit()
|
||
assert await sm.check_target(session, sid) == sm.NOT_MAINTAINED
|
||
finally:
|
||
await drop_source(sid)
|