mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 19:42:02 +00:00
ThearchyHelios
656a550408
M1 途经驳回（编排者已定）：outbox.stage_run_finished 收可选的 rejected_via（缺省 None 即没途经驳回），事件内容带 rejected 和
rejected_via，说法接「（途经驳回）」，模板有 {{ event.rejected }}（true / false）。触发条件缺省只认没途经驳回的，trigger 加
include_rejected（界面「也认途经驳回的」勾选，说明照实写）。runner._finalize 把 _rejected_via(state) 传进来那一下留给编排者
在合并时接（runner 的 _rejected_via 在 W1-b 的分支上）；测试直接调 stage_run_finished 传参覆盖。
C1 去抖积压：以前一拍只走一个窗口，积压时后面的越等越久、全过了宽限，而且每个窗口各记一条跳过，原因还说调度没在跑。
现在 _drain 按发生时刻把等着的事件切成窗口，一拍里把到点的处理完（最多 100 个窗口、5000 条）；排在前面已过补发宽限的合成
一条 skipped_misfire（合并了几次事件、几个窗口、时间跨度，上传表格另写合掉的期次），派给任务时已经晚于宽限才说「调度未运行」，
否则照实说积压；宽限内的照常发起（照重叠规则跳过的照实记）。
C3 source.changed：这一处确实没法和变化记录同一个事务（一次探查、剖析的变化分好几批落库，一批只触发一次要等任务结束才知道），
改成说实话：models_triggers 的说明写清哪两种同事务、哪一种不是。trigger_events 加 dedupe_key（source.changed:<任务号>，唯一），
emit_source_changed 按它去重；调度的一拍（刚起的服务第一拍就补，之后每 10 分钟）由 backfill_source_changes 按任务号补写：2 天内
测到、还开着的要处理级变化，任务已结束却没有事件的补一条。分组在 Python 里做，不用各库写法不同的 JSON 取值。
C4 按期次分窗口：triggers.window_key 让上传表格按期次各自成窗口，9 月、10 月连着启用各有各的触发；同一期的重传照旧合并。
两个窗口结束在同一刻时后一个挪 1 微秒，fire_at 不撞唯一约束。
C5 应用补算：workflow.finished 缺省不认 app_backfill 发起的（升版补算连发好几条、各是某一期的历史输入），trigger 加
include_backfill（界面「也认应用补算的」），选了「只认应用补算」时本来就认；说法写「途经驳回、应用补算的不算」。
C6 示例出口字段：triggers.outputs_of 一处过滤「_」开头的键，真派发、预览、立即运行共用；示例事件也按 matches 跳过缺省不认的
途经驳回、应用补算，取合条件的最近一次。
测试：新加 test_schedule_events_review.py（11 项，先写、先看着失败；去掉 matches 的驳回判断、把每拍窗口数改回 1 回跑，对应用例
各自失败），test_schedule_events_api.py 加一项「也认」两项经接口存取和说法；check-schedules 加「上游工作流的请求体」一段、纯函数
几条、触发历史里合并跳过那一行的夹具。
验证：后端全量 14479 通过；tsc、check-copy；check-schedules 447 项、check-runs 327 项通过（拿锁、--stacks 1）。
278 lines
16 KiB
Python
278 lines
16 KiB
Python
"""数据事件触发复核意见的修正（docs/design/orchestration-round/00-总方案.md §4；scheduler/outbox.py、triggers.py）。
|
||
|
||
- M1 途经驳回：正式运行走驳回出口跑完（succeeded，run.finished 带 rejected_via）照发 workflow.finished，事件内容带 rejected_via、
|
||
说法接「（途经驳回）」、模板有 {{ event.rejected }}；触发条件缺省只认没途经驳回的，include_rejected 时也认；
|
||
- C1 去抖积压：一拍处理到没有到点的窗口为止；已过宽限的几个窗口合成一条跳过（合并了几条、时间跨度），「调度未运行」只在
|
||
派发确实晚到（派给任务的时刻离事件发生超过宽限）时说，否则照实说积压；积压清完之后新来的事件照常触发；
|
||
- C3 source.changed 按任务补写：同一个任务只记一条（dedupe），任务结束后没记上的（进程死了、库被锁）在调度的一拍按 job_id 补写；
|
||
- C4 不同期次的 table.version 不合进同一个窗口：各期有各自的触发；同一期的重传照旧合并；
|
||
- C5 应用补算（app_backfill）发的 workflow.finished 缺省不认，include_backfill 或只认补算时才认；
|
||
- C6 预览、立即运行的示例事件和真派发同一套出口字段过滤（「_」开头的不给模板）。
|
||
|
||
工作流、表格、数据源都是虚构的。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import uuid
|
||
from datetime import timedelta
|
||
|
||
from sqlalchemy import select
|
||
|
||
from app.db.base import SessionLocal, utcnow
|
||
from app.db.models import DataSource, Run, Schedule
|
||
from app.db.models_sources import IntrospectJob, SourceChange
|
||
from app.db.models_triggers import TriggerDelivery, TriggerEvent
|
||
from app.scheduler import outbox, triggers
|
||
from tests.test_schedule_events import ( # noqa: F401 （engine_up：每个测试清掉定时任务和事件）
|
||
deliveries, engine_up, event_schedule, events, fires, formal_run, later, put_event, settle, workflow,
|
||
)
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# M1 途经驳回
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
async def test_rejected_via_is_carried_and_excluded_by_default():
|
||
upstream = await workflow("门店日报")
|
||
plain = await event_schedule(await workflow("区域汇总"),
|
||
{"event": "workflow.finished", "workflow_id": upstream, "debounce_s": 0}, name="不认驳回")
|
||
also = await event_schedule(await workflow("驳回复核", fields=[{"name": "rej"}]),
|
||
{"event": "workflow.finished", "workflow_id": upstream, "debounce_s": 0,
|
||
"include_rejected": True}, template={"rej": "{{ event.rejected }}"}, name="也认驳回")
|
||
a = await formal_run(upstream)
|
||
async with SessionLocal() as session:
|
||
# runner 收尾时把这次运行途经的驳回交进来（W1-b 的 _rejected_via(state)）：这里直接调，覆盖那一下
|
||
await session.execute(TriggerEvent.__table__.delete().where(TriggerEvent.subject_id == upstream))
|
||
outbox.stage_run_finished(session, await session.get(Run, a.id),
|
||
rejected_via=[{"node_id": "gate", "label": "主管审核"}])
|
||
await session.commit()
|
||
[ev] = await events("workflow.finished", upstream)
|
||
assert ev.payload["rejected"] is True and ev.payload["rejected_via"] == [{"node_id": "gate", "label": "主管审核"}]
|
||
assert triggers.event_text(ev.event, ev.subject_name, ev.payload).endswith("（途经驳回）")
|
||
[fire] = await outbox.dispatch_due(later())
|
||
assert fire.schedule_id == also and fire.status == "started" and fire.inputs["inputs"] == {"rej": "true"}
|
||
assert fire.inputs["event"]["text"].endswith("（途经驳回）")
|
||
assert await fires(plain) == [] and await deliveries(plain) == []
|
||
await settle(fire.run_id)
|
||
|
||
|
||
async def test_plain_runs_say_not_rejected():
|
||
upstream = await workflow("门店日报")
|
||
sid = await event_schedule(await workflow("区域汇总", fields=[{"name": "rej"}]),
|
||
{"event": "workflow.finished", "workflow_id": upstream, "debounce_s": 0},
|
||
template={"rej": "{{ event.rejected }}"})
|
||
await formal_run(upstream) # runner 收尾没交 rejected_via：没途经驳回
|
||
[ev] = await events("workflow.finished", upstream)
|
||
assert ev.payload["rejected"] is False and ev.payload["rejected_via"] is None
|
||
[fire] = await outbox.dispatch_due(later())
|
||
assert fire.schedule_id == sid and fire.inputs["inputs"] == {"rej": "false"}
|
||
assert "途经驳回" not in fire.inputs["event"]["text"]
|
||
await settle(fire.run_id)
|
||
|
||
|
||
def test_rejected_and_backfill_options_are_described():
|
||
base = triggers.normalize({"event": "workflow.finished", "workflow_id": "wf"})
|
||
assert "（途经驳回、应用补算的不算；去抖 1 分钟）" in triggers.describe(base, "门店日报")
|
||
both = triggers.normalize({"event": "workflow.finished", "workflow_id": "wf", "include_rejected": True,
|
||
"include_backfill": True})
|
||
assert both["include_rejected"] is True and both["include_backfill"] is True
|
||
assert "（途经驳回、应用补算的也算；去抖 1 分钟）" in triggers.describe(both, "门店日报")
|
||
# 只认应用补算的：补算本来就要认，不再说「补算的不算」
|
||
only = triggers.normalize({"event": "workflow.finished", "workflow_id": "wf", "origin_kind": "app_backfill"})
|
||
text = triggers.describe(only, "门店日报")
|
||
assert "只认来源为「应用补算」的" in text and "应用补算的不算" not in text and "途经驳回的不算" in text
|
||
# 别的事件不收这两项
|
||
for key in ("include_rejected", "include_backfill"):
|
||
try:
|
||
triggers.normalize({"event": "table.version", "source_id": "x", key: True})
|
||
except triggers.TriggerError as e:
|
||
assert "上游工作流运行成功结束" in str(e)
|
||
else:
|
||
raise AssertionError(key)
|
||
assert "event.rejected" in triggers.names("workflow.finished")
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# C5 应用补算
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
def test_backfill_runs_are_excluded_unless_asked():
|
||
payload = {"origin_kind": "app_backfill", "rejected": False}
|
||
base = triggers.normalize({"event": "workflow.finished", "workflow_id": "wf"})
|
||
assert not triggers.matches(base, "workflow.finished", "wf", payload)
|
||
assert triggers.matches(base, "workflow.finished", "wf", {**payload, "origin_kind": "launcher"})
|
||
assert triggers.matches({**base, "include_backfill": True}, "workflow.finished", "wf", payload)
|
||
assert triggers.matches({**base, "origin_kind": "app_backfill"}, "workflow.finished", "wf", payload)
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# C6 示例事件的出口字段
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
async def test_sample_outputs_use_the_same_filter_as_dispatch():
|
||
upstream = await workflow("门店日报")
|
||
run = await formal_run(upstream)
|
||
async with SessionLocal() as session:
|
||
await session.execute(Run.__table__.update().where(Run.id == run.id).values(
|
||
output={"r": "日报完成", "_issuance": {"tier": "full"}}))
|
||
await session.commit()
|
||
got = await triggers.sample(session, {"event": "workflow.finished", "workflow_id": upstream})
|
||
assert got.real and got.outputs == {"r": "日报完成"}
|
||
assert triggers.outputs_of({"r": 1, "_x": 2}) == {"r": 1}
|
||
|
||
|
||
async def test_sample_skips_rejected_and_backfill_runs_by_default():
|
||
upstream = await workflow("门店日报")
|
||
plain = await formal_run(upstream)
|
||
backfill = await formal_run(upstream, origin_kind="app_backfill")
|
||
async with SessionLocal() as session:
|
||
got = await triggers.sample(session, {"event": "workflow.finished", "workflow_id": upstream})
|
||
wide = await triggers.sample(session, {"event": "workflow.finished", "workflow_id": upstream,
|
||
"include_backfill": True})
|
||
assert got.payload["run_id"] == plain.id and wide.payload["run_id"] == backfill.id
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# C1 去抖积压
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
async def _backlog(upstream: str, sid: str, *, n: int, gap_s: int, routed_late: bool) -> tuple:
|
||
"""n 条相隔 gap_s 秒的事件，派给了任务、还没处理。routed_late：派给任务的时刻是现在（调度停过）；否则发生后 1 秒就派了
|
||
（调度在跑，只是没处理完）。"""
|
||
t0 = utcnow().replace(microsecond=0) - timedelta(seconds=gap_s * (n - 1) + 61)
|
||
ids = []
|
||
async with SessionLocal() as session:
|
||
for i in range(n):
|
||
at = t0 + timedelta(seconds=gap_s * i)
|
||
ev = TriggerEvent(event="workflow.finished", subject_id=upstream, subject_name="门店日报",
|
||
payload={"workflow_id": upstream, "run_id": f"run-{i}", "rejected": False},
|
||
occurred_at=at, created_at=at, status="routed", matched=1)
|
||
session.add(ev)
|
||
await session.flush()
|
||
session.add(TriggerDelivery(schedule_id=sid, event_id=ev.id, occurred_at=at,
|
||
created_at=utcnow() if routed_late else at + timedelta(seconds=1)))
|
||
ids.append(ev.id)
|
||
await session.commit()
|
||
return t0, ids
|
||
|
||
|
||
async def test_backlog_is_drained_in_one_tick_and_missed_windows_merge_into_one_skip():
|
||
"""12 条相隔 5 分钟、去抖 60 秒、宽限 600 秒：一拍处理完。过了宽限的 10 个窗口合成一条跳过（合并 10 次、写时间跨度），
|
||
宽限内的两个照常发起（第二个照重叠规则可能跳过）；积压清完以后新来的事件照常触发"""
|
||
upstream = await workflow("门店日报")
|
||
sid = await event_schedule(await workflow("区域汇总", fields=[{"name": "run"}]),
|
||
{"event": "workflow.finished", "workflow_id": upstream, "debounce_s": 60}, grace=600,
|
||
template={"run": "{{ event.run_id }}"})
|
||
t0, ids = await _backlog(upstream, sid, n=12, gap_s=300, routed_late=True)
|
||
done = await outbox.dispatch_due(utcnow())
|
||
assert len(done) == 3, [(f.status, f.reason) for f in done]
|
||
skip, *rest = done
|
||
assert skip.status == "skipped_misfire" and skip.kind == "event"
|
||
assert skip.inputs["event"]["merged"] == 10 and skip.inputs["event"]["windows"] == 10
|
||
assert skip.inputs["event"]["event_ids"] == ids[:10]
|
||
assert "服务当时没有派发（调度未运行或已停止）" in skip.reason and " 至 " in skip.reason and "10 次事件" in skip.reason
|
||
assert [f.inputs["event"]["event_ids"] for f in rest] == [[ids[10]], [ids[11]]]
|
||
assert rest[0].status == "started" and rest[1].status in ("started", "skipped_overlap")
|
||
assert {d.status for d in await deliveries(sid)} == {"fired"}
|
||
for f in rest:
|
||
if f.run_id:
|
||
await settle(f.run_id)
|
||
# 积压清完：新来的一条照常触发
|
||
new = await put_event("workflow.finished", upstream, {"workflow_id": upstream, "run_id": "run-new"}, utcnow())
|
||
[fresh] = await outbox.dispatch_due(later(61))
|
||
assert fresh.status == "started" and fresh.inputs["event"]["event_ids"] == [new]
|
||
await settle(fresh.run_id)
|
||
|
||
|
||
async def test_backlog_while_scheduler_ran_says_backlog_not_scheduler_off():
|
||
upstream = await workflow("门店日报")
|
||
sid = await event_schedule(await workflow("区域汇总"),
|
||
{"event": "workflow.finished", "workflow_id": upstream, "debounce_s": 60}, grace=600)
|
||
await _backlog(upstream, sid, n=4, gap_s=600, routed_late=False)
|
||
skip, *rest = await outbox.dispatch_due(utcnow())
|
||
assert skip.status == "skipped_misfire" and skip.inputs["event"]["merged"] == 3
|
||
assert "调度未运行" not in skip.reason and "积压" in skip.reason, skip.reason
|
||
for f in rest:
|
||
if f.run_id:
|
||
await settle(f.run_id)
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# C4 不同期次各自一个窗口
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
async def test_different_periods_get_their_own_windows():
|
||
table = uuid.uuid4().hex
|
||
sid = await event_schedule(await workflow("月报", fields=[{"name": "month"}]),
|
||
{"event": "table.version", "source_id": table, "debounce_s": 60},
|
||
template={"month": "{{ event.period }}"})
|
||
t0 = utcnow().replace(microsecond=0)
|
||
payload = lambda p, s, e: {"table": "门店销售", "period": p, "period_start": s, "period_end": e, # noqa: E731
|
||
"version": f"snap-{p}", "replaced": False}
|
||
sep = await put_event("table.version", table, payload("2026-09", "2026-09-01", "2026-09-30"), t0, name="门店销售")
|
||
oct_ = await put_event("table.version", table, payload("2026-10", "2026-10-01", "2026-10-31"),
|
||
t0 + timedelta(seconds=30), name="门店销售")
|
||
again = await put_event("table.version", table, {**payload("2026-09", "2026-09-01", "2026-09-30"), "replaced": True},
|
||
t0 + timedelta(seconds=40), name="门店销售")
|
||
[first] = await outbox.dispatch_due(t0 + timedelta(seconds=61))
|
||
# 9 月一个窗口：9 月和 9 月的重传合在一起（内容取最新的那一条），10 月不合进来
|
||
assert first.inputs["event"]["event_ids"] == [sep, again] and first.inputs["inputs"] == {"month": "2026-09"}
|
||
await settle(first.run_id)
|
||
[second] = await outbox.dispatch_due(t0 + timedelta(seconds=91))
|
||
assert second.inputs["event"]["event_ids"] == [oct_] and second.inputs["inputs"] == {"month": "2026-10"}
|
||
assert second.fire_at == t0 + timedelta(seconds=90) and second.status == "started"
|
||
await settle(second.run_id)
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# C3 source.changed 按任务补写
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
async def _source_with_changes(job_status: str) -> tuple[str, str]:
|
||
async with SessionLocal() as session:
|
||
source = DataSource(name=f"门店库-{uuid.uuid4().hex[:6]}", kind="sqlite", database=":memory:")
|
||
session.add(source)
|
||
await session.flush()
|
||
job = IntrospectJob(source_id=source.id, status=job_status,
|
||
finished_at=utcnow() if job_status != "running" else None)
|
||
session.add(job)
|
||
await session.flush()
|
||
for i in range(2):
|
||
session.add(SourceChange(source_id=source.id, source_name=source.name, table_name="orders",
|
||
column_name=f"c{i}", kind="column_removed", severity="action",
|
||
detected_by={"job_kind": "introspect", "job_id": job.id},
|
||
fingerprint=uuid.uuid4().hex, status="open"))
|
||
await session.commit()
|
||
return source.id, job.id
|
||
|
||
|
||
async def test_source_changed_is_backfilled_once_per_job():
|
||
"""任务结束了、变化记下了，那条事件却没记上（进程在中间死了）：调度的一拍按 job_id 补写，只补一次；之后再走到
|
||
notify_after_job 也不重复"""
|
||
source_id, job_id = await _source_with_changes("succeeded")
|
||
running_source, _ = await _source_with_changes("running")
|
||
assert await events("source.changed", source_id) == []
|
||
assert await outbox.backfill_source_changes(utcnow()) >= 1
|
||
[ev] = await events("source.changed", source_id)
|
||
assert ev.payload["job_id"] == job_id and ev.payload["counts"]["new"] == 2 and ev.dedupe_key == f"source.changed:{job_id}"
|
||
# 还在进行的任务不补：它结束时自己会记
|
||
assert await events("source.changed", running_source) == []
|
||
await outbox.backfill_source_changes(utcnow())
|
||
from app.data import source_changes
|
||
|
||
await source_changes.notify_after_job(source_id, job_id)
|
||
assert len(await events("source.changed", source_id)) == 1
|
||
|
||
|
||
async def test_dispatch_tick_runs_the_backfill():
|
||
source_id, _ = await _source_with_changes("succeeded")
|
||
outbox._backfilled_at = None # 刚起的服务：第一拍就补
|
||
await outbox.dispatch_due(utcnow())
|
||
assert len(await events("source.changed", source_id)) == 1
|