mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 15:12:00 +00:00
ThearchyHelios
281 lines
16 KiB
Python
281 lines
16 KiB
Python
"""同一个子工作流节点在一次运行里执行多次（循环体里、退回修改绕回来）：子图里的节点每次执行都要分得开。
|
|
|
|
以前子图每次都从空状态起步，里面的节点每次都记成「第 1 次执行」：
|
|
- 台账里同一个节点（「sub/fetch」）几条查询都记 exec 1，分不出是第几次；
|
|
- 写外部系统的幂等键按 (运行, 节点 id, 第几次执行) 算，子图里的节点 id 还是子图里的（pay，不带前缀）：循环第二轮的
|
|
写入算出同一个键，找到第一轮生效的记录，交回第一轮的回执、这一轮不发送——外部系统少收一笔，界面上只有一句「已经生效」；
|
|
父图里同名的节点、两个子工作流节点套同一个工作流时也撞同一个键。
|
|
现在：子图里的节点接着父运行里同一个子工作流节点之前几次数（exec 1、2…），幂等键按这次运行里的节点 id（「sub/pay」）算；
|
|
升级前停下的运行（子图里的写入按子图里的 id 记的），恢复时照旧认得出已经生效的那一笔，不重发。
|
|
图、能力、外部系统都是虚构的。
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
|
|
import pytest
|
|
from sqlalchemy import select
|
|
|
|
from app.db.base import SessionLocal
|
|
from app.db.models import ActionRecord, Run, RunEvent
|
|
from app.engine.runner import run_manager
|
|
from app.process import actions
|
|
from tests import ledger_samples as S
|
|
from tests.fakes.fake_erp import PAY, FakeErp, endpoint_row, install
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
async def engine_up():
|
|
await run_manager.setup()
|
|
yield
|
|
await run_manager.shutdown()
|
|
|
|
|
|
def node(nid, ntype, label=None, **config):
|
|
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": label or nid, "config": config}}
|
|
|
|
|
|
def edge(source, target, handle=None):
|
|
return {"source": source, "target": target, **({"sourceHandle": handle} if handle else {})}
|
|
|
|
|
|
async def _finish(graph: dict, **kwargs) -> Run:
|
|
run = await run_manager.start(graph=graph, input_payload={}, **kwargs)
|
|
for _ in range(600):
|
|
await asyncio.sleep(0.05)
|
|
async with SessionLocal() as session:
|
|
row = await session.get(Run, run.id)
|
|
if row.status in ("succeeded", "failed", "interrupted") and not run_manager.is_active(run.id):
|
|
return row
|
|
raise AssertionError(f"运行 {run.id} 没跑完：{row.status}")
|
|
|
|
|
|
async def _events(run_id: str, kind: str, node_id: str | None = None) -> list[RunEvent]:
|
|
async with SessionLocal() as session:
|
|
q = select(RunEvent).where(RunEvent.run_id == run_id, RunEvent.type == kind)
|
|
if node_id:
|
|
q = q.where(RunEvent.node_id == node_id)
|
|
return list((await session.execute(q.order_by(RunEvent.seq))).scalars())
|
|
|
|
|
|
def looped(sub_id: str, *, before: list[dict] | None = None) -> dict:
|
|
"""开始 → (before…) → 循环两轮，每轮跑一次子工作流 → 成果。"""
|
|
nodes = [node("start", "input"), *(before or []),
|
|
node("each", "loop", "逐项", mode="foreach", items="[1, 2]", item_var="it", max_iterations=5),
|
|
node("sub", "subgraph", "子流程", workflow_id=sub_id, workflow_version=1, input={"n": "{{ vars.it }}"}),
|
|
node("out", "output", fields=[{"name": "r", "value": "完成"}])]
|
|
chain = ["start", *[b["id"] for b in before or []], "each"]
|
|
edges = [edge(a, b) for a, b in zip(chain, chain[1:])]
|
|
edges += [edge("each", "sub", "body"), edge("sub", "each"), edge("each", "out", "done")]
|
|
return {"nodes": nodes, "edges": edges}
|
|
|
|
|
|
async def test_inner_steps_count_on_across_invocations(tmp_path):
|
|
await S.fresh_source(tmp_path / "shop.db")
|
|
inner = S.chain(node("s", "input"), node("fetch", "tool", tool=S.tool(), args={"sql": "SELECT COUNT(*) AS n FROM orders"}),
|
|
node("o", "output", fields=[{"name": "n", "value": "{{ nodes.fetch.text }}"}]))
|
|
sub_id = await S.publish("取数子流程", graph=inner, level="draft")
|
|
row = await _finish(looped(sub_id))
|
|
assert row.status == "succeeded", row.error
|
|
finished = await _events(row.id, "node.finished", "sub/fetch")
|
|
execs = [e["exec"] for f in finished for e in f.data.get("evidence") or [] if e.get("kind") == "query"]
|
|
assert execs == [1, 2], "第二轮是子图里那一步的第 2 次执行，不再又记成第 1 次"
|
|
|
|
|
|
@pytest.fixture
|
|
def erp() -> FakeErp:
|
|
return FakeErp()
|
|
|
|
|
|
async def _unattended(monkeypatch, erp) -> None:
|
|
"""付款能力设成可以无人值守写入、节点不要确认：两轮都直接发，不停下等人（和确认无关，只看幂等键分不分得开）。"""
|
|
caps = install(monkeypatch, erp, await endpoint_row(test=True), lookup=False)
|
|
caps[PAY].unattended_write = True
|
|
|
|
|
|
def pay(nid: str = "pay") -> dict:
|
|
return node(nid, "tool", "付款", tool=PAY, approval="never", args={"amount": "{{ input.n }}", "card": "6222020200112233"})
|
|
|
|
|
|
async def test_a_write_inside_a_loop_is_sent_on_every_round(monkeypatch, erp):
|
|
await _unattended(monkeypatch, erp)
|
|
inner = S.chain(node("s", "input", fields=[{"name": "n"}]), pay(),
|
|
node("o", "output", fields=[{"name": "单号", "value": "{{ nodes.pay.receipt.id }}"}]))
|
|
sub_id = await S.publish("付款子流程", graph=inner, level="draft")
|
|
row = await _finish(looped(sub_id))
|
|
assert row.status == "succeeded", row.error
|
|
assert len(erp.writes("test")) == 2, "第二轮的写入照发，不交回第一轮的回执"
|
|
async with SessionLocal() as session:
|
|
records = list((await session.execute(select(ActionRecord).where(ActionRecord.run_id == row.id)
|
|
.order_by(ActionRecord.created_at))).scalars())
|
|
assert [(r.node_id, r.exec_no) for r in records] == [("sub/pay", 1), ("sub/pay", 2)]
|
|
assert len({r.action_key for r in records}) == 2
|
|
assert not [e for e in await _events(row.id, "log") if e.data.get("code") == "action_already_applied"]
|
|
|
|
|
|
async def test_a_namesake_in_the_parent_does_not_share_the_key(monkeypatch, erp):
|
|
"""父图里也有一个 pay：子图里的 pay 不再和它撞同一个幂等键。"""
|
|
await _unattended(monkeypatch, erp)
|
|
inner = S.chain(node("s", "input", fields=[{"name": "n"}]), pay(), node("o", "output", fields=[{"name": "r", "value": "x"}]))
|
|
sub_id = await S.publish("付款子流程", graph=inner, level="draft")
|
|
graph = S.chain(node("start", "input", fields=[{"name": "n"}]), pay(),
|
|
node("sub", "subgraph", "子流程", workflow_id=sub_id, workflow_version=1, input={"n": "7"}),
|
|
node("out", "output", fields=[{"name": "r", "value": "完成"}]))
|
|
run = await run_manager.start(graph=graph, input_payload={"n": 5})
|
|
for _ in range(600):
|
|
await asyncio.sleep(0.05)
|
|
async with SessionLocal() as session:
|
|
row = await session.get(Run, run.id)
|
|
if row.status in ("succeeded", "failed") and not run_manager.is_active(run.id):
|
|
break
|
|
assert row.status == "succeeded", row.error
|
|
assert len(erp.writes("test")) == 2
|
|
async with SessionLocal() as session:
|
|
ids = sorted(r.node_id for r in (await session.execute(
|
|
select(ActionRecord).where(ActionRecord.run_id == row.id))).scalars())
|
|
assert ids == ["pay", "sub/pay"]
|
|
|
|
|
|
async def test_a_write_recorded_before_the_upgrade_is_still_found(monkeypatch, erp):
|
|
"""升级前子图里的写入按子图里的 id（pay）记幂等键、记录上的 node_id 也是它；事件一向按这次运行里的 id（sub/pay）记。
|
|
写入节点本身重新执行时（结果未知核实之后接着跑）：事件说是这个节点的、内容指纹也是这一次的，照旧按那个键交回回执、不重发；
|
|
事件说是父图同名节点的、或者内容不一样（跨升级的循环，第二轮按老算法也是第 1 次执行）的不认。"""
|
|
from app.core.events import EventType
|
|
from app.db.models import RunEvent
|
|
|
|
caps = install(monkeypatch, erp, await endpoint_row(test=True), lookup=False)
|
|
args = {"amount": "5", "card": "6222020200112233"}
|
|
digest = actions.payload_hash(PAY, actions.typed_args(caps[PAY], args))
|
|
|
|
async def old_record(writer: str, payload: str) -> tuple[str, str]:
|
|
async with SessionLocal() as session:
|
|
run = Run(graph={"nodes": [node("sub", "subgraph")], "edges": []}, status="running", run_class="exploratory")
|
|
session.add(run)
|
|
await session.flush()
|
|
legacy = actions.action_key(run.id, "pay", 1)
|
|
rec = ActionRecord(action_key=legacy, attempt=1, idem_key=f"{legacy}:1", run_id=run.id, node_id="pay",
|
|
exec_no=1, capability=PAY, environment="test", payload_hash=payload, state="confirmed",
|
|
receipt={"summary": {"id": "PAY-OLD"}})
|
|
session.add(rec)
|
|
await session.flush()
|
|
session.add(RunEvent(run_id=run.id, seq=1, type=str(EventType.ACTION_PREPARED), node_id=writer, ts=0.0,
|
|
data={"action_id": rec.id, "capability": PAY, "payload_hash": payload}))
|
|
await session.commit()
|
|
return run.id, legacy
|
|
|
|
run_id, legacy = await old_record("sub/pay", digest)
|
|
found = await actions.prepare(run_id=run_id, node_id="sub/pay", exec_no=1, capability=caps[PAY], args=args,
|
|
legacy_node_id="pay")
|
|
assert found.key == legacy and found.receipt is not None
|
|
fresh = await actions.prepare(run_id=run_id, node_id="sub/pay", exec_no=1, capability=caps[PAY], args=args)
|
|
assert fresh.key != legacy and fresh.receipt is None
|
|
# 父图里同名的 pay 按自己的键（和旧键一样）找到这一笔：事件说是 sub/pay 的，不是自己的，换一个只属于自己的键
|
|
mine = await actions.prepare(run_id=run_id, node_id="pay", exec_no=1, capability=caps[PAY], args=args)
|
|
assert mine.key != legacy and mine.receipt is None
|
|
# 内容不一样（这一次是 7）：不当成这一次已经生效
|
|
other = await actions.prepare(run_id=run_id, node_id="sub/pay", exec_no=1, capability=caps[PAY],
|
|
args={**args, "amount": "7"}, legacy_node_id="pay")
|
|
assert other.key != legacy and other.receipt is None
|
|
# 那一笔是父图 pay 写的（事件说 pay）：子图里的 pay 不认
|
|
run_id, legacy = await old_record("pay", digest)
|
|
theirs = await actions.prepare(run_id=run_id, node_id="sub/pay", exec_no=1, capability=caps[PAY], args=args,
|
|
legacy_node_id="pay")
|
|
assert theirs.key != legacy and theirs.receipt is None
|
|
own = await actions.prepare(run_id=run_id, node_id="pay", exec_no=1, capability=caps[PAY], args=args)
|
|
assert own.key == legacy and own.receipt is not None
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# 跨升级：升级前的运行按老算法记的键，升级后接着跑
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
def _old_code(monkeypatch):
|
|
"""装成升级前的代码：子工作流里的写入按子图里的 id 算幂等键、不认旧键，子图跑过的步骤不交回父图（exec 每次从 1 数）。
|
|
交回一个函数，调了就换回现在的代码（同一个进程里接力跑，等于升级后重启接着跑；能力登记的替身要重新装）。"""
|
|
from app.engine.nodes import multi, tools
|
|
|
|
monkeypatch.setattr(tools, "_action_ids", lambda ctx: (ctx.node.id, None))
|
|
monkeypatch.setattr(multi, "_forwarded_trail", lambda ran, prefix: [])
|
|
return monkeypatch.undo
|
|
|
|
|
|
def _amounts(erp) -> list[str]:
|
|
return [str((w.get("body") or {}).get("amount")) for w in erp.writes("test")]
|
|
|
|
|
|
async def _until_status(run_id: str, statuses: tuple[str, ...]) -> Run:
|
|
for _ in range(600):
|
|
await asyncio.sleep(0.05)
|
|
async with SessionLocal() as session:
|
|
row = await session.get(Run, run_id)
|
|
if row.status in statuses and not run_manager.is_active(run_id):
|
|
return row
|
|
raise AssertionError(f"运行 {run_id} 停在 {row.status}")
|
|
|
|
|
|
async def test_an_unknown_write_inside_a_subworkflow_with_a_namesake_in_the_parent_survives_the_upgrade(monkeypatch, erp):
|
|
"""升级前：子工作流里的 pay（1）发出去、结果未知、运行挂起，键按子图里的 id（pay）记——和父图里同名的 pay 一样。升级后
|
|
核实为已生效、接着跑：子图里的 pay 认出自己那一笔（事件记着是「sub/pay」写的），不再发；父图的 pay（99）按自己的键找到
|
|
的是子图那一笔、不是自己的，换用自己的键照常发。1 和 99 各只发一次。"""
|
|
from app.db.models import Task
|
|
from app.process import tasks
|
|
from tests.test_process_actions import ANN
|
|
|
|
await _unattended(monkeypatch, erp)
|
|
inner = S.chain(node("s", "input", fields=[{"name": "n"}]), pay(),
|
|
node("o", "output", fields=[{"name": "r", "value": "x"}]))
|
|
sub_id = await S.publish("付款子流程", graph=inner, level="draft")
|
|
graph = S.chain(node("start", "input", fields=[{"name": "n"}]),
|
|
node("sub", "subgraph", "子流程", workflow_id=sub_id, workflow_version=1, input={"n": "1"}),
|
|
pay(), node("out", "output", fields=[{"name": "r", "value": "完成"}]))
|
|
upgrade = _old_code(monkeypatch)
|
|
erp.mode = "timeout_after_write"
|
|
run = await run_manager.start(graph=graph, input_payload={"n": 99})
|
|
parked = await _until_status(run.id, ("interrupted", "failed", "succeeded"))
|
|
assert parked.status == "interrupted", parked.error
|
|
assert _amounts(erp) == ["1"]
|
|
upgrade()
|
|
await _unattended(monkeypatch, erp) # undo 连能力登记的替身一起撤了：重新装上
|
|
erp.mode = "ok"
|
|
async with SessionLocal() as session:
|
|
task = (await session.execute(select(Task).where(Task.run_id == run.id, Task.kind == "reconcile",
|
|
Task.status == "pending"))).scalars().first()
|
|
assert task is not None
|
|
await tasks.respond(task.id, ANN, decision="confirmed", values={"external_id": "PAY-1"})
|
|
final = await _until_status(run.id, ("succeeded", "failed"))
|
|
assert final.status == "succeeded", final.error
|
|
assert sorted(_amounts(erp)) == ["1", "99"], _amounts(erp)
|
|
|
|
|
|
async def test_a_loop_across_the_upgrade_sends_the_second_round(monkeypatch, erp):
|
|
"""升级前跑完第一轮（子图里 pay 发了 1）、停在父图的审批上；升级后批准，第二轮按老算法同样是第 1 次执行，找到第一轮那一笔：
|
|
内容指纹不一样（这一轮是 2），不当成已经生效，照常发；也不记「已经生效」的提示。"""
|
|
await _unattended(monkeypatch, erp)
|
|
inner = S.chain(node("s", "input", fields=[{"name": "n"}]), pay(),
|
|
node("o", "output", fields=[{"name": "r", "value": "x"}]))
|
|
sub_id = await S.publish("付款子流程", graph=inner, level="draft")
|
|
nodes = [node("start", "input"),
|
|
node("each", "loop", "逐项", mode="foreach", items="[1, 2]", item_var="it", max_iterations=5),
|
|
node("sub", "subgraph", "子流程", workflow_id=sub_id, workflow_version=1, input={"n": "{{ vars.it }}"}),
|
|
node("gate", "human", "把关", mode="approve", title="下一轮？"),
|
|
node("out", "output", fields=[{"name": "r", "value": "完成"}])]
|
|
edges = [edge("start", "each"), edge("each", "sub", "body"), edge("sub", "gate"), edge("gate", "each", "approved"),
|
|
edge("each", "out", "done")]
|
|
upgrade = _old_code(monkeypatch)
|
|
run = await run_manager.start(graph={"nodes": nodes, "edges": edges}, input_payload={})
|
|
parked = await _until_status(run.id, ("interrupted", "failed", "succeeded"))
|
|
assert parked.status == "interrupted", parked.error
|
|
assert _amounts(erp) == ["1"]
|
|
upgrade()
|
|
await _unattended(monkeypatch, erp)
|
|
await run_manager.resume(run.id, {"approved": True})
|
|
parked = await _until_status(run.id, ("interrupted", "failed", "succeeded"))
|
|
assert parked.status == "interrupted", parked.error
|
|
assert _amounts(erp) == ["1", "2"], "第二轮照发，不交回第一轮的回执"
|
|
assert not [e for e in await _events(run.id, "log") if e.data.get("code") == "action_already_applied"]
|
|
await run_manager.resume(run.id, {"approved": True})
|
|
final = await _until_status(run.id, ("succeeded", "failed"))
|
|
assert final.status == "succeeded", final.error
|
|
assert _amounts(erp) == ["1", "2"]
|