From 87f0d62f9b35d1efec645bdaf2e123821ef65421 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Mon, 19 Oct 2026 11:40:33 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A4=9A=E8=BF=9B=E7=A8=8B=E5=8D=8F=E8=B0=83?= =?UTF-8?q?=EF=BC=88W2-b=EF=BC=89=E7=AC=AC=E4=B8=80=E6=AD=A5=EF=BC=9A?= =?UTF-8?q?=E6=96=B0=E5=A2=9E=20app/cluster=E2=80=94=E2=80=94=E5=B7=A5?= =?UTF-8?q?=E4=BD=9C=E8=BF=9B=E7=A8=8B=E5=9C=A8=E5=BA=93=E9=87=8C=E7=99=BB?= =?UTF-8?q?=E8=AE=B0=E3=80=81=E5=BF=83=E8=B7=B3=E7=BB=AD=E7=A7=9F=E7=BA=A6?= =?UTF-8?q?=EF=BC=88=E5=B1=9E=E4=B8=BB=E9=9A=8F=E8=BF=9B=E7=A8=8B=EF=BC=8C?= =?UTF-8?q?=E8=BF=9B=E7=A8=8B=E6=B2=A1=E4=BA=86=E5=AE=83=E8=AE=A4=E9=A2=86?= =?UTF-8?q?=E7=9A=84=E8=A1=8C=E4=B8=80=E8=B5=B7=E5=88=B0=E6=9C=9F=EF=BC=89?= =?UTF-8?q?=EF=BC=8C=E8=B0=83=E5=BA=A6=E5=BD=93=E7=8F=AD=E7=9A=84=E7=A7=9F?= =?UTF-8?q?=E7=BA=A6=EF=BC=8C=E5=8F=91=E7=BB=99=E5=B1=9E=E4=B8=BB=E8=BF=9B?= =?UTF-8?q?=E7=A8=8B=E7=9A=84=E5=81=9C=E6=AD=A2=20/=20=E8=AE=B0=E4=BA=8B?= =?UTF-8?q?=E4=BB=B6=E8=AF=B7=E6=B1=82=EF=BC=8CPostgreSQL=20=E7=9A=84=20LI?= =?UTF-8?q?STEN/NOTIFY=20=E5=8F=AB=E9=86=92=EF=BC=8C=E8=B7=A8=E8=BF=9B?= =?UTF-8?q?=E7=A8=8B=E7=9A=84=E4=BA=92=E6=96=A5=E9=94=81=EF=BC=88=E5=92=A8?= =?UTF-8?q?=E8=AF=A2=E9=94=81=EF=BC=89=EF=BC=8CSQLite=20=E5=8D=95=E8=BF=9B?= =?UTF-8?q?=E7=A8=8B=E5=AE=88=E5=8D=AB=EF=BC=88=E7=AC=AC=E4=BA=8C=E4=B8=AA?= =?UTF-8?q?=E8=BF=9B=E7=A8=8B=E6=8B=92=E7=BB=9D=E5=90=AF=E5=8A=A8=EF=BC=89?= =?UTF-8?q?=EF=BC=9B=E8=BF=90=E8=A1=8C=E3=80=81=E9=97=AE=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E4=B8=80=E8=BD=AE=E3=80=81=E8=B5=B7=E8=8D=89=20/=20=E5=89=96?= =?UTF-8?q?=E6=9E=90=20/=20=E6=8E=A2=E6=9F=A5=20/=20=E8=AF=84=E6=B5=8B=20/?= =?UTF-8?q?=20=E5=BD=B1=E5=AD=90=E8=B5=B7=E8=8D=89=E3=80=81=E6=96=87?= =?UTF-8?q?=E6=A1=A3=E3=80=81=E5=8F=91=E5=B8=83=E5=8D=95=E8=AE=B0=20owner?= =?UTF-8?q?=EF=BC=8C=E5=90=AF=E5=8A=A8=E6=95=B4=E7=90=86=E6=94=B9=E6=88=90?= =?UTF-8?q?=E5=8F=AA=E6=8E=A5=E6=89=8B=E5=B1=9E=E4=B8=BB=E4=B8=8D=E5=9C=A8?= =?UTF-8?q?=E4=BA=86=E7=9A=84=E8=A1=8C=EF=BC=8C=E5=BD=93=E7=8F=AD=E8=BF=9B?= =?UTF-8?q?=E7=A8=8B=E9=9A=94=E4=B8=80=E4=BC=9A=E5=84=BF=E5=86=8D=E6=8E=A5?= =?UTF-8?q?=E6=89=8B=E4=B8=80=E6=AC=A1=EF=BC=9B=E8=BF=90=E8=A1=8C=E7=9A=84?= =?UTF-8?q?=E5=B9=B6=E5=8F=91=E5=90=8D=E9=A2=9D=E6=94=B9=E6=88=90=E5=BA=93?= =?UTF-8?q?=E9=87=8C=E6=8C=89=E4=BA=A4=E4=BA=92=20/=20=E5=BA=94=E7=94=A8?= =?UTF-8?q?=20/=20=E5=AE=9A=E6=97=B6=20/=20=E7=B3=BB=E7=BB=9F=E5=9B=9B?= =?UTF-8?q?=E7=B1=BB=E5=8F=91=EF=BC=88=E5=90=84=E6=9C=89=E5=85=A8=E7=AB=99?= =?UTF-8?q?=E4=B8=8A=E9=99=90=E5=92=8C=E4=BC=98=E5=85=88=E7=BA=A7=EF=BC=8C?= =?UTF-8?q?=E8=AE=BE=E7=BD=AE=E9=87=8C=E5=8F=AF=E8=B0=83=EF=BC=89=EF=BC=8C?= =?UTF-8?q?=E6=9B=BF=E6=8E=89=E8=BF=9B=E7=A8=8B=E5=86=85=E7=9A=84=E4=BF=A1?= =?UTF-8?q?=E5=8F=B7=E9=87=8F=EF=BC=9B=E5=90=8C=E6=BA=90=E9=98=9F=E8=AE=B0?= =?UTF-8?q?=E5=9C=A8=E5=BA=93=E9=87=8C=EF=BC=9B=E8=BF=90=E8=A1=8C=E5=81=9C?= =?UTF-8?q?=E6=AD=A2=E3=80=81=E8=AE=B0=E4=BA=8B=E4=BB=B6=E8=B7=A8=E8=BF=9B?= =?UTF-8?q?=E7=A8=8B=E4=BA=A4=E7=BB=99=E5=B1=9E=E4=B8=BB=EF=BC=9B=E8=BF=90?= =?UTF-8?q?=E8=A1=8C=E4=BA=8B=E4=BB=B6=E3=80=81=E9=97=AE=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E4=BA=8B=E4=BB=B6=E8=B7=A8=E8=BF=9B=E7=A8=8B=E8=BD=AC=E5=8F=91?= =?UTF-8?q?=EF=BC=88=E5=BA=93=E9=87=8C=E6=8C=89=20seq=20=E8=BF=BD=EF=BC=8C?= =?UTF-8?q?=E4=B8=8D=E8=90=BD=E5=BA=93=E7=9A=84=E6=B5=81=E5=BC=8F=E5=A2=9E?= =?UTF-8?q?=E9=87=8F=E5=92=8C=E4=BF=9D=E6=B4=BB=E5=BF=83=E8=B7=B3=E5=8F=AA?= =?UTF-8?q?=E8=BD=AC=E7=BB=99=E5=9C=A8=E7=9C=8B=E7=9A=84=E8=BF=9B=E7=A8=8B?= =?UTF-8?q?=EF=BC=89=EF=BC=9B=E6=AF=8F=E6=97=A5=E8=8A=B1=E8=B4=B9=E8=B4=A6?= =?UTF-8?q?=E5=85=88=E5=8D=A0=E4=BD=8F=E9=82=A3=E4=B8=80=E8=A1=8C=E5=86=8D?= =?UTF-8?q?=E8=AF=BB=E5=86=99=EF=BC=9B=E8=AE=BF=E9=97=AE=E5=AF=86=E9=92=A5?= =?UTF-8?q?=E9=99=90=E6=B5=81=E3=80=81=E7=99=BB=E5=BD=95=E5=A4=B1=E8=B4=A5?= =?UTF-8?q?=E8=AE=A1=E6=95=B0=E5=9C=A8=20PostgreSQL=20=E6=97=B6=E8=AE=B0?= =?UTF-8?q?=E5=9C=A8=E5=BA=93=E9=87=8C=EF=BC=9B=E7=99=BB=E5=BD=95=E8=AE=BE?= =?UTF-8?q?=E7=BD=AE=E3=80=81=E5=AF=B9=E5=A4=96=E5=9C=B0=E5=9D=80=E3=80=81?= =?UTF-8?q?=E5=90=91=E9=87=8F=E6=A8=A1=E5=9E=8B=E3=80=81=E8=83=BD=E5=8A=9B?= =?UTF-8?q?=E5=BF=AB=E7=85=A7=E3=80=81=E6=95=B0=E6=8D=AE=E6=BA=90=E8=BF=9E?= =?UTF-8?q?=E6=8E=A5=E6=B1=A0=E3=80=81=E5=BA=94=E7=94=A8=E9=A1=B5=E9=9D=A2?= =?UTF-8?q?=E7=BC=93=E5=AD=98=E8=B7=A8=E8=BF=9B=E7=A8=8B=E5=A4=B1=E6=95=88?= =?UTF-8?q?=EF=BC=9B=E4=B8=8A=E4=BC=A0=E7=89=88=E6=9C=AC=E5=BA=93=E3=80=81?= =?UTF-8?q?=E8=AE=B0=E5=BD=95=E8=A1=A8=E3=80=81=E6=8C=89=E9=9C=80=E6=8E=A2?= =?UTF-8?q?=E6=9F=A5=E3=80=81=E5=87=BA=E5=85=B7=E3=80=81=E5=8F=AC=E5=9B=9E?= =?UTF-8?q?=E3=80=81=E8=A1=A5=E7=AE=97=E3=80=81=E6=8C=89=E9=9C=80=E8=A7=86?= =?UTF-8?q?=E5=9B=BE=E3=80=81=E4=B8=AA=E4=BA=BA=E5=88=B7=E6=96=B0=E3=80=81?= =?UTF-8?q?=E5=AE=9A=E6=97=B6=E5=8F=91=E8=B5=B7=E7=9A=84=E9=94=81=E6=8D=A2?= =?UTF-8?q?=E6=88=90=E8=B7=A8=E8=BF=9B=E7=A8=8B=E7=9A=84=E9=94=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/api/access_keys.py | 3 + backend/app/api/apps.py | 6 + backend/app/api/auth.py | 10 +- backend/app/api/evidence.py | 12 +- backend/app/api/knowledge.py | 4 + backend/app/api/runs.py | 28 +- backend/app/api/schedules.py | 2 +- backend/app/api/settings.py | 39 +++ backend/app/api/source_changes.py | 2 +- backend/app/apps/backfill.py | 7 +- backend/app/apps/resolve.py | 9 +- backend/app/auth/keys.py | 10 + backend/app/auth/middleware.py | 2 +- backend/app/auth/sessions.py | 27 ++ backend/app/blueprints/personal.py | 7 +- backend/app/blueprints/publish_plan.py | 21 +- backend/app/capabilities/registry.py | 7 +- backend/app/chat/job.py | 82 +++++- backend/app/chat/log.py | 36 ++- backend/app/cluster/__init__.py | 14 + backend/app/cluster/caches.py | 109 +++++++ backend/app/cluster/claims.py | 45 +++ backend/app/cluster/counters.py | 117 ++++++++ backend/app/cluster/guard.py | 86 ++++++ backend/app/cluster/identity.py | 48 +++ backend/app/cluster/leader.py | 109 +++++++ backend/app/cluster/mutex.py | 111 +++++++ backend/app/cluster/reaper.py | 56 ++++ backend/app/cluster/relay.py | 332 +++++++++++++++++++++ backend/app/cluster/requests.py | 114 ++++++++ backend/app/cluster/service.py | 227 +++++++++++++++ backend/app/cluster/signal.py | 215 ++++++++++++++ backend/app/cluster/slots.py | 320 ++++++++++++++++++++ backend/app/cluster/state.py | 41 +++ backend/app/cluster/workers.py | 145 +++++++++ backend/app/core/bus.py | 12 +- backend/app/core/config.py | 23 ++ backend/app/core/job_runtime.py | 356 ++++++++++++++++++----- backend/app/data/catalog_jobs.py | 29 +- backend/app/data/catalog_profile_jobs.py | 10 +- backend/app/data/engine.py | 7 +- backend/app/data/introspect_jobs.py | 6 +- backend/app/data/source_maintenance.py | 2 +- backend/app/data/source_objects.py | 19 +- backend/app/data/table_versions.py | 50 ++-- backend/app/db/base.py | 21 ++ backend/app/db/models.py | 13 +- backend/app/db/models_blueprints.py | 2 + backend/app/db/models_chat.py | 7 +- backend/app/db/models_cluster.py | 118 ++++++++ backend/app/db/models_eval.py | 6 +- backend/app/db/models_sources.py | 4 +- backend/app/engine/runner.py | 160 +++++++--- backend/app/engine/spend.py | 55 +++- backend/app/eval/runner.py | 19 +- backend/app/eval/snapshots.py | 9 +- backend/app/ledger/issue.py | 8 +- backend/app/ledger/match.py | 12 + backend/app/ledger/recall.py | 134 +++++---- backend/app/main.py | 268 +++++++++++------ backend/app/notify/service.py | 8 +- backend/app/process/actions.py | 18 +- backend/app/process/tasks.py | 16 +- backend/app/records/actions.py | 8 +- backend/app/records/store.py | 11 +- backend/app/records/tables.py | 6 +- backend/app/scheduler/loop.py | 32 +- backend/app/scheduler/service.py | 33 ++- backend/tests/test_eval_snapshots.py | 2 +- backend/tests/test_introspect_jobs.py | 19 +- backend/tests/test_job_runtime.py | 16 +- backend/tests/test_run_append.py | 12 +- backend/tests/test_source_upkeep.py | 5 +- frontend/src/lib/termsConcurrency.ts | 29 ++ 74 files changed, 3521 insertions(+), 447 deletions(-) create mode 100644 backend/app/cluster/__init__.py create mode 100644 backend/app/cluster/caches.py create mode 100644 backend/app/cluster/claims.py create mode 100644 backend/app/cluster/counters.py create mode 100644 backend/app/cluster/guard.py create mode 100644 backend/app/cluster/identity.py create mode 100644 backend/app/cluster/leader.py create mode 100644 backend/app/cluster/mutex.py create mode 100644 backend/app/cluster/reaper.py create mode 100644 backend/app/cluster/relay.py create mode 100644 backend/app/cluster/requests.py create mode 100644 backend/app/cluster/service.py create mode 100644 backend/app/cluster/signal.py create mode 100644 backend/app/cluster/slots.py create mode 100644 backend/app/cluster/state.py create mode 100644 backend/app/cluster/workers.py create mode 100644 backend/app/db/models_cluster.py create mode 100644 frontend/src/lib/termsConcurrency.ts diff --git a/backend/app/api/access_keys.py b/backend/app/api/access_keys.py index ba50d365..23b374f8 100644 --- a/backend/app/api/access_keys.py +++ b/backend/app/api/access_keys.py @@ -131,6 +131,9 @@ async def put_settings(payload: SettingsIn, session: AsyncSession = Depends(get_ data={"before": bool(before.get(PERSONAL_KEYS)), "after": payload.personal_keys}) await session.commit() remember_settings(value) + from app.cluster import caches + + await caches.broadcast("auth_settings") return await get_settings() diff --git a/backend/app/api/apps.py b/backend/app/api/apps.py index 38f99a3a..5dd1fcea 100644 --- a/backend/app/api/apps.py +++ b/backend/app/api/apps.py @@ -552,6 +552,9 @@ async def publish_draft(session: AsyncSession, app: App, *, note: str, dispositi await session.rollback() raise CodedHTTPException(409, "有人刚刚发布了这个应用,请刷新后再试", PUBLISH_CONFLICT) from e clear_cache() + from app.cluster import caches + + await caches.broadcast("app_pages") return {"ok": True, "version": number, "level": level, "spec_hash": row.spec_hash, "issues": issues} @@ -1045,4 +1048,7 @@ async def backfill(app_id: str, view_id: str, session: AsyncSession = Depends(ge except LaunchRefused as e: raise _view_refused(e) from e clear_cache() + from app.cluster import caches + + await caches.broadcast("app_pages") return out diff --git a/backend/app/api/auth.py b/backend/app/api/auth.py index 45fd4d20..73b32785 100644 --- a/backend/app/api/auth.py +++ b/backend/app/api/auth.py @@ -94,13 +94,13 @@ async def login(payload: LoginIn, request: Request, response: Response, if mode == "off": raise HTTPException(409, "未开启登录,无需登录") keys = (f"u:{payload.username.strip().lower()}", f"ip:{_client_ip(request)}") - if wait := sessions.wait_seconds(*keys): + if wait := await sessions.wait_for(*keys): raise HTTPException(429, f"登录失败次数过多,请 {wait} 秒后再试") user = (await session.execute(select(User).where(User.username == payload.username.strip()))).scalar_one_or_none() if user is None or user.disabled or not verify_password(payload.password, user.password_hash): - sessions.failed(*keys) + await sessions.note_failed(*keys) raise HTTPException(401, "用户名或密码不正确") - sessions.succeeded(*keys) + await sessions.note_succeeded(*keys) token = await sessions.create_session(session, user, ip=_client_ip(request), user_agent=request.headers.get("user-agent")) await session.commit() @@ -317,6 +317,10 @@ async def put_settings(payload: SettingsIn, request: Request, response: Response user_agent=request.headers.get("user-agent")) await session.commit() remember_settings(value) + # 多进程:别的进程的登录设置缓存也要换(开没开登录!),见 app/cluster/caches.py + from app.cluster import caches + + await caches.broadcast("auth_settings") out = _settings_out(value) if token is not None and created is not None: _login_response(request, response, token) diff --git a/backend/app/api/evidence.py b/backend/app/api/evidence.py index fb232c6e..8acb580d 100644 --- a/backend/app/api/evidence.py +++ b/backend/app/api/evidence.py @@ -4190,7 +4190,7 @@ async def _append_judged(sealed: _Sealed, report: _Report, units: list[str], 核对和写入是同一个事务里的条件更新,和接着跑占住运行(runner._claim)抢的是同一行:谁先提交算谁的。 """ from app.core.bus import bus - from app.engine.runner import _stage, run_manager + from app.engine.runner import _seq_floor, _stage, run_manager same_seal = (Run.manifest_seq.is_(None) if sealed.run_manifest_seq is None else Run.manifest_seq == sealed.run_manifest_seq, @@ -4200,8 +4200,9 @@ async def _append_judged(sealed: _Sealed, report: _Report, units: list[str], run = await session.get(Run, sealed.run_id) if run is None: return None - # 进程重启过的话内存里的序号从 0 起:接在库里最后一条后面,不和已有的撞号 - bus.set_seq(sealed.run_id, run.last_seq or 0) + # 接在库里最后一条后面(进程重启过、或者别的进程追加过的话,内存里的序号对不上) + last = run.last_seq + bus.set_seq(sealed.run_id, await _seq_floor(session, sealed.run_id, last)) event = run_manager._event( sealed.run_id, EventType.EVIDENCE_JUDGED, node_id=report.node_id, data={ "report": report.node_id, "doc_artifact": report.doc_artifact, "units": units, @@ -4212,8 +4213,11 @@ async def _append_judged(sealed: _Sealed, report: _Report, units: list[str], "calls": outcome["calls"], "duration_ms": outcome["duration_ms"], "budget": outcome["budget"], "limits_hit": outcome["limits_hit"], "unjudged": outcome["unjudged"], "gaps": outcome["gaps"], "notes": outcome["notes"], "skipped": outcome["skipped"], "post_seal": True}) + # last_seq 也要还是读到的那个:多进程时别的进程可能同时往这次运行里追加(另一次裁判、批注),各拿着同一个号就撞了; + # 同 engine/run_append.append_events,谁先提交算谁的,后到的这一次不记(判定照样交回给点的人) claimed = await session.execute( - update(Run).where(Run.id == sealed.run_id, Run.status.in_(_TERMINAL), *same_seal) + update(Run).where(Run.id == sealed.run_id, Run.status.in_(_TERMINAL), *same_seal, + Run.last_seq.is_(None) if last is None else Run.last_seq == last) .values(last_seq=event.seq)) if claimed.rowcount != 1: # 运行已经不是那份封存下的终态了。烧掉一个序号不要紧:不落库的事件(令牌流)本来就占序号, diff --git a/backend/app/api/knowledge.py b/backend/app/api/knowledge.py index a9dac9b2..2288f1e6 100644 --- a/backend/app/api/knowledge.py +++ b/backend/app/api/knowledge.py @@ -404,6 +404,10 @@ async def set_embedding( else: session.add(Setting(key=emb.EMBEDDING_SETTING_KEY, value=value)) await session.commit() + # 多进程:别的进程照新的设置换向量模型(不换的话它们按旧模型检索、写旧模型的向量) + from app.cluster import caches + + await caches.broadcast("embeddings") return { "embedder": emb.embedder_id(), "dim": emb.embedder_dim(), diff --git a/backend/app/api/runs.py b/backend/app/api/runs.py index bd87fc14..d1b79a1a 100644 --- a/backend/app/api/runs.py +++ b/backend/app/api/runs.py @@ -637,18 +637,22 @@ async def stream_run(websocket: WebSocket, run_id: str) -> None: await _end(websocket, run.status if run else None) return - # 占位期间攒下的那几条历史里也有,按 seq 滤掉,否则前端同一步收两遍 - pump = asyncio.create_task(_pump(websocket, subscription.stream(after=last_seq), run_id)) - try: - # 客户端断开时读操作会抛异常,用它来结束这条连接 - while True: - await websocket.receive_text() - except WebSocketDisconnect: - pass - finally: - pump.cancel() - with contextlib.suppress(asyncio.CancelledError): - await pump + # 占位期间攒下的那几条历史里也有,按 seq 滤掉,否则前端同一步收两遍。多进程时这次运行可能在别的进程里执行: + # 跟随它(app/cluster/relay.py),本进程从库里按 seq 追、在本进程的总线上发布,这里收到的和在执行的进程上一样 + from app.cluster.relay import relay + + with relay.follow_run(run_id, last_seq): + pump = asyncio.create_task(_pump(websocket, subscription.stream(after=last_seq), run_id)) + try: + # 客户端断开时读操作会抛异常,用它来结束这条连接 + while True: + await websocket.receive_text() + except WebSocketDisconnect: + pass + finally: + pump.cancel() + with contextlib.suppress(asyncio.CancelledError): + await pump except WebSocketDisconnect: return except Exception: # noqa: BLE001 diff --git a/backend/app/api/schedules.py b/backend/app/api/schedules.py index f924e9db..ff6688cb 100644 --- a/backend/app/api/schedules.py +++ b/backend/app/api/schedules.py @@ -472,7 +472,7 @@ async def _fire_outs(session: AsyncSession, fires: list[ScheduleFire]) -> list[d for d in rows: notified.setdefault(d.fire_id, []).append({"event": d.event, "status": d.status, "at": _iso(d.created_at)}) # 正在发起的触发:库里是占位的「未能发起」(为了进程被强杀时照实说),进程还在、只是还没发起完,按已发起给 - launching = {f.id for f in fires if service.launching(f.id)} + launching = {f.id for f in fires if service.launching(f.id, f)} return [{ "id": f.id, "fire_at": _iso(f.fire_at), diff --git a/backend/app/api/settings.py b/backend/app/api/settings.py index c9b32957..46c7ee2d 100644 --- a/backend/app/api/settings.py +++ b/backend/app/api/settings.py @@ -900,6 +900,16 @@ def _mail_public(stored: Any) -> dict[str, Any]: return mail.public_value(stored) +#: 运行并发一组(app/cluster/slots.py) +CONCURRENCY_GROUP = "concurrency" + + +def concurrency_values(stored: Any) -> dict[str, Any]: + from app.cluster import slots + + return slots.values(stored) + + def _internal(key: str) -> bool: """通知自己记的账(每天汇总的认领行、自动建过邮件渠道的人,notify/digest.STATE_PREFIX):不是用户设置,不列、不收。""" from app.notify.digest import STATE_PREFIX @@ -945,6 +955,8 @@ DEFAULT_SETTINGS: dict[str, Any] = { EVAL_GROUP: eval_values(None), # 邮件服务(SMTP):通知渠道的邮件格式用它发信。缺省关 MAIL_GROUP: _mail_public(None), + # 运行并发(app/cluster/slots.py):全站同时运行的上限,交互、应用、定时、系统四类各自的上限和优先级。多进程时几个进程合起来算 + CONCURRENCY_GROUP: concurrency_values(None), } @@ -1011,6 +1023,7 @@ async def get_settings(session: AsyncSession = Depends(get_session)) -> dict[str merged[APPS_GROUP] = apps_values(stored.get(APPS_GROUP)) merged[LEDGER_GROUP] = ledger_values(stored.get(LEDGER_GROUP)) merged[EVAL_GROUP] = eval_values(stored.get(EVAL_GROUP)) + merged[CONCURRENCY_GROUP] = concurrency_values(stored.get(CONCURRENCY_GROUP)) # 口令不回(password 恒为 null),只说配没配(password_set) merged[MAIL_GROUP] = _mail_public(stored.get(MAIL_GROUP)) for key, value in stored.items(): @@ -1122,6 +1135,14 @@ async def put_settings( if problem := eval_problem(payload.values[EVAL_GROUP]): raise HTTPException(422, problem) payload.values[EVAL_GROUP] = eval_values(payload.values[EVAL_GROUP]) + if CONCURRENCY_GROUP in payload.values: + from app.cluster import slots + + if problem := slots.problem(payload.values[CONCURRENCY_GROUP]): + raise HTTPException(422, problem) + saved_row = await session.get(Setting, CONCURRENCY_GROUP) + payload.values[CONCURRENCY_GROUP] = slots.merged(saved_row.value if saved_row is not None else None, + payload.values[CONCURRENCY_GROUP]) if MAIL_GROUP in payload.values: from app.notify import mail @@ -1144,9 +1165,17 @@ async def put_settings( else: session.add(Setting(key=key, value=value)) await session.commit() + if CONCURRENCY_GROUP in payload.values: + # 本进程马上按新的发名额(别的进程最迟几秒,slots.POLICY_TTL_S) + from app.cluster import slots + + slots.forget_policy() if NOTIFY_GROUP in payload.values: # 中间件、单点登录的回调按进程缓存取对外访问地址:写完库就刷新,不用重启 public_url.remember(payload.values[NOTIFY_GROUP][public_url.SETTING_KEY]) + from app.cluster import caches + + await caches.broadcast("public_url") return await get_settings(session) @@ -1226,4 +1255,14 @@ async def system_info() -> dict[str, Any]: "max_run_seconds": app_settings.max_run_seconds, "model_timeout_seconds": app_settings.model_timeout_seconds, }, + # 多进程协调(app/cluster):登记着的工作进程、谁在调度当班、各类运行在跑 / 在等几个 + "cluster": await _cluster_info(), } + + +async def _cluster_info() -> dict[str, Any]: + from app.cluster import identity, leader, slots, workers + + return {"me": identity.owner(), "database": app_settings.db_dialect, "workers": await workers.listing(), + "leader": await leader.current(), "runs": await slots.snapshot(), + "lease_s": workers.lease_s(), "heartbeat_s": workers.heartbeat_s()} diff --git a/backend/app/api/source_changes.py b/backend/app/api/source_changes.py index 904fc00b..e899b1e2 100644 --- a/backend/app/api/source_changes.py +++ b/backend/app/api/source_changes.py @@ -310,7 +310,7 @@ async def background_jobs(source_id: str, session: AsyncSession = Depends(get_se brief = await module.job_brief(session, job) if job is not None else None if job is None or brief is None: continue - ahead = runtime.lanes.ahead(row.id, job.id) + ahead = await runtime.lanes.ahead(row.id, job.id) position = len(ahead) if ahead is not None else None found.append((position, job.created_at, { "kind": kind, "id": job.id, "status": brief["status"], "total": brief.get("total", 0), diff --git a/backend/app/apps/backfill.py b/backend/app/apps/backfill.py index 97630d6e..eab353a8 100644 --- a/backend/app/apps/backfill.py +++ b/backend/app/apps/backfill.py @@ -219,8 +219,11 @@ async def run_backfill(session: AsyncSession, app: Any, version: AppVersion, vie _LOCKS: dict[tuple[str, str], asyncio.Lock] = {} -def _lock(app_id: str, view_id: str) -> asyncio.Lock: - return _LOCKS.setdefault((app_id, view_id), asyncio.Lock()) +def _lock(app_id: str, view_id: str) -> Any: + """跨进程的锁(app/cluster/mutex):同一个视图的补算多进程时也只跑一份。""" + from app.cluster import mutex + + return mutex.lock(f"app-backfill:{app_id}:{view_id}") async def _run_backfill(session: AsyncSession, app: Any, version: AppVersion, view_id: str, *, principal: Any, diff --git a/backend/app/apps/resolve.py b/backend/app/apps/resolve.py index cd260aa5..0ce0bfe9 100644 --- a/backend/app/apps/resolve.py +++ b/backend/app/apps/resolve.py @@ -387,10 +387,11 @@ def _records_gap(entry: Any) -> str | None: _LOCKS: dict[str, asyncio.Lock] = {} -def _lock(key: str) -> asyncio.Lock: - if key not in _LOCKS: - _LOCKS[key] = asyncio.Lock() - return _LOCKS[key] +def _lock(key: str) -> Any: + """按需视图的单飞锁(app/cluster/mutex,多进程之间也是一把):同一个键正在运行的那次接上,不另发。""" + from app.cluster import mutex + + return mutex.lock(f"app-view:{key}") def normalize_params(form: dict[str, Any] | None, params: Any) -> dict[str, Any]: diff --git a/backend/app/auth/keys.py b/backend/app/auth/keys.py index 5f228c6c..0739e135 100644 --- a/backend/app/auth/keys.py +++ b/backend/app/auth/keys.py @@ -255,6 +255,16 @@ async def introspect(session: Any, principal: Any) -> dict[str, Any]: _BUCKETS: dict[str, tuple[float, float]] = {} +async def take_shared(key_id: str) -> int: + """同 take,多进程(PostgreSQL)时几个进程共用一个桶(记在库里,app/cluster/counters.py):不然每个进程各有一个满桶, + 实际的上限是进程数乘每分钟 RATE_PER_MINUTE 次。SQLite 只有一个进程,照旧用进程里的桶。""" + from app.cluster import counters + + if counters.shared(): + return await counters.take_token(f"key:{key_id}", per_minute=RATE_PER_MINUTE) + return take(key_id) + + def take(key_id: str, *, now: float | None = None) -> int: """这把密钥再发一个请求:放行交回 0,超了交回还要等几秒(Retry-After)。令牌桶:满桶 RATE_PER_MINUTE 个,每秒补 RATE_PER_MINUTE / 60 个。""" diff --git a/backend/app/auth/middleware.py b/backend/app/auth/middleware.py index 40152483..9e90f469 100644 --- a/backend/app/auth/middleware.py +++ b/backend/app/auth/middleware.py @@ -197,7 +197,7 @@ class AuthMiddleware: keys.note(principal.key_id, method=method, route=path, status=principal.status, run_id=None, ip=ip) await _reply(send, principal.status, principal.detail, principal.code) return - if wait := keys.take(principal.key_id): + if wait := await keys.take_shared(principal.key_id): keys.note(principal.key_id, method=method, route=path, status=429, run_id=None, ip=ip) await _reply(send, 429, f"这把访问密钥的请求太频繁(每分钟最多 {keys.RATE_PER_MINUTE} 次),请 {wait} 秒后再试", keys.KEY_RATE_LIMITED, [(b"retry-after", str(wait).encode())]) diff --git a/backend/app/auth/sessions.py b/backend/app/auth/sessions.py index 2951366e..fe637d04 100644 --- a/backend/app/auth/sessions.py +++ b/backend/app/auth/sessions.py @@ -129,3 +129,30 @@ def failed(*keys: str) -> None: def succeeded(*keys: str) -> None: for key in keys: _failures.pop(key, None) + + +# 多进程(PostgreSQL)时登录失败记在库里(app/cluster/counters.py),几个进程看到的是同一份:不然把尝试分散到几个进程上, +# 退避就弱了几倍。SQLite 只有一个进程,照旧用上面进程里的那份 + +async def wait_for(*keys: str) -> int: + from app.cluster import counters + + return await counters.wait_seconds(*keys) if counters.shared() else wait_seconds(*keys) + + +async def note_failed(*keys: str) -> None: + from app.cluster import counters + + if counters.shared(): + await counters.failed(*keys, free_tries=_FREE_TRIES, max_wait_s=_MAX_WAIT_S) + else: + failed(*keys) + + +async def note_succeeded(*keys: str) -> None: + from app.cluster import counters + + if counters.shared(): + await counters.succeeded(*keys) + else: + succeeded(*keys) diff --git a/backend/app/blueprints/personal.py b/backend/app/blueprints/personal.py index 770d5864..4eee74b2 100644 --- a/backend/app/blueprints/personal.py +++ b/backend/app/blueprints/personal.py @@ -342,8 +342,11 @@ async def event(session: AsyncSession, blueprint: Any, op: str, principal: Any, _LOCKS: dict[str, asyncio.Lock] = {} -def _lock(who: str | None) -> asyncio.Lock: - return _LOCKS.setdefault(who or "", asyncio.Lock()) +def _lock(who: str | None) -> Any: + """跨进程的锁(app/cluster/mutex):查上限和发起在一起,多进程时同一个人连点两下也不会多跑一次。""" + from app.cluster import mutex + + return mutex.lock(f"personal-refresh:{who or ''}") def _today_start() -> datetime: diff --git a/backend/app/blueprints/publish_plan.py b/backend/app/blueprints/publish_plan.py index 80438a16..bc5a90db 100644 --- a/backend/app/blueprints/publish_plan.py +++ b/backend/app/blueprints/publish_plan.py @@ -629,8 +629,12 @@ async def _start(session: AsyncSession, plan_id: str, *, principal: Principal, w # 前面建好的东西不见了:接着做会把后面几步接到一个对不上的状态上,只能重新出单(新单按现在的样子重新算) raise PlanRefused(409, "这份发布单前面建好的东西不见了,不能接着做:请重新出一份发布单", OBJECT_MISSING) # 条件更新抢到执行权:两个人同时点「确认发布」,只有先到的那个做 + # 属主记成本进程(多进程:别的进程启动时只收拾属主不在了的那些,见 startup) + from app.db.base import current_owner + claimed = await session.execute(update(PublishPlan).where(PublishPlan.id == row.id, PublishPlan.status == want) - .values(status="running").execution_options(synchronize_session=False)) + .values(status="running", owner=current_owner()) + .execution_options(synchronize_session=False)) if claimed.rowcount != 1: await session.rollback() raise PlanRefused(409, "这份发布单刚刚被别人开始执行了,请刷新后查看进度", PLAN_STATE) @@ -1010,15 +1014,24 @@ _STEPS = { # -------------------------------------------------------------------------- -async def startup() -> int: +async def startup(*, live: Any = None) -> int: """上次进程没了时还在执行的发布单(status=running,执行是在请求里做的,进程一没就再没有人接着推):改成停下,停在第一个 还没做完的步骤上、写明执行中断,这样就能「接着做」。那一步可能已经做了一半:接着做时按图哈希复用版本、按应用 id 更新 - 同一个应用,不会另建一份工作流版本或应用。交回整理了几份。""" + 同一个应用,不会另建一份工作流版本或应用。交回整理了几份。 + + 只收拾属主不在了的(多进程:别的活着的进程正在执行的不碰;live 见 app/cluster/workers.Liveness,不给就当场查库)。""" from app.apps.audit import record + from app.cluster import claims, state from app.db.base import SessionLocal + live = live if live is not None else await state.current(fresh=True) + claimed = [plan_id for plan_id, _ in await claims.orphans(PublishPlan, live, running=lambda _id: False, + statuses=("running",))] + if not claimed: + return 0 async with SessionLocal() as session: - rows = list((await session.execute(select(PublishPlan).where(PublishPlan.status == "running"))).scalars()) + rows = list((await session.execute(select(PublishPlan).where(PublishPlan.id.in_(claimed), + PublishPlan.status == "running"))).scalars()) for row in rows: kind = next((st.get("kind") for st in row.steps or [] if st.get("status") not in ("done", "skipped")), None) if kind: diff --git a/backend/app/capabilities/registry.py b/backend/app/capabilities/registry.py index 77a49d13..7b7068f1 100644 --- a/backend/app/capabilities/registry.py +++ b/backend/app/capabilities/registry.py @@ -184,7 +184,7 @@ def _platform_defaults_apply(row: Any, name: str) -> bool: return name in GOVERNANCE_DEFAULTS and row.version == 1 and row.updated_by is None -async def refresh_cache(session: AsyncSession) -> None: +async def refresh_cache(session: AsyncSession, *, broadcast: bool = True) -> None: """重算登记的进程内快照。能作证的端点读取:绑定登记端点的 http 自定义工具,治理后效果为 read,声明了 tabular。 Python 自定义工具(用户代码在算)和 MCP(没有结构化回执)不算,不管登记里写了什么。""" from app.capabilities.endpoints import bound_endpoint_ref @@ -200,6 +200,11 @@ async def refresh_cache(session: AsyncSession) -> None: _endpoint_prod.clear() _endpoint_prod.update({e.id: bool((e.environments or {}).get("prod")) for e in (await session.execute(select(Endpoint))).scalars()}) + if broadcast: + # 多进程:别的进程的这份快照也要重算(app/cluster/caches.py) + from app.cluster import caches + + await caches.broadcast("capabilities") def cached_view(name: str) -> dict[str, Any] | None: diff --git a/backend/app/chat/job.py b/backend/app/chat/job.py index f93b59ab..b29b3a8c 100644 --- a/backend/app/chat/job.py +++ b/backend/app/chat/job.py @@ -55,7 +55,7 @@ from app.chat import log as turn_log from app.chat import record from app.chat.drive import TurnDriver from app.core import job_runtime as runtime -from app.db.base import SessionLocal, utcnow +from app.db.base import SessionLocal, current_owner, utcnow logger = logging.getLogger(__name__) @@ -431,9 +431,10 @@ class ChatTurnExecution(runtime.JobRun): fields, meta, outcome, output, usage = await _settled(driver, self) await self.save(meta=meta, usage=usage, **fields) await self._end(outcome, output) - # 晚到的复核先登记、再改任务状态:实时流看到任务结束时,要知道还得等这一次复核回来 + # 晚到的复核先登记、再改任务状态:实时流看到任务结束时,要知道还得等这一次复核回来(库里也记一笔:别的进程上的流) if driver.late_review is not None: _track_late(self.turn_id, driver.late_review) + await _mark_late(self.turn_id, True) await self._job(status="succeeded", stage=None, finished_at=utcnow()) async def _end(self, outcome: str, output: Any = None) -> None: @@ -499,6 +500,11 @@ class ChatTurnManager(runtime.JobManager): def __init__(self) -> None: super().__init__(ChatTurnExecution, label="问数据的一轮", name="chat-turn") + def model(self) -> Any: + from app.db.models_chat import ChatTurnJob + + return ChatTurnJob + def start(self, job_id: str) -> bool: # type: ignore[override] """同一轮在这个进程里还有一个后台任务活着就不起第二个、也不把登记换成新的(换了的话,停止只停得到后起的那个, 先起的那个跑完把「已取消」盖成「已完成」)。起了交回 True。""" @@ -543,13 +549,46 @@ def _track_late(turn_id: str, task: asyncio.Task[Any]) -> None: watcher = asyncio.ensure_future(finish()) _late[turn_id] = watcher + watcher.add_done_callback(lambda _t: _later(_mark_late(turn_id, False))) + + +async def _mark_late(turn_id: str, pending: bool) -> None: + """晚到的复核记在任务行上(late_review_at):多进程时别的进程上的流也知道还要等它(stream_status)。""" + from app.db.models_chat import ChatTurnJob + + try: + async with SessionLocal() as session: + await session.execute(update(ChatTurnJob).where(ChatTurnJob.id == turn_id).values( + late_review_at=utcnow() if pending else None).execution_options(synchronize_session=False)) + await session.commit() + except Exception: # noqa: BLE001 - 记不上只是别的进程上的流可能早一点结束 + logger.exception("问数据:记晚到的复核状态出错") + if not pending: + turn_log.notify(turn_id) + + +def _later(coro: Any) -> None: + with contextlib.suppress(RuntimeError): + asyncio.ensure_future(coro) def late_pending(turn_id: str) -> bool: + """本进程里晚到的复核还在不在跑。""" task = _late.get(turn_id) return task is not None and not task.done() +async def late_waiting(job: Any) -> bool: + """晚到的复核还要不要等:本进程里在跑,或者库里记着、记它的进程(这一轮的属主)还活着。""" + if late_pending(job.id): + return True + if getattr(job, "late_review_at", None) is None: + return False + from app.cluster import state + + return (await state.live_for([job.owner])).alive(job.owner) + + # -------------------------------------------------------------------------- # 写那一行 # -------------------------------------------------------------------------- @@ -995,7 +1034,7 @@ async def _claim(session: AsyncSession, *, conversation: Any, question: str, dat row.meta = {k: v for k, v in new_meta.items() if v is not None} row.status, row.error, row.answer, row.review = "running", "", "", {} row.run_id = None - values = {"status": "queued", "stage": None, "mode": mode, "request": request, + values = {"status": "queued", "stage": None, "mode": mode, "request": request, "owner": current_owner(), "run_id": run_id if mode in ("continue", "resume") else None, "attempt": 0, "principal": principal_snapshot(), "actor": actor, "error": None, "started_at": None, "finished_at": None} @@ -1083,13 +1122,14 @@ async def state(session: AsyncSession, turn_id: str) -> dict[str, Any] | None: from app.db.models_chat import ChatTurnJob job = await session.get(ChatTurnJob, turn_id) - return None if job is None else job_view(job) + return None if job is None else job_view(job, late=await late_waiting(job)) -def job_view(job: Any) -> dict[str, Any]: +def job_view(job: Any, *, late: bool | None = None) -> dict[str, Any]: return {"status": job.status, "stage": job.stage, "mode": job.mode, "run_id": job.run_id, "attempt": job.attempt, "last_seq": job.last_seq, "attempt_seq": job.attempt_seq, "error": job.error, - "late_review": late_pending(job.id)} + "late_review": (late_pending(job.id) or getattr(job, "late_review_at", None) is not None) if late is None + else late} async def stream_status(turn_id: str) -> str | None: @@ -1100,7 +1140,7 @@ async def stream_status(turn_id: str) -> str | None: job = await session.get(ChatTurnJob, turn_id) if job is None: return "gone" - if job.status in runtime.ACTIVE or late_pending(turn_id): + if job.status in runtime.ACTIVE or await late_waiting(job): return None return job.status @@ -1131,8 +1171,10 @@ async def forget(session: AsyncSession, conversation_id: str) -> None: # -------------------------------------------------------------------------- -async def startup(*, resume: bool | None = None) -> dict[str, int]: - """服务启动时(run_manager.setup 之后:服务重启时在跑的运行这时已经标成挂起): +async def startup(*, resume: bool | None = None, live: Any = None) -> dict[str, int]: + """服务启动时(run_manager.setup 之后:服务重启时在跑的运行这时已经标成挂起);多进程时当班进程隔一会儿再来一次 + (app/cluster/reaper.py),只碰属主不在了的轮次(进程没了;或者就是本进程、却不在本进程里跑),别的活着的进程在跑的一概 + 不碰。认领那一下是带条件的 UPDATE,几个进程同时接手同一轮只有一个改得动;接手的进程照下面的规矩办: - 还在建图(没有运行)的这一轮标中断:那一行记 error 和原因(build_interrupted),可以重试本轮; - 已经发起了运行的:resume 为真(缺省看 AGENTLAB_SCHEDULER)就接着等那次运行、照常复核收尾;为假就停着(任务记 @@ -1145,11 +1187,17 @@ async def startup(*, resume: bool | None = None) -> dict[str, int]: from app.db.models import Conversation, ConversationTurn from app.db.models_chat import ChatTurnJob + from app.cluster import claims, state + resume = bool(settings.scheduler) if resume is None else resume counts = {"interrupted": 0, "resumed": 0, "parked": 0, "legacy": 0} + live = live if live is not None else await state.current(fresh=True) + claimed = [job_id for job_id, _ in await claims.orphans(ChatTurnJob, live, running=manager.running, + statuses=runtime.ACTIVE)] async with SessionLocal() as session: - jobs = [j for j in (await session.execute(select(ChatTurnJob).where( - ChatTurnJob.status.in_(runtime.ACTIVE)))).scalars() if not manager.running(j.id)] + jobs = list((await session.execute(select(ChatTurnJob).where(ChatTurnJob.id.in_(claimed), + ChatTurnJob.status.in_(runtime.ACTIVE)))).scalars()) \ + if claimed else [] legacy = list((await session.execute( select(ConversationTurn).join(Conversation, Conversation.id == ConversationTurn.conversation_id) .outerjoin(ChatTurnJob, ChatTurnJob.id == ConversationTurn.id) @@ -1174,6 +1222,18 @@ async def startup(*, resume: bool | None = None) -> dict[str, int]: await session.commit() turn_log.notify(job.id) counts["parked"] += 1 + # 晚到的复核记在库里、记它的进程却不在了:不会再回来,别让别的进程上的流一直等 + async with SessionLocal() as session: + late = (await session.execute(select(ChatTurnJob.id, ChatTurnJob.owner).where( + ChatTurnJob.late_review_at.is_not(None)))).all() + gone = [turn_id for turn_id, owner in late if not live.alive(owner, local=late_pending(turn_id))] + if gone: + async with SessionLocal() as session: + await session.execute(update(ChatTurnJob).where(ChatTurnJob.id.in_(gone)).values(late_review_at=None) + .execution_options(synchronize_session=False)) + await session.commit() + for turn_id in gone: + turn_log.notify(turn_id) for row in legacy: if turn_run_id(row): continue # 有运行的老轮次:页面打开时请服务端接着收尾 diff --git a/backend/app/chat/log.py b/backend/app/chat/log.py index 7a104110..b6366414 100644 --- a/backend/app/chat/log.py +++ b/backend/app/chat/log.py @@ -5,8 +5,8 @@ 也不用像运行的事件总线那样处理慢消费者的积压。订阅方在读库**之前**先登记等待(watch),读完没东西才睡:读和写之间新写进来的 那一条会把它叫醒,不会漏掉。 -叫醒只在本进程里(进程内的事件);隔一拍心跳(HEARTBEAT_S)订阅方也会回库里看一次,所以换成多进程(第 2 波)时,别的进程写的 -事件最迟一拍心跳就送到——那时再换成 PostgreSQL 的 LISTEN/NOTIFY,这里的接口不用变。 +叫醒在本进程里是进程内的事件;多进程(PostgreSQL)时另发 NOTIFY(app/cluster/relay.py),别的进程写的事件、改的状态马上叫醒这里 +的流;NOTIFY 丢了的话,隔一拍心跳(HEARTBEAT_S)订阅方也会回库里看一次。SQLite 只有一个进程,不额外轮询。 线上格式(SSE,每条一行 data: ):{seq, type, data, ts};回放完、这一轮也不在进行中了,补一条不带 seq 的 {type: stream.end, status}(任务的状态)再关;隔一拍没有新事件发一行注释心跳「: ping」(页面据此知道连接还活着)。 @@ -16,7 +16,8 @@ 以前每 3 秒落一条,一轮几百条。现在换了阶段的那一拍(规划 → 查看 → 搭建……,说的是到了哪一步)照落,刷新、重开时过程里那一行 阶段回放得出来;同一阶段接着的那些是纯保活,只记在本进程里(beat:每一轮最近的一拍),流上发一条不带 seq 的 {type: op, data}; 重连时流先补最近的那一拍。之后落了任何一条事件,那一拍就作废(心跳只说「上一条事件之后还在等」,重连时补旧的会让页面的进度 -退回去)。其余事件照落:第 2 波跨进程的实时流从库里按 seq 追,别的进程的保活心跳那时由流按任务状态自己合成。 +退回去)。其余事件照落:跨进程的实时流从库里按 seq 追;保活心跳不落库,执行这一轮的进程只在别的进程有流在看时把它转过去 +(app/cluster/relay.py 的 w / b 两种通知),丢了无妨——心跳本来就不进回放。 **清理。** 已结束(成功、失败、取消、中断)超过 RETAIN_DAYS 天的轮次整轮删事件日志(purge,挂在调度循环的那一拍里,一小时最多一次): 页面重开已结束的一轮读的是 conversation_turns 那一行(答案、图、复核、用量、几次尝试都在那里),只有还在进行中的轮次才回放 @@ -85,8 +86,12 @@ waiters = _Waiters() def notify(turn_id: str) -> None: - """这一轮有了变化(新事件、任务状态变了):叫醒本进程里等着的订阅方。""" + """这一轮有了变化(新事件、任务状态变了):叫醒本进程里等着的订阅方;多进程时另发 NOTIFY t|turn_id,别的进程上的流 + 也叫醒(app/cluster/relay.py;SQLite 只有一个进程,什么都不发)。""" waiters.notify(turn_id) + from app.cluster.signal import signals + + signals.publish("t", turn_id) #: 每一轮最近的一拍心跳:(第几拍, 操作)。只在本进程里,不落库(见模块说明) @@ -95,9 +100,19 @@ _beat_no = itertools.count(1) def beat(turn_id: str, op: dict[str, Any]) -> None: - """助手的一拍心跳:记下最近的这一拍、叫醒订阅方,由流发出去(不落库、不占 seq)。""" + """助手的一拍心跳:记下最近的这一拍、叫醒订阅方,由流发出去(不落库、不占 seq)。别的进程上有流在看这一轮时转过去 + (app/cluster/relay.py:不落库的东西只能这样跨进程)。""" _beats[turn_id] = (next(_beat_no), dict(op)) - notify(turn_id) + waiters.notify(turn_id) + from app.cluster.relay import relay + + relay.beat(turn_id, op) + + +def remote_beat(turn_id: str, op: dict[str, Any]) -> None: + """别的进程转过来的一拍心跳(这一轮在那边执行、这边有流在看):同本进程的一拍。""" + _beats[turn_id] = (next(_beat_no), dict(op)) + waiters.notify(turn_id) def latest_beat(turn_id: str) -> dict[str, Any] | None: @@ -169,6 +184,15 @@ async def stream(turn_id: str, after: int, *, active: Callable[[], Awaitable[str """SSE 的每一帧:先回放 seq > after 的,再推实时的;这一轮不在进行中、也没有新的了,补一条 stream.end 再结束。 active() 交回这一轮还在进行中时为 None,不在进行中时交回任务的状态(stream.end 里带的就是它)。""" + from app.cluster.relay import relay + + with relay.watch_turn(turn_id): + async for frame_ in _stream(turn_id, after, active=active, heartbeat=heartbeat): + yield frame_ + + +async def _stream(turn_id: str, after: int, *, active: Callable[[], Awaitable[str | None]], + heartbeat: float | None) -> AsyncIterator[str]: interval = HEARTBEAT_S if heartbeat is None else heartbeat last = after #: 这条连接发过的最近一拍助手心跳(_beats 的第几拍):新连上的从 0 起,先补最近的那一拍 diff --git a/backend/app/cluster/__init__.py b/backend/app/cluster/__init__.py new file mode 100644 index 00000000..78e4742c --- /dev/null +++ b/backend/app/cluster/__init__.py @@ -0,0 +1,14 @@ +"""多进程协调(docs/design/orchestration-round/00-总方案.md §5 W2-b):应用库是 PostgreSQL 时可以起几个工作进程(uvicorn +--workers N,或者几个容器指着同一个库、同一个数据目录),SQLite 只支持单进程。 + +- identity:这个进程是谁(OWNER);workers:登记、租约、谁还活着;state:手上的那一份「谁还活着」 +- leader:调度当班(全站只许一个进程做的周期活);service:心跳、起停 +- requests:发给别的进程的请求(停止、往运行里记事件);signal:PostgreSQL 的 LISTEN / NOTIFY 叫醒 +- slots:运行的并发名额(四类:交互、应用、定时、系统,各有全站上限和优先级) +- reaper:接手属主不在了的行;mutex:跨进程的互斥锁(PostgreSQL 的咨询锁);guard:SQLite 的单进程守卫 +- counters:按时间窗的计数(访问密钥限流、登录失败),PostgreSQL 记在库里、SQLite 在进程里 +""" +from app.cluster.identity import owner +from app.cluster.service import leading, on_beat, on_leadership, start, started, stop + +__all__ = ["leading", "on_beat", "on_leadership", "owner", "start", "started", "stop"] diff --git a/backend/app/cluster/caches.py b/backend/app/cluster/caches.py new file mode 100644 index 00000000..5f04de72 --- /dev/null +++ b/backend/app/cluster/caches.py @@ -0,0 +1,109 @@ +"""进程里缓存着的东西跨进程失效(docs/design/orchestration-round/00-总方案.md §5 W2-b「进程内的账和缓存」)。 + +有几样东西每个进程读进内存、改的时候只刷新改它的那个进程:登录设置(开没开登录!)、对外访问地址、向量模型、能力登记的 +治理快照、数据源的连接池、应用页面的渲染缓存。多进程时别的进程会一直用旧的——登录方式那一样是安全问题。 + +做法:改的进程照旧刷新自己,再 broadcast(名字, 键) 写一条 cache_events、发 NOTIFY cache;别的进程收到通知(或者隔一拍心跳, +NOTIFY 丢了的兜底)按 id 往后读,调登记过的处理(on)清掉、重读自己那一份。处理按名字登记在各自的模块里。SQLite 只有 +一个进程,broadcast 什么都不做(改的进程已经刷新过自己)。 +""" +from __future__ import annotations + +import asyncio +import logging +from collections.abc import Callable +from datetime import timedelta +from typing import Any + +from sqlalchemy import delete, func, select + +from app.cluster import identity +from app.cluster.signal import signals +from app.core.config import settings +from app.db.base import SessionLocal, utcnow + +logger = logging.getLogger(__name__) + +_handlers: dict[str, Callable[[str | None], Any]] = {} +_last_seen: int | None = None +_lock: asyncio.Lock | None = None + + +def on(name: str, handler: Callable[[str | None], Any]) -> None: + """登记名为 name 的缓存怎么失效:handler(键),键为 None 是整个清掉。可以是协程函数。""" + _handlers[name] = handler + + +def _enabled() -> bool: + from app.cluster.service import started + + return settings.db_dialect == "postgresql" and started() + + +async def broadcast(name: str, key: str | None = None) -> None: + """name(某个键)变了:告诉别的进程。改的进程自己要先刷新好自己那一份(这里不调本进程的处理)。""" + from app.db.models_cluster import CacheEvent + + if not _enabled(): + return + try: + async with SessionLocal() as session: + session.add(CacheEvent(name=name, key=key, owner=identity.owner())) + await session.commit() + except Exception: # noqa: BLE001 - 写不进去只是别的进程晚一点(它们的缓存各有自己的兜底) + logger.exception("通知别的进程缓存「%s」变了,没写进库", name) + return + signals.publish("cache", name) + + +async def start() -> None: + """起服务时:从现在的最后一条往后读(之前的变化,自己启动时读进来的已经是新的)。""" + global _last_seen + from app.db.models_cluster import CacheEvent + + signals.on("cache", lambda _parts: asyncio.ensure_future(poll())) + async with SessionLocal() as session: + _last_seen = int(await session.scalar(select(func.coalesce(func.max(CacheEvent.id), 0))) or 0) + + +async def poll() -> int: + """读别的进程写的、还没处理过的几条,照着失效。交回处理了几条。""" + global _last_seen, _lock + from app.db.models_cluster import CacheEvent + + if not _enabled() or _last_seen is None: + return 0 + if _lock is None: + _lock = asyncio.Lock() + async with _lock: + async with SessionLocal() as session: + rows = (await session.execute(select(CacheEvent).where(CacheEvent.id > _last_seen) + .order_by(CacheEvent.id).limit(500))).scalars().all() + done = 0 + for row in rows: + _last_seen = max(_last_seen, int(row.id)) + if row.owner == identity.owner(): + continue + handler = _handlers.get(row.name) + if handler is None: + continue + try: + out = handler(row.key) + if asyncio.iscoroutine(out): + await out + done += 1 + except Exception: # noqa: BLE001 + logger.exception("按别的进程的通知重读缓存「%s」出错", row.name) + return done + + +async def prune() -> int: + from app.db.models_cluster import CacheEvent + + async with SessionLocal() as session: + done = await session.execute(delete(CacheEvent).where(CacheEvent.created_at < utcnow() - timedelta(hours=1))) + await session.commit() + return int(done.rowcount or 0) + + +__all__ = ["broadcast", "on", "poll", "prune", "start"] diff --git a/backend/app/cluster/claims.py b/backend/app/cluster/claims.py new file mode 100644 index 00000000..6c75e746 --- /dev/null +++ b/backend/app/cluster/claims.py @@ -0,0 +1,45 @@ +"""认领属主不在了的行(cluster/reaper 的各种规矩共用):先看库里还没结束的那些行和它们的属主,属主不在了的逐行用带条件的 +UPDATE 改成自己(状态还是刚才看到的、属主还是刚才看到的那个)——几个进程同时接手,同一行只有一个改得动。""" +from __future__ import annotations + +from collections.abc import Callable, Iterable +from typing import Any + +from sqlalchemy import select, update + +from app.cluster import identity +from app.cluster.workers import Liveness +from app.db.base import SessionLocal + + +def same(column: Any, value: Any) -> Any: + """column 还是 value(None 用 IS NULL 比)。""" + return column.is_(None) if value is None else column == value + + +async def orphans(model: Any, live: Liveness, *, running: Callable[[str], bool], statuses: Iterable[str], + extra: Any = None) -> list[tuple[str, str]]: + """model 里状态在 statuses、属主不在了的行认领过来(owner 改成本进程),交回 [(id, 原来的状态)]。running:这一行在本进程里 + 是不是真的在跑。extra:另加的筛选条件。""" + wanted = tuple(statuses) + stmt = select(model.id, model.owner, model.status).where(model.status.in_(wanted)) + if extra is not None: + stmt = stmt.where(extra) + async with SessionLocal() as session: + rows = (await session.execute(stmt)).all() + me = identity.owner() + out: list[tuple[str, str]] = [] + for row_id, owner, status in rows: + if live.alive(owner, local=running(row_id)): + continue + async with SessionLocal() as session: + done = await session.execute(update(model).where(model.id == row_id, model.status == status, + same(model.owner, owner)) + .values(owner=me).execution_options(synchronize_session=False)) + await session.commit() + if done.rowcount == 1: + out.append((row_id, status)) + return out + + +__all__ = ["orphans", "same"] diff --git a/backend/app/cluster/counters.py b/backend/app/cluster/counters.py new file mode 100644 index 00000000..63025527 --- /dev/null +++ b/backend/app/cluster/counters.py @@ -0,0 +1,117 @@ +"""几个进程共用的计数:访问密钥每分钟限流的令牌桶(auth/keys)、登录失败的退避(auth/sessions)。 + +应用库是 PostgreSQL(可以多进程)时记在库里(rate_counters),一次请求一句带条件的 upsert:令牌桶「补多少、扣一个」在一句 +INSERT … ON CONFLICT DO UPDATE … WHERE 里算完,几个进程同时来也扣不过头;登录失败先把次数加一、占住那一行,再按次数算要等 +多久。SQLite 只有一个进程,计数照旧在进程里(调用方自己的那份),不为每个请求多写一次库——单进程时进程就是全站。 + +时间用墙钟(time.time):几台机器各自的单调时钟不能比,墙钟要对时(NTP),同租约。 +""" +from __future__ import annotations + +import math +import time +from datetime import datetime, timedelta, timezone + +from sqlalchemy import delete, select + +from app.cluster.service import started +from app.core.config import settings +from app.db.base import SessionLocal, utcnow + + +def shared() -> bool: + """计数要不要记在库里:PostgreSQL、而且起了协调(服务在跑)。测试、脚本里直接调用时照旧在进程里。""" + return settings.db_dialect == "postgresql" and started() + + +def _expiry(seconds: float) -> datetime: + return utcnow() + timedelta(seconds=seconds) + + +async def take_token(key: str, *, per_minute: int, now: float | None = None) -> int: + """令牌桶:满桶 per_minute 个,每秒补 per_minute / 60 个。放行交回 0,超了交回还要等几秒。""" + from sqlalchemy.dialects.postgresql import insert + + from app.db.models_cluster import RateCounter as Row + + rate = per_minute / 60.0 + cap = float(per_minute) + moment = time.time() if now is None else now + refill = func_least(cap, Row.value + (moment - Row.at) * rate) + stmt = insert(Row).values(key=key, count=1, value=cap - 1.0, at=moment, expires_at=_expiry(120)) \ + .on_conflict_do_update(index_elements=[Row.key], + set_={"value": refill - 1.0, "at": moment, "count": Row.count + 1, + "expires_at": _expiry(120)}, + where=refill >= 1.0) \ + .returning(Row.value) + async with SessionLocal() as session: + got = (await session.execute(stmt)).first() + if got is not None: + await session.commit() + return 0 + row = await session.get(Row, key) + await session.rollback() + tokens = min(cap, float(row.value) + (moment - float(row.at)) * rate) if row is not None else 0.0 + return max(1, math.ceil((1.0 - tokens) / rate)) + + +def func_least(a: float, b: object) -> object: + from sqlalchemy import func + + return func.least(a, b) + + +async def wait_seconds(*keys: str) -> int: + """登录失败退避:这几个键里还要等多久(秒),不用等为 0。""" + from app.db.models_cluster import RateCounter as Row + + if not keys: + return 0 + now = time.time() + async with SessionLocal() as session: + rows = (await session.execute(select(Row.value).where(Row.key.in_([f"login:{k}" for k in keys])))).scalars() + until = max((float(v) for v in rows), default=0.0) + return int(until - now) + 1 if until > now else 0 + + +async def failed(*keys: str, free_tries: int, max_wait_s: int) -> None: + """记一次登录失败:次数加一,满 free_tries 次之后要等 2^(次数-free_tries) 秒(最多 max_wait_s)。""" + from sqlalchemy import update + from sqlalchemy.dialects.postgresql import insert + + from app.db.models_cluster import RateCounter as Row + + now = time.time() + async with SessionLocal() as session: + for key in keys: + name = f"login:{key}" + count = (await session.execute( + insert(Row).values(key=name, count=1, value=0.0, at=now, expires_at=_expiry(86400)) + .on_conflict_do_update(index_elements=[Row.key], set_={"count": Row.count + 1, "at": now, + "expires_at": _expiry(86400)}) + .returning(Row.count))).scalar_one() + wait = 0 if count < free_tries else min(2 ** (count - free_tries), max_wait_s) + await session.execute(update(Row).where(Row.key == name).values(value=now + wait) + .execution_options(synchronize_session=False)) + await session.commit() + + +async def succeeded(*keys: str) -> None: + from app.db.models_cluster import RateCounter as Row + + async with SessionLocal() as session: + await session.execute(delete(Row).where(Row.key.in_([f"login:{k}" for k in keys]))) + await session.commit() + + +async def prune(now: datetime | None = None) -> int: + """到期的计数删掉(当班进程做)。""" + from app.db.models_cluster import RateCounter as Row + + async with SessionLocal() as session: + done = await session.execute(delete(Row).where(Row.expires_at < (now or datetime.now(timezone.utc)))) + await session.commit() + return int(done.rowcount or 0) + + +__all__ = ["failed", "prune", "shared", "succeeded", "take_token", "wait_seconds"] diff --git a/backend/app/cluster/guard.py b/backend/app/cluster/guard.py new file mode 100644 index 00000000..355b8a94 --- /dev/null +++ b/backend/app/cluster/guard.py @@ -0,0 +1,86 @@ +"""SQLite 只支持单进程:服务启动时拿数据目录上的文件锁(data/.agentlab.lock,flock 独占、不等),拿不到就拒绝启动。 + +**为什么拒绝而不是凑合。** SQLite 应用库下,运行的序号、并发名额、同源队、调度当班……这些在库里协调的东西照样能用,但 +上传表格的版本库、填报记录表、断点库都是数据目录里的 SQLite 文件,两个进程一起写同一个文件靠的是文件锁,换不成库里的锁; +uvicorn --workers 2 起来的第二个进程以前也只是悄悄地不写上传版本(503),别的照跑、各算各的,最难查。现在照实说:第二个进程 +起不来,报错写明「SQLite 只支持单进程;要多进程把应用库换成 PostgreSQL」。 + +进程退出(含强杀)时操作系统放掉文件锁,重启(dev.sh 的 --reload 先停旧的再起新的)不受影响。同一个进程里重复取(测试里 +一次次起停服务)是幂等的;数据目录换了就放掉旧的、对新目录重取。没有 fcntl 的平台(Windows)只记一条警告。 +""" +from __future__ import annotations + +import logging +import os +from pathlib import Path + +from app.core.config import settings + +logger = logging.getLogger(__name__) + +LOCK_FILE = ".agentlab.lock" + +SINGLE_PROCESS = ("应用库是 SQLite 时只支持单进程:数据目录 {dir} 已经被另一个 AgentLab 进程使用({hint})。" + "SQLite 不能多进程部署——不要给 uvicorn 加 --workers、不要让两个服务指着同一个数据目录;" + "要多进程,把应用库换成 PostgreSQL(AGENTLAB_DATABASE_URL,见 README「用 PostgreSQL」「多进程」)") + + +class SingleProcessOnly(RuntimeError): + """SQLite 下第二个进程启动:拒绝。""" + + +_held: tuple[Path, int] | None = None + + +def _path() -> Path: + return settings.data_dir / LOCK_FILE + + +def acquire() -> None: + """SQLite 时取数据目录上的独占锁,拿不到抛 SingleProcessOnly。PostgreSQL 时什么都不做(可以多进程)。""" + global _held + if settings.db_dialect != "sqlite": + return + if settings.workers > 1: + raise SingleProcessOnly(SINGLE_PROCESS.format(dir=settings.data_dir, hint=f"AGENTLAB_WORKERS={settings.workers}")) + path = _path() + if _held is not None and _held[0] == path: + return + release() + try: + import fcntl + except ImportError: + logger.warning("这个平台没有 fcntl,无法确认数据目录只被一个进程使用:SQLite 只支持单进程部署") + return + path.parent.mkdir(parents=True, exist_ok=True) + fd = os.open(path, os.O_RDWR | os.O_CREAT, 0o600) + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + os.close(fd) + holder = "" + try: + holder = path.read_text(encoding="utf-8").strip() + except OSError: + pass + raise SingleProcessOnly(SINGLE_PROCESS.format( + dir=settings.data_dir, hint=f"占着它的是 {holder}" if holder else "另一个进程占着它")) from None + try: + os.ftruncate(fd, 0) + os.write(fd, f"pid {os.getpid()}".encode()) + except OSError: + pass + _held = (path, fd) + + +def release() -> None: + global _held + if _held is not None: + try: + os.close(_held[1]) + except OSError: + pass + _held = None + + +__all__ = ["LOCK_FILE", "SINGLE_PROCESS", "SingleProcessOnly", "acquire", "release"] diff --git a/backend/app/cluster/identity.py b/backend/app/cluster/identity.py new file mode 100644 index 00000000..1a0d6695 --- /dev/null +++ b/backend/app/cluster/identity.py @@ -0,0 +1,48 @@ +"""这个工作进程是谁:主机 + pid + 启动随机串(OWNER)。 + +库里认领了的行(运行、问数据的一轮、后台任务、并发名额、同源队)都记着 owner;进程在 cluster_workers 里登记、续租约, +别的进程据此判断「这一行的属主还在不在」。随机串让同一个 pid 被复用(容器里 pid 常是 1)、或者同一个进程里测试一次次起停 +服务时,都分得清是哪一次启动认领的。 + +在导入时定下:建表的缺省值(db 模型的 owner 列)直接取它,不必等服务启动。fork 出来的子进程(uvicorn --workers)各自重新导入 +应用、各有各的;万一是 fork 之后没重新导入(multiprocessing 的 fork 方式),pid 对不上时 owner() 重新生成。 +""" +from __future__ import annotations + +import os +import secrets +import socket + +_HOST = (socket.gethostname() or "localhost")[:80] + + +def _make(pid: int) -> str: + return f"{_HOST}:{pid}:{secrets.token_hex(4)}" + + +_PID = os.getpid() +OWNER = _make(_PID) + + +def owner() -> str: + """这个进程的 owner id。""" + global OWNER, _PID + pid = os.getpid() + if pid != _PID: + _PID, OWNER = pid, _make(pid) + return OWNER + + +def host() -> str: + return _HOST + + +def describe(value: str | None) -> str: + """写进日志、界面的样子:主机和 pid(随机串不写)。""" + if not value: + return "(没有记录)" + parts = value.split(":") + return f"{parts[0]} 上的进程 {parts[1]}" if len(parts) >= 2 else value + + +__all__ = ["OWNER", "describe", "host", "owner"] diff --git a/backend/app/cluster/leader.py b/backend/app/cluster/leader.py new file mode 100644 index 00000000..1662bfdf --- /dev/null +++ b/backend/app/cluster/leader.py @@ -0,0 +1,109 @@ +"""调度当班:全站只许一个进程做的事(定时认领、事件派发、Webhook 投递、流程的时限和核实、各类周期清理、接手不在了的 +进程留下的活)由库里的一份租约定下谁做(cluster_leases 的 name=scheduler 那一行)。 + +一句条件 UPDATE 同时管「接班」和「续租」:这一行的 owner 是自己,或者租约已经过期(当班的进程没了)、没人当班,就改成自己、 +租约往后推。改到了一行就是当班。当班的进程平稳关停时交班(owner 清空),待命的下一拍心跳就接上;强杀的等租约过期。 + +两个进程短暂地都以为自己当班(续租那一拍库慢了、本地还没过期)时,重复不了:定时认领本来就是条件 UPDATE、触发记录 +(任务, 时刻, 种类) 唯一,事件派发按投递记录的唯一约束,待办的升级、核实各自也是条件更新。当班只是让它们平常不撞。 +本地另留一截余量(MARGIN):租约快到期还没续上时,自己先当作不当班了。 +""" +from __future__ import annotations + +import logging +from datetime import datetime, timedelta + +from sqlalchemy import case, or_, update + +from app.cluster import identity +from app.cluster.workers import lease_s +from app.db.base import SessionLocal, utcnow + +logger = logging.getLogger(__name__) + +SCHEDULER = "scheduler" +#: 本地判断「还在当班」时留的余量(秒) +MARGIN_S = 2.0 + + +class Leadership: + def __init__(self, name: str) -> None: + self.name = name + self.until: datetime | None = None + self.epoch = 0 + + def held(self, now: datetime | None = None) -> bool: + """本地看:还在当班(租约离到期还有余量)。""" + if self.until is None: + return False + return (now or utcnow()) + timedelta(seconds=MARGIN_S) < self.until + + async def claim(self, now: datetime | None = None) -> bool: + """接班或续租。交回这一拍之后是不是当班。""" + from app.db.models_cluster import ClusterLease + from app.db.portable import insert_ignore + + now = now or utcnow() + until = now + timedelta(seconds=lease_s()) + me = identity.owner() + mine = ClusterLease.owner == me + async with SessionLocal() as session: + done = await session.execute( + update(ClusterLease).where( + ClusterLease.name == self.name, + or_(mine, ClusterLease.owner.is_(None), ClusterLease.lease_until.is_(None), + ClusterLease.lease_until < now)) + .values(owner=me, lease_until=until, + acquired_at=case((mine, ClusterLease.acquired_at), else_=now), + epoch=case((mine, ClusterLease.epoch), else_=ClusterLease.epoch + 1)) + .returning(ClusterLease.epoch) + .execution_options(synchronize_session=False)) + epoch = done.scalar_one_or_none() + if epoch is None: + # 还没有这一行(新库):插一行,撞了说明别人刚插上,这一拍不当班 + inserted = await session.execute(insert_ignore(session, ClusterLease.__table__).values( + name=self.name, owner=me, lease_until=until, acquired_at=now, epoch=1) + .on_conflict_do_nothing(index_elements=["name"])) + epoch = 1 if inserted.rowcount == 1 else None + await session.commit() + was = self.held(now) + if epoch is None: + if was or self.until is not None: + logger.warning("调度当班交给了别的进程(本进程 %s 不再当班)", identity.describe(me)) + self.until = None + return False + if not was and self.epoch != epoch: + logger.info("本进程(%s)接手调度当班(第 %d 班)", identity.describe(me), epoch) + self.until, self.epoch = until, int(epoch) + return True + + async def release(self) -> None: + """交班(平稳关停时)。""" + from app.db.models_cluster import ClusterLease + + if self.until is None: + return + self.until = None + async with SessionLocal() as session: + await session.execute(update(ClusterLease).where( + ClusterLease.name == self.name, ClusterLease.owner == identity.owner()) + .values(owner=None, lease_until=None).execution_options(synchronize_session=False)) + await session.commit() + + +scheduler = Leadership(SCHEDULER) + + +async def current(name: str = SCHEDULER) -> dict[str, object] | None: + """库里记着谁当班(系统信息、验收脚本看)。""" + from app.db.models_cluster import ClusterLease + + async with SessionLocal() as session: + row = await session.get(ClusterLease, name) + if row is None: + return None + return {"owner": row.owner, "until": row.lease_until.isoformat() if row.lease_until else None, "epoch": row.epoch, + "me": row.owner == identity.owner()} + + +__all__ = ["Leadership", "MARGIN_S", "SCHEDULER", "current", "scheduler"] diff --git a/backend/app/cluster/mutex.py b/backend/app/cluster/mutex.py new file mode 100644 index 00000000..bf71a30a --- /dev/null +++ b/backend/app/cluster/mutex.py @@ -0,0 +1,111 @@ +"""跨进程的互斥锁:本进程里一把 asyncio 锁(按事件循环各一把),应用库是 PostgreSQL 时另取一把会话级咨询锁 +(pg_advisory_lock,按锁名的哈希取号),持锁期间占着一条专门的连接,放锁时在同一条连接上放。SQLite 只有一个进程, +本进程的锁就是全站的锁。 + +用在「先看库里有什么、再改文件」这种一个库事务包不住的地方:上传表格的版本存储(发布、回收、启动清理、固定版本, +data/table_versions.store_lock)、填报记录表的追加(records/store)。 + +用法同 asyncio.Lock:`async with lock:`。另留 acquire() / release() / _waiters / locked(),只碰本进程那一把(测试用来在 +本进程里卡住锁)。不可重入。 +""" +from __future__ import annotations + +import asyncio +import hashlib +import weakref +from typing import Any + +from app.core.config import settings + + +def advisory_key(name: str) -> int: + """锁名 → 咨询锁的号(有符号 64 位)。加前缀,不和别处直接写的号(db/base 的建表锁、断点库的建表锁)撞。""" + digest = hashlib.blake2b(f"agentlab:{name}".encode(), digest_size=8).digest() + return int.from_bytes(digest, "big", signed=True) + + +class ClusterLock: + def __init__(self, name: str) -> None: + self.name = name + self.key = advisory_key(name) + self._local = asyncio.Lock() + self._conn: Any = None + + # 只碰本进程那一把(同 asyncio.Lock) + async def acquire(self) -> bool: + return await self._local.acquire() + + def release(self) -> None: + self._local.release() + + def locked(self) -> bool: + return self._local.locked() + + @property + def _waiters(self) -> Any: + return self._local._waiters + + async def __aenter__(self) -> ClusterLock: + await self._local.acquire() + if settings.db_dialect != "postgresql": + return self + try: + from sqlalchemy import text + + from app.db.base import engine + + conn = await engine.connect() + try: + conn = await conn.execution_options(isolation_level="AUTOCOMMIT") + await conn.execute(text("SELECT pg_advisory_lock(:k)"), {"k": self.key}) + except BaseException: + await conn.close() + raise + self._conn = conn + except BaseException: + self._local.release() + raise + return self + + async def __aexit__(self, *exc: object) -> None: + conn, self._conn = self._conn, None + try: + if conn is not None: + from sqlalchemy import text + + try: + await asyncio.shield(conn.execute(text("SELECT pg_advisory_unlock(:k)"), {"k": self.key})) + finally: + # 放不掉(连接断了)也要关:会话一结束,库那边的咨询锁随之释放 + await asyncio.shield(conn.close()) + finally: + self._local.release() + + +class NamedLocks: + """按事件循环各一组的命名锁(测试每个用例一个循环:asyncio.Lock 不能换循环用)。""" + + def __init__(self) -> None: + self._by_loop: weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, dict[str, ClusterLock]] = \ + weakref.WeakKeyDictionary() + + def get(self, name: str) -> ClusterLock: + loop = asyncio.get_running_loop() + group = self._by_loop.get(loop) + if group is None: + group = self._by_loop[loop] = {} + lock = group.get(name) + if lock is None: + lock = group[name] = ClusterLock(name) + return lock + + +locks = NamedLocks() + + +def lock(name: str) -> ClusterLock: + """名为 name 的跨进程锁(本事件循环里同一个名字是同一把)。""" + return locks.get(name) + + +__all__ = ["ClusterLock", "NamedLocks", "advisory_key", "lock", "locks"] diff --git a/backend/app/cluster/reaper.py b/backend/app/cluster/reaper.py new file mode 100644 index 00000000..a48e6013 --- /dev/null +++ b/backend/app/cluster/reaper.py @@ -0,0 +1,56 @@ +"""接手属主不在了的行(进程没了、租约过期):各种东西各按各的规矩,在各自的模块里登记(register),这里按登记的先后 +一个个调。 + +谁调:每个进程启动时调一次(那时自己手上什么都没在跑,这一次也收拾自己上一次留下的——测试里同一个进程一次次起停服务), +之后当班的进程隔一会儿(或者收到别的进程走了的通知)再调。各规矩里认领那一下都是带条件的 UPDATE(状态没变、属主还是 +刚才看到的那个),几个进程同时接手同一行只有一个改得动;活着的进程认领的行一概不碰(Liveness.alive)。 + +登记的规矩(启动时的整理就是它们):运行(标中断 / 排队的标失败,不自动重跑)、问数据的一轮(建图中的标中断;发起了运行的 +接着等、照常复核收尾)、数据目录起草 / 剖析 / 探查、评测、影子起草(标中断)、文档处理(标失败)、发布单(改成停下)…… +见各模块的 reap / startup。 +""" +from __future__ import annotations + +import logging +from collections.abc import Awaitable, Callable +from typing import Any + +from app.cluster.workers import Liveness + +logger = logging.getLogger(__name__) + +Reaper = Callable[[Liveness], Awaitable[Any]] +_REAPERS: list[tuple[str, Reaper]] = [] + + +def register(name: str, fn: Reaper) -> Reaper: + """登记一种接手规矩(同名的换成新的:模块重新导入时不重复登记)。""" + for i, (known, _) in enumerate(_REAPERS): + if known == name: + _REAPERS[i] = (name, fn) + return fn + _REAPERS.append((name, fn)) + return fn + + +def names() -> list[str]: + return [name for name, _ in _REAPERS] + + +async def reap(live: Liveness) -> dict[str, Any]: + """按登记的先后调一遍,交回各自整理了多少。一种出错不影响后面的。""" + out: dict[str, Any] = {} + for name, fn in list(_REAPERS): + try: + got = await fn(live) + except Exception: # noqa: BLE001 + logger.exception("接手不在了的进程留下的「%s」出错,下一次再试", name) + continue + if got: + out[name] = got + if out: + logger.info("接手了不在了的进程留下的:%s", out) + return out + + +__all__ = ["Reaper", "names", "reap", "register"] diff --git a/backend/app/cluster/relay.py b/backend/app/cluster/relay.py new file mode 100644 index 00000000..c5d14ac5 --- /dev/null +++ b/backend/app/cluster/relay.py @@ -0,0 +1,332 @@ +"""实时流跨进程(docs/design/orchestration-round/00-总方案.md §5 W2-b「实时流跨进程」)。 + +**为什么。** 运行的事件先落库、再在本进程的事件总线上推给订阅方(core/bus);问数据一轮的事件同样先落库,再叫醒本进程里 +等着的流(chat/log)。多进程时,页面的 WebSocket / SSE 连到的进程不一定是执行这次运行、这一轮的那个进程。 + +**怎么做。** 订阅方在哪个进程,就由那个进程的转发器(Relay)从库里按 seq 追: +- 运行:WebSocket 接上之后登记跟随(follow_run)。执行它的进程每提交一条事件,发 NOTIFY r|run_id|seq(cluster/signal, + 攒一下批量发);跟随它的进程收到后,只要这次运行不在本进程里执行,就从库里读 seq 大于手上游标的事件、在本进程的总线上 + 发布一遍(relayed,不再往外转),订阅方收到的和执行它的进程上的一模一样,序号由执行的进程发、不重复。到了终态(或者 + 挂起、属主不在了被标中断)再关掉本进程的订阅方,同执行进程上的规矩。NOTIFY 丢了也不怕:跟随的运行隔 POLL_S 秒回库里看一次。 +- 流式增量(正文 token、思考 delta)不落库:跟随的进程隔几秒发一次「我在看」(w|r:run_id),执行的进程只给有人在看的运行把 + 增量攒一小段、放进载荷转过去(e|run_id|[事件…])。 +- 问数据一轮:流本来就是从库里读的,这里只管叫醒——追加一条事件、任务状态变了都发 t|turn_id,收到的进程叫醒本进程里等着 + 这一轮的流;助手的保活心跳(不落库,chat/log.beat)同样只给有人在看的那几轮转过去(b|turn_id|心跳)。 +- SQLite 只有一个进程:signal 不收不发,这里什么都不做(单进程照旧走进程内的叫醒,不额外轮询)。 + +WebSocket / SSE 的回放、重连、心跳的说法都不变:回放照旧读库,转发器只补「实时」那一段。 +""" +from __future__ import annotations + +import asyncio +import contextlib +import json +import logging +import time +from collections.abc import Iterator +from typing import Any + +from sqlalchemy import select + +from app.cluster.signal import signals +from app.core.events import EventType, RunEventModel +from app.db.base import SessionLocal + +logger = logging.getLogger(__name__) + +#: 在看的一方隔多久重发一次「我在看」;执行的一方收到后记多久 +WATCH_S = 5.0 +WATCH_TTL_S = 15.0 +#: 流式增量攒多久转一次 +EPHEMERAL_FLUSH_S = 0.05 +#: 跟随的运行隔多久回库里看一次(NOTIFY 丢了的兜底) +POLL_S = 2.0 +#: 一条载荷最多多少字节(PostgreSQL 的 NOTIFY 上限是 8000) +PAYLOAD_MAX = 7000 +#: 不落库的事件(同 engine/runner._EPHEMERAL) +EPHEMERAL = {EventType.LLM_TOKEN, EventType.LLM_THINKING_DELTA} +_TERMINAL = ("succeeded", "failed", "cancelled") + + +class Relay: + def __init__(self) -> None: + #: 本进程里在跟随的运行:{run_id: 跟随的连接数}、游标(已经在本进程发布过的最大 seq) + self._follow: dict[str, int] = {} + self._cursor: dict[str, int] = {} + self._pumping: dict[str, asyncio.Lock] = {} + #: 本进程里在看的问数据一轮 + self._turns: dict[str, int] = {} + #: 别的进程在看的(执行的一方记):{run_id / turn_id: 到期的时刻} + self._watched_runs: dict[str, float] = {} + self._watched_turns: dict[str, float] = {} + #: 攒着要转的流式增量:{run_id: [事件]} + self._buffer: dict[str, list[dict[str, Any]]] = {} + self._flusher: asyncio.Task[None] | None = None + self._loop_task: asyncio.Task[None] | None = None + self._installed = False + + # ---------------- 起停 ---------------- + + def install(self) -> None: + """登记收通知的处理、接上事件总线(起服务时,PostgreSQL 才有意义;重复调用无害)。""" + from app.core.bus import bus + + if not self._installed: + signals.on("r", self._on_run) + signals.on("e", self._on_ephemeral, raw=True) + signals.on("w", self._on_watch) + signals.on("t", self._on_turn) + signals.on("b", self._on_beat, raw=True) + signals.on_resync(self._resync) + self._installed = True + bus.tap = self._tap + + async def start(self) -> None: + self.install() + if signals.active and (self._loop_task is None or self._loop_task.done()): + self._loop_task = asyncio.create_task(self._loop(), name="agentlab-cluster-relay") + + async def stop(self) -> None: + from app.core.bus import bus + + if bus.tap == self._tap: + bus.tap = None + for task in (self._loop_task, self._flusher): + if task is not None: + task.cancel() + with contextlib.suppress(asyncio.CancelledError, Exception): + await task + self._loop_task = self._flusher = None + self._buffer.clear() + + # ---------------- 跟随(订阅方所在的进程) ---------------- + + @contextlib.contextmanager + def follow_run(self, run_id: str, after: int = 0) -> Iterator[None]: + """这条连接在看这次运行(WebSocket 接上、回放完之后):在本进程里没在执行时从库里追。""" + self._follow[run_id] = self._follow.get(run_id, 0) + 1 + self._cursor[run_id] = max(self._cursor.get(run_id, 0), int(after or 0)) + if signals.active: + signals.publish("w", f"r:{run_id}") + self._kick(run_id) + try: + yield + finally: + left = self._follow.get(run_id, 1) - 1 + if left <= 0: + self._follow.pop(run_id, None) + self._cursor.pop(run_id, None) + self._pumping.pop(run_id, None) + else: + self._follow[run_id] = left + + @contextlib.contextmanager + def watch_turn(self, turn_id: str) -> Iterator[None]: + """这条 SSE 在看这一轮:执行它的进程把保活心跳转过来。""" + self._turns[turn_id] = self._turns.get(turn_id, 0) + 1 + if signals.active: + signals.publish("w", f"t:{turn_id}") + try: + yield + finally: + left = self._turns.get(turn_id, 1) - 1 + if left <= 0: + self._turns.pop(turn_id, None) + else: + self._turns[turn_id] = left + + def _kick(self, run_id: str) -> None: + if run_id in self._follow: + task = asyncio.ensure_future(self.pump(run_id)) + task.add_done_callback(_quiet) + + async def pump(self, run_id: str) -> int: + """把库里 seq 大于游标的事件在本进程发布一遍(这次运行在本进程里执行时什么都不做)。交回发布了几条。""" + from app.core.bus import bus + from app.db.models import Run, RunEvent + from app.engine.runner import run_manager + + if run_id not in self._follow or run_manager.is_active(run_id): + return 0 + lock = self._pumping.setdefault(run_id, asyncio.Lock()) + async with lock: + after = self._cursor.get(run_id, 0) + async with SessionLocal() as session: + # 先读状态、再读事件:终态和终态事件是同一个事务提交的,读到终态时终态事件一定也读得到 + status = await session.scalar(select(Run.status).where(Run.id == run_id)) + rows = list((await session.execute(select(RunEvent).where(RunEvent.run_id == run_id, + RunEvent.seq > after) + .order_by(RunEvent.seq))).scalars()) + last_log = None + for row in rows: + event = RunEventModel(id=row.id, run_id=run_id, seq=row.seq, type=EventType(row.type), + node_id=row.node_id, ts=row.ts, data=row.data or {}) + await bus.publish(event, relayed=True) + self._cursor[run_id] = max(self._cursor.get(run_id, 0), int(row.seq or 0)) + last_log = row + ended = status is None or status in _TERMINAL or ( + status == "interrupted" and last_log is not None and last_log.type == EventType.LOG + and (last_log.data or {}).get("code") == "server_shutdown") + if ended and (rows or status is None or status in _TERMINAL): + # 同执行进程上的规矩(runner._finalize):到了终态、关停挂起的,让本进程的订阅方收到终态再结束 + await bus.close(run_id) + return len(rows) + + def _on_run(self, parts: list[str]) -> None: + if parts: + self._kick(parts[0]) + + def _on_ephemeral(self, parts: list[str]) -> None: + from app.core.bus import bus + from app.engine.runner import run_manager + + if len(parts) < 2 or parts[0] not in self._follow or run_manager.is_active(parts[0]): + return + try: + items = json.loads(parts[1]) + except ValueError: + return + for item in items if isinstance(items, list) else []: + try: + event = RunEventModel(**item) + except Exception: # noqa: BLE001 + continue + asyncio.ensure_future(bus.publish(event, relayed=True)).add_done_callback(_quiet) + + def _on_turn(self, parts: list[str]) -> None: + if parts: + from app.chat import log as turn_log + + turn_log.waiters.notify(parts[0]) + + def _on_beat(self, parts: list[str]) -> None: + if len(parts) < 2 or parts[0] not in self._turns: + return + try: + op = json.loads(parts[1]) + except ValueError: + return + if isinstance(op, dict): + from app.chat import log as turn_log + + turn_log.remote_beat(parts[0], op) + + async def _resync(self) -> None: + for run_id in list(self._follow): + await self.pump(run_id) + from app.chat import log as turn_log + + for turn_id in list(self._turns): + turn_log.waiters.notify(turn_id) + + async def _loop(self) -> None: + """跟随方:隔 WATCH_S 秒重发「我在看」,隔 POLL_S 秒回库里看一次跟随的运行。""" + last_watch = 0.0 + while True: + await asyncio.sleep(POLL_S) + now = time.monotonic() + if now - last_watch >= WATCH_S: + last_watch = now + for run_id in list(self._follow): + signals.publish("w", f"r:{run_id}") + for turn_id in list(self._turns): + signals.publish("w", f"t:{turn_id}") + for run_id in list(self._follow): + try: + await self.pump(run_id) + except Exception: # noqa: BLE001 + logger.debug("转发运行 %s 的事件出错", run_id, exc_info=True) + + # ---------------- 执行的一方 ---------------- + + def _on_watch(self, parts: list[str]) -> None: + if not parts or ":" not in parts[0]: + return + kind, target = parts[0].split(":", 1) + until = time.monotonic() + WATCH_TTL_S + if kind == "r": + self._watched_runs[target] = until + elif kind == "t": + self._watched_turns[target] = until + + def _watched(self, book: dict[str, float], key: str) -> bool: + until = book.get(key) + if until is None: + return False + if until < time.monotonic(): + book.pop(key, None) + return False + return True + + def turn_watched(self, turn_id: str) -> bool: + return signals.active and self._watched(self._watched_turns, turn_id) + + def _tap(self, event: RunEventModel) -> None: + if not signals.active: + return + if event.type in EPHEMERAL: + if self._watched(self._watched_runs, event.run_id): + self._buffer.setdefault(event.run_id, []).append(event.model_dump(mode="json")) + if self._flusher is None or self._flusher.done(): + self._flusher = asyncio.ensure_future(self._flush()) + return + if event.run_id in self._follow: + self._cursor[event.run_id] = max(self._cursor.get(event.run_id, 0), int(event.seq or 0)) + signals.publish("r", event.run_id, event.seq) + + async def _flush(self) -> None: + await asyncio.sleep(EPHEMERAL_FLUSH_S) + buffer, self._buffer = self._buffer, {} + for run_id, items in buffer.items(): + for chunk in _chunks(items): + signals.publish("e", run_id, json.dumps(chunk, ensure_ascii=False, default=str), keep=True) + + def beat(self, turn_id: str, op: dict[str, Any]) -> None: + """问数据助手的一拍保活心跳:有别的进程在看这一轮就转过去。""" + if self.turn_watched(turn_id): + text = json.dumps(op, ensure_ascii=False, default=str) + if len(text.encode()) <= PAYLOAD_MAX: + signals.publish("b", turn_id, text) + + +def _chunks(items: list[dict[str, Any]]) -> Iterator[list[dict[str, Any]]]: + """按载荷上限切开(同一个节点接着的正文增量先合成一条)。太长的单条截断增量。""" + merged: list[dict[str, Any]] = [] + for item in items: + prev = merged[-1] if merged else None + if prev is not None and prev.get("type") == item.get("type") and prev.get("node_id") == item.get("node_id") \ + and set((prev.get("data") or {})) == {"delta"} == set((item.get("data") or {})): + prev["data"] = {"delta": str(prev["data"]["delta"]) + str(item["data"]["delta"])} + prev["seq"] = item.get("seq", prev.get("seq")) + continue + merged.append(dict(item)) + chunk: list[dict[str, Any]] = [] + size = 2 + for item in merged: + text = json.dumps(item, ensure_ascii=False, default=str) + n = len(text.encode()) + if n > PAYLOAD_MAX - 2: + data = dict(item.get("data") or {}) + if "delta" in data: + data["delta"] = str(data["delta"])[: PAYLOAD_MAX // 4] + item = {**item, "data": data} + n = len(json.dumps(item, ensure_ascii=False, default=str).encode()) + if n > PAYLOAD_MAX - 2: + continue + if chunk and size + n + 1 > PAYLOAD_MAX: + yield chunk + chunk, size = [], 2 + chunk.append(item) + size += n + 1 + if chunk: + yield chunk + + +def _quiet(task: asyncio.Future[Any]) -> None: + if not task.cancelled() and task.exception() is not None: + logger.debug("转发出错", exc_info=task.exception()) + + +relay = Relay() + +__all__ = ["EPHEMERAL", "POLL_S", "Relay", "WATCH_S", "relay"] diff --git a/backend/app/cluster/requests.py b/backend/app/cluster/requests.py new file mode 100644 index 00000000..fc5fd42f --- /dev/null +++ b/backend/app/cluster/requests.py @@ -0,0 +1,114 @@ +"""发给别的进程的请求(cluster_requests):停止它在跑的东西、往它在执行的运行里记一条事件。 + +**为什么不直接改库。** 一次运行、一轮问数据、一个后台任务的执行在认领它的那个进程里(一个 asyncio 任务),手上拿着模型调用、 +查询、写到一半的库操作。别的进程收到「停止」时能做的只是告诉它:写一条请求(收件人是这一行的 owner),PostgreSQL 上 NOTIFY +叫醒它;它隔一拍心跳(最迟 worker_heartbeat_s 秒)也会来取,照本进程里的停止办(停掉执行、按停止的原因收尾、发 run.cancelled +带上是谁停的),界面上的「停止」和单进程时一样。收件人不在了(租约过期)的请求不办:那一行按「属主不在了」的规矩接手 +(cluster/reaper.py),请求跟着删掉。 + +往运行里记事件同理:执行中的运行,序号由执行它的进程发(core/bus 的进程内计数),别的进程直接写会撞号;所以交给它写。 +""" +from __future__ import annotations + +import asyncio +import logging +import weakref +from collections.abc import Awaitable, Callable +from datetime import datetime, timedelta +from typing import Any + +from sqlalchemy import delete, select + +from app.cluster import identity +from app.cluster.signal import signals +from app.db.base import SessionLocal, utcnow + +logger = logging.getLogger(__name__) + +#: (kind, target) → 办这件事的函数 (target_id, actor, payload) +_HANDLERS: dict[tuple[str, str], Callable[[str, str | None, dict[str, Any]], Awaitable[Any]]] = {} +#: 取请求时本进程里一次只取一份(心跳和 NOTIFY 同时来时不重复办) +_draining: "weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, asyncio.Lock]" = weakref.WeakKeyDictionary() + + +def handle(kind: str, target: str) -> Callable[[Callable[..., Awaitable[Any]]], Callable[..., Awaitable[Any]]]: + """登记一种请求怎么办(在各自的模块里,导入时登记)。""" + def wrap(fn: Callable[..., Awaitable[Any]]) -> Callable[..., Awaitable[Any]]: + _HANDLERS[(kind, target)] = fn + return fn + return wrap + + +async def send(owner: str, kind: str, target: str, target_id: str, *, actor: str | None = None, + payload: dict[str, Any] | None = None) -> None: + """发一条请求给 owner 那个进程(提交之后 NOTIFY 叫醒它)。""" + from app.db.models_cluster import ClusterRequest + + async with SessionLocal() as session: + session.add(ClusterRequest(owner=owner, kind=kind, target=target, target_id=target_id, actor=actor, + payload=dict(payload or {}))) + await session.commit() + signals.publish("q", owner) + + +async def drain() -> int: + """取走发给本进程的请求、照办(心跳每一拍、收到 NOTIFY 时)。交回办了几件。一件出错不影响别的。""" + from app.db.models_cluster import ClusterRequest + + loop = asyncio.get_running_loop() + lock = _draining.get(loop) + if lock is None: + lock = _draining[loop] = asyncio.Lock() + async with lock: + return await _drain() + + +async def _drain() -> int: + from app.db.models_cluster import ClusterRequest + + me = identity.owner() + async with SessionLocal() as session: + rows = list((await session.execute(select(ClusterRequest).where(ClusterRequest.owner == me) + .order_by(ClusterRequest.created_at))).scalars()) + if not rows: + return 0 + await session.execute(delete(ClusterRequest).where(ClusterRequest.id.in_([r.id for r in rows]))) + await session.commit() + done = 0 + for row in rows: + fn = _HANDLERS.get((row.kind, row.target)) + if fn is None: + logger.warning("收到不认识的进程间请求 %s/%s,不办", row.kind, row.target) + continue + try: + await fn(row.target_id, row.actor, dict(row.payload or {})) + done += 1 + except Exception: # noqa: BLE001 + logger.exception("办进程间请求 %s/%s %s 出错", row.kind, row.target, row.target_id) + return done + + +async def discard_for(owners: set[str]) -> int: + """收件人已经不在了的请求删掉(接手的进程做:那些行按属主不在了的规矩处理)。""" + from app.db.models_cluster import ClusterRequest + + if not owners: + return 0 + async with SessionLocal() as session: + done = await session.execute(delete(ClusterRequest).where(ClusterRequest.owner.in_(sorted(owners)))) + await session.commit() + return int(done.rowcount or 0) + + +async def prune(now: datetime | None = None, *, older_than_s: float = 3600.0) -> int: + """很久没人取的请求删掉(当班进程做)。""" + from app.db.models_cluster import ClusterRequest + + floor = (now or utcnow()) - timedelta(seconds=older_than_s) + async with SessionLocal() as session: + done = await session.execute(delete(ClusterRequest).where(ClusterRequest.created_at < floor)) + await session.commit() + return int(done.rowcount or 0) + + +__all__ = ["discard_for", "drain", "handle", "prune", "send"] diff --git a/backend/app/cluster/service.py b/backend/app/cluster/service.py new file mode 100644 index 00000000..d4d5b39f --- /dev/null +++ b/backend/app/cluster/service.py @@ -0,0 +1,227 @@ +"""本进程在协调里的那一份:启动时登记、起心跳,关停时交班、删登记。 + +心跳每一拍(AGENTLAB_WORKER_HEARTBEAT_S,缺省 5 秒): +1. 续自己的租约(登记不见了就重新登记); +2. 刷新「别的进程谁还活着」; +3. 取走发给本进程的请求照办(停止、往运行里记事件,cluster/requests.py); +4. 本进程自己的周期活(on_beat 登记的:访问密钥的请求记录写库……——这些是每个进程攒在自己内存里的,只能自己写); +5. 争 / 续调度当班(cluster/leader.py);当班的进程在有进程走了(少了一个、收到 gone 通知)或者隔一个租约时长时,接手 + 不在了的进程留下的行(cluster/reaper.py),顺手删掉早就过期的登记和没人取的请求。 + +调度循环(scheduler/loop.py)在每个进程里都起,每一拍先看本进程当不当班,不当班就什么都不做。 +""" +from __future__ import annotations + +import asyncio +import contextlib +import logging +import time +from collections.abc import Awaitable, Callable +from typing import Any + +from app.cluster import guard, identity, leader, reaper, requests, state, workers +from app.cluster.signal import signals +from app.core.config import settings + +logger = logging.getLogger(__name__) + +_beats: list[tuple[str, Callable[[], Awaitable[Any]]]] = [] +_leader_listeners: list[Callable[[bool], Any]] = [] + + +def on_beat(name: str, fn: Callable[[], Awaitable[Any]]) -> None: + """登记一件本进程每拍心跳都要做的事(同名换成新的)。""" + for i, (known, _) in enumerate(_beats): + if known == name: + _beats[i] = (name, fn) + return + _beats.append((name, fn)) + + +def on_leadership(fn: Callable[[bool], Any]) -> None: + """当班换了(接手 True、交出 False)时调。""" + if fn not in _leader_listeners: + _leader_listeners.append(fn) + + +def leading() -> bool: + """本进程现在当不当调度班。没起协调(测试、脚本里直接调调度循环、直接 run_manager.setup)时算当班:那时只有这一个进程。""" + return not service.started or leader.scheduler.held() + + +class _Service: + def __init__(self) -> None: + self.task: asyncio.Task[None] | None = None + self.wake: asyncio.Event | None = None + self.reap_now = False + self.last_reap = 0.0 + self.others: frozenset[str] = frozenset() + self.started = False + + async def start(self) -> None: + """登记、开始收发通知、起心跳。SQLite 时先取单进程的守卫(拿不到抛 guard.SingleProcessOnly)。""" + if self.started: + return + guard.acquire() + exclusive = settings.db_dialect == "sqlite" + await workers.register(exclusive=exclusive) + live = await workers.liveness() + state.remember(live) + self.others = live.others + await signals.start() + signals.on("q", self._on_request) + signals.on("gone", self._on_gone) + if settings.db_dialect == "postgresql": + await signals.ready(timeout=5.0) + self.wake = asyncio.Event() + self.started = True + from app.cluster import caches + + await caches.start() + await self._lead() + self.task = asyncio.create_task(self._loop(), name="agentlab-cluster-heartbeat") + logger.info("工作进程 %s 已登记(%s,租约 %.0f 秒、心跳 %.0f 秒;现在另有 %d 个进程)", identity.describe(identity.owner()), + "PostgreSQL,可以多进程" if settings.db_dialect == "postgresql" else "SQLite,单进程", + workers.lease_s(), workers.heartbeat_s(), len(live.others)) + + def _on_request(self, parts: list[str]) -> None: + if parts and parts[0] == identity.owner() and self.wake is not None: + asyncio.ensure_future(requests.drain()) + + def _on_gone(self, _parts: list[str]) -> None: + self.reap_now = True + if self.wake is not None: + self.wake.set() + + async def _lead(self) -> bool: + was = leader.scheduler.held() + try: + now_leading = await leader.scheduler.claim() + except Exception: # noqa: BLE001 - 库一时不通:本地按没当班算 + logger.exception("争调度当班时出错") + now_leading = False + if now_leading != was: + for fn in list(_leader_listeners): + try: + out = fn(now_leading) + if asyncio.iscoroutine(out): + await out + except Exception: # noqa: BLE001 + logger.exception("当班换了之后的处理出错") + if now_leading: + self.reap_now = True + return now_leading + + async def beat(self) -> None: + """心跳的一拍(测试里直接调)。""" + if not await workers.renew(): + logger.warning("本进程的登记不见了(可能被当成没了),重新登记") + await workers.register() + live = await workers.liveness() + state.remember(live) + if self.others - live.others: + self.reap_now = True + self.others = live.others + try: + await requests.drain() + except Exception: # noqa: BLE001 + logger.exception("取发给本进程的请求时出错") + try: + from app.cluster import caches + + await caches.poll() + except Exception: # noqa: BLE001 + logger.exception("读别的进程的缓存变化时出错") + for name, fn in list(_beats): + try: + await fn() + except Exception: # noqa: BLE001 + logger.exception("心跳里的「%s」出错,下一拍再试", name) + if await self._lead() and (self.reap_now or time.monotonic() - self.last_reap >= workers.lease_s()): + # 隔一会儿的这一次信得过自己:属主是本进程的行(刚建、还没交给后台任务的)不当成没人管 + await self.reap(live.trusting()) + + async def reap(self, live: workers.Liveness | None = None) -> dict[str, Any]: + self.reap_now = False + self.last_reap = time.monotonic() + live = live or await state.current(fresh=True) + dead = await _dead_owners(live) + out = await reaper.reap(live) + if dead: + await requests.discard_for(dead) + with contextlib.suppress(Exception): + await workers.prune() + await requests.prune() + if settings.db_dialect == "postgresql": + from app.cluster import caches, counters + + await counters.prune() + await caches.prune() + return out + + async def _loop(self) -> None: + assert self.wake is not None + while True: + with contextlib.suppress(asyncio.TimeoutError): + await asyncio.wait_for(self.wake.wait(), timeout=workers.heartbeat_s()) + self.wake.clear() + try: + await self.beat() + except asyncio.CancelledError: + raise + except Exception: # noqa: BLE001 - 一拍出错(库一时不通)不该让心跳停下 + logger.exception("心跳这一拍出错,下一拍照常进行") + + async def stop(self) -> None: + """关停:停心跳、交班、删登记(别的进程马上接手本进程留下的行),停收发通知。""" + if not self.started: + return + self.started = False + if self.task is not None: + self.task.cancel() + with contextlib.suppress(asyncio.CancelledError, Exception): + await self.task + self.task = None + with contextlib.suppress(Exception): + await leader.scheduler.release() + with contextlib.suppress(Exception): + await workers.unregister() + signals.publish("gone", identity.owner()) + await signals.stop() + state.forget() + guard.release() + + +async def startup_reap() -> dict[str, Any]: + """启动时的那一次接手(每个进程都做;那时本进程什么都还没在跑,属主是自己的只会是前一次起停留下的)。""" + return await service.reap(await state.current(fresh=True)) + + +async def _dead_owners(live: workers.Liveness) -> set[str]: + """库里登记着、租约已过期的进程(它们的请求删掉)。""" + from sqlalchemy import select + + from app.db.base import SessionLocal + from app.db.models_cluster import ClusterRequest + + async with SessionLocal() as session: + owners = set((await session.execute(select(ClusterRequest.owner).distinct())).scalars()) + return {o for o in owners if o != identity.owner() and o not in live.others} + + +service = _Service() + + +async def start() -> None: + await service.start() + + +async def stop() -> None: + await service.stop() + + +def started() -> bool: + return service.started + + +__all__ = ["leading", "on_beat", "on_leadership", "service", "start", "started", "startup_reap", "stop"] diff --git a/backend/app/cluster/signal.py b/backend/app/cluster/signal.py new file mode 100644 index 00000000..a2592d97 --- /dev/null +++ b/backend/app/cluster/signal.py @@ -0,0 +1,215 @@ +"""进程之间的叫醒:PostgreSQL 的 LISTEN / NOTIFY(频道 agentlab)。SQLite 只有一个进程,什么都不发、也不听——本进程里的 +叫醒(事件总线、问数据的等待者、名额和同源队的等待者)各自照旧。 + +**只叫醒,不带数据。** 载荷是「类别|发出方|哪一个|到第几条」:运行事件 r|…||,问数据一轮的事件 +t|…||,发给某个进程的请求 q|…|,名额 / 同源队有了空位 slot、lane|…|,进程走了 gone|…|, +缓存要重读 cache|…。发出方是 OWNER 的随机串:自己发的自己不处理(本进程里已经叫醒过了)。收到的一方按 id 回库里读——NOTIFY +丢了、连接断了重连期间漏了,靠各处的兜底轮询补上(运行的转发、问数据的流、名额等待都隔一会儿回库里看一次),叫醒只让它们不必 +等到那一拍。 + +例外是不落库的那两样:运行里的流式增量(正文 token、思考 delta)和问数据助手的保活心跳,只在有别的进程在看的时候(它们发 +w 登记「我在看」,cluster/relay.py)由执行的进程把内容放进载荷转过去(攒一小段合成一条、截在 7000 字节以内)。丢了无妨: +它们本来就不进回放,下一条落库的事件会把画面带到正确的样子。 + +**发在提交之后。** 调用方提交了事务再 publish:收到的一方去读时一定读得到。发是攒一下再批量发的(FLUSH_S),同一个 id 只发 +最后一个 seq:一次运行几十条事件,跨进程的订阅方只要知道「到第几条了」。用一条专门的连接(不占连接池),收和发都走它; +断了隔一会儿重连,重连之后叫一遍各处的「补一次」(on_resync),漏掉的那一段由它们回库里读。 +""" +from __future__ import annotations + +import asyncio +import contextlib +import logging +from collections.abc import Awaitable, Callable +from typing import Any + +from app.cluster import identity +from app.core.config import settings + +logger = logging.getLogger(__name__) + +CHANNEL = "agentlab" +#: 攒多久发一批(秒) +FLUSH_S = 0.02 +#: 重连的间隔(秒) +RETRY_S = 2.0 + +Handler = Callable[[list[str]], Any] + + +def _tag() -> str: + return identity.owner().rsplit(":", 1)[-1] + + +class Signals: + def __init__(self) -> None: + self._handlers: dict[str, list[Handler]] = {} + self._raw: set[str] = set() + self._seq = 0 + self._resync: list[Callable[[], Any]] = [] + self._pending: dict[tuple[str, ...], str] = {} + self._conn: Any = None + self._task: asyncio.Task[None] | None = None + self._flusher: asyncio.Task[None] | None = None + self._wake: asyncio.Event | None = None + self._stopping = False + + # ---------------- 登记 ---------------- + + def on(self, kind: str, handler: Handler, *, raw: bool = False) -> None: + """收到 kind 这一类时调 handler(其余几段)。可以是协程函数。同一个 handler 重复登记无害。raw:载荷里带着内容(JSON), + 只按第一段 id 切开,交回 [id, 余下的原文]。""" + handlers = self._handlers.setdefault(kind, []) + if handler not in handlers: + handlers.append(handler) + if raw: + self._raw.add(kind) + + def on_resync(self, handler: Callable[[], Any]) -> None: + """重连之后(可能漏了通知)调一次。""" + if handler not in self._resync: + self._resync.append(handler) + + # ---------------- 发 ---------------- + + @property + def active(self) -> bool: + return self._task is not None and not self._task.done() + + def publish(self, kind: str, *parts: object, keep: bool = False) -> None: + """攒一条要发的(没在听,也就是 SQLite、没起服务的测试里,什么都不做)。同一类同一个 id 只留最后一条;keep 时每条都发 + (带内容的转发:前后几段都要)。""" + if not self.active: + return + text = [str(p) for p in parts] + if keep: + self._seq += 1 + key: tuple[str, ...] = (kind, "#", str(self._seq)) + else: + key = (kind, text[0]) if text else (kind,) + self._pending[key] = "|".join([kind, _tag(), *text]) + if self._wake is not None: + self._wake.set() + + async def _flush_loop(self) -> None: + assert self._wake is not None + while not self._stopping: + await self._wake.wait() + self._wake.clear() + await asyncio.sleep(FLUSH_S) + batch, self._pending = list(self._pending.values()), {} + if not batch or self._conn is None: + continue + try: + await self._conn.execute(f"SELECT pg_notify('{CHANNEL}', p) FROM unnest($1::text[]) AS p", batch) + except Exception: # noqa: BLE001 - 发不出去只是别的进程晚一拍知道(它们会回库里看) + logger.debug("NOTIFY 没发出去", exc_info=True) + + # ---------------- 收 ---------------- + + def _received(self, _conn: Any, _pid: int, _channel: str, payload: str) -> None: + head = payload.split("|", 2) + if len(head) < 2 or head[1] == _tag(): + return + kind, body = head[0], head[2] if len(head) > 2 else "" + rest = (body.split("|", 1) if body else []) if kind in self._raw else (body.split("|") if body else []) + for handler in list(self._handlers.get(kind, ())): + try: + out = handler(rest) + if asyncio.iscoroutine(out): + task = asyncio.ensure_future(out) + task.add_done_callback(_quiet) + except Exception: # noqa: BLE001 + logger.exception("处理进程间通知 %s 出错", kind) + + async def _listen(self) -> None: + import asyncpg + + from app.db.base import APPLICATION_NAME + from app.db.urls import libpq_conninfo + + dsn = libpq_conninfo(settings.database_url) + first = True + while not self._stopping: + lost = asyncio.Event() + try: + conn = await asyncpg.connect(dsn, server_settings={"application_name": APPLICATION_NAME}) + except Exception as e: # noqa: BLE001 + logger.warning("进程间通知的连接没连上(%s),%.0f 秒后重试", e, RETRY_S) + await asyncio.sleep(RETRY_S) + continue + try: + conn.add_termination_listener(lambda _c: lost.set()) + await conn.add_listener(CHANNEL, self._received) + self._conn = conn + if not first: + logger.info("进程间通知的连接已重连") + self._fire_resync() + first = False + await lost.wait() + except asyncio.CancelledError: + raise + except Exception: # noqa: BLE001 + logger.warning("进程间通知的连接出错,%.0f 秒后重连", RETRY_S, exc_info=True) + finally: + self._conn = None + with contextlib.suppress(Exception): + await asyncio.wait_for(conn.close(), timeout=2.0) + if not self._stopping: + await asyncio.sleep(RETRY_S) + + def _fire_resync(self) -> None: + for handler in list(self._resync): + try: + out = handler() + if asyncio.iscoroutine(out): + asyncio.ensure_future(out).add_done_callback(_quiet) + except Exception: # noqa: BLE001 + logger.exception("重连之后补一次出错") + + # ---------------- 起停 ---------------- + + async def start(self) -> None: + """应用库是 PostgreSQL 时开始收发;SQLite 什么都不做。""" + if settings.db_dialect != "postgresql" or self.active: + return + self._stopping = False + self._wake = asyncio.Event() + self._task = asyncio.create_task(self._listen(), name="agentlab-cluster-listen") + self._flusher = asyncio.create_task(self._flush_loop(), name="agentlab-cluster-notify") + + async def ready(self, timeout: float = 5.0) -> bool: + """等连接连上(测试、启动时用)。""" + loop = asyncio.get_running_loop() + deadline = loop.time() + timeout + while self.active and self._conn is None and loop.time() < deadline: + await asyncio.sleep(0.02) + return self._conn is not None + + async def stop(self) -> None: + self._stopping = True + tasks = [t for t in (self._flusher, self._task) if t is not None] + # 停之前把攒着的发出去:关停时交班、删登记这几条别的进程要尽快知道 + if self._pending and self._conn is not None: + with contextlib.suppress(Exception): + await asyncio.wait_for(self._conn.execute( + f"SELECT pg_notify('{CHANNEL}', p) FROM unnest($1::text[]) AS p", list(self._pending.values())), + timeout=1.0) + self._pending = {} + for task in tasks: + task.cancel() + for task in tasks: + with contextlib.suppress(asyncio.CancelledError, Exception): + await task + self._task = self._flusher = None + self._conn = None + + +def _quiet(task: asyncio.Future[Any]) -> None: + if not task.cancelled() and task.exception() is not None: + logger.error("处理进程间通知出错", exc_info=task.exception()) + + +signals = Signals() + +__all__ = ["CHANNEL", "Signals", "signals"] diff --git a/backend/app/cluster/slots.py b/backend/app/cluster/slots.py new file mode 100644 index 00000000..06e2312f --- /dev/null +++ b/backend/app/cluster/slots.py @@ -0,0 +1,320 @@ +"""运行的并发名额(run_slots,docs/design/orchestration-round/00-总方案.md §5 W2-b「优先级与并发」)。 + +**为什么。** 以前是 runner 里一把进程内的信号量(AGENTLAB_MAX_CONCURRENT_RUNS,缺省 20),谁先到谁先跑:一次评测几十道题、 +一批口径召回,就能把 20 个名额占满,问数据、画布试跑在后面排队。多进程之后信号量也只管得了本进程。 + +**现在。** 每段执行(发起、审批之后接着跑、继续运行;停在审批上的不占)开始前要一个名额,按运行的来源归四类: + + 交互 interactive:问数据、画布试跑、正式运行(手动点的)、蓝图试运行、个人应用的刷新——有人在屏幕前等 + 应用 app:应用入口、按需视图、外部系统的钩子、接口 + 定时 schedule:定时任务(含事件触发) + 系统 system:评测、口径召回、对账、补算、按口径正式出具——服务端替人跑的 + +每类有全站的并发上限和优先级(设置里「运行并发」一组,管理员可调),另有全站总上限。名额按库里的计数发:等着的按 +优先级高的先、同一优先级先来先得,只要它那一类没到上限、全站没到总上限就发给它。所以评测再多也只占得到系统那一类的 +几个名额,问数据不用等它;全站满了的时候,空出来的名额先给交互。已经在跑的不抢。 + +缺省(总上限取 AGENTLAB_MAX_CONCURRENT_RUNS):交互 = 总上限、应用 2/5、定时 1/5、系统 1/5(至少各 1),优先级 40 / 30 / 20 / 10。 +应用、定时、系统三类加起来不到总上限:交互总留得出名额。单进程时总上限和以前的信号量一样大,问数据、试跑不再被评测挤占。 + +**怎么发。** 要名额的一方先在本进程登记、再插一行「等着」,然后自己去发一次(grant:把所有等着的按规矩过一遍,谁够格就 +改成「已发」);没轮到就等叫醒(本进程放名额时叫醒,PostgreSQL 上别的进程放名额时 NOTIFY slot 叫醒),另隔一会儿自己再 +发一次兜底。发的那一下全站串行:PostgreSQL 上在事务里取咨询锁,SQLite 只有一个进程,本进程的锁就够。名额用完删掉那一行。 +属主不在了的行(进程没了;本进程里登记已经没了的——测试里事件循环关了留下的)发的时候顺手删掉,名额不会漏。 +""" +from __future__ import annotations + +import asyncio +import contextlib +import logging +import time +import weakref +from collections import Counter +from collections.abc import AsyncIterator +from dataclasses import dataclass, field +from typing import Any + +from sqlalchemy import delete, select, text, update + +from app.cluster import identity +from app.cluster.signal import signals +from app.core.config import settings +from app.db.base import SessionLocal, utcnow + +logger = logging.getLogger(__name__) + +CLASSES = ("interactive", "app", "schedule", "system") +CLASS_LABEL = {"interactive": "交互", "app": "应用", "schedule": "定时", "system": "系统"} +#: 运行来源 → 类别(engine/origin.ORIGIN_KINDS)。没记来源的(直接调 run_manager.start、升级前的)算交互:同以前不设限 +ORIGIN_CLASS = { + "chat": "interactive", "canvas": "interactive", "launcher": "interactive", "trial": "interactive", + "personal_app": "interactive", + "app_entry": "app", "app_view": "app", "hook": "app", "api": "app", + "schedule": "schedule", + "eval": "system", "recall": "system", "reconcile": "system", "app_backfill": "system", "ledger_issue": "system", +} +SETTING_KEY = "concurrency" +DEFAULT_PRIORITY = {"interactive": 40, "app": 30, "schedule": 20, "system": 10} +#: 上限、优先级的取值范围 +LIMIT_MAX = 1000 +PRIORITY_MAX = 100 +#: 等名额时隔多久自己再发一次(秒):PostgreSQL 上别的进程放的名额靠 NOTIFY 叫醒,这是兜底;SQLite 只有本进程放名额 +POLL_S = {"postgresql": 1.0, "sqlite": 5.0} +#: 设置读一次管多久(秒):改了设置,最迟这么久之后各进程按新的发 +POLICY_TTL_S = 3.0 +#: 咨询锁的号(发名额那一下全站串行) +_GRANT_KEY = 0x4167536C6F747321 # "AgSlots!" + + +def class_of(origin_kind: str | None) -> str: + return ORIGIN_CLASS.get(origin_kind or "", "interactive") + + +def defaults(total: int | None = None) -> dict[str, Any]: + """缺省的设置(总上限取 AGENTLAB_MAX_CONCURRENT_RUNS)。""" + total = max(1, int(total if total is not None else settings.max_concurrent_runs)) + share = {"interactive": total, "app": max(1, total * 2 // 5), "schedule": max(1, total // 5), + "system": max(1, total // 5)} + return {"max_total": total, + "classes": {k: {"limit": share[k], "priority": DEFAULT_PRIORITY[k]} for k in CLASSES}} + + +def values(stored: Any) -> dict[str, Any]: + """存着的(可能缺项、存坏了)→ 生效的:缺的、坏的补缺省。""" + base = defaults() + out = {"max_total": base["max_total"], "classes": {k: dict(v) for k, v in base["classes"].items()}} + if not isinstance(stored, dict): + return out + total = stored.get("max_total") + if isinstance(total, int) and not isinstance(total, bool) and 1 <= total <= LIMIT_MAX: + out["max_total"] = total + classes = stored.get("classes") if isinstance(stored.get("classes"), dict) else {} + for k in CLASSES: + item = classes.get(k) if isinstance(classes.get(k), dict) else {} + limit, prio = item.get("limit"), item.get("priority") + if isinstance(limit, int) and not isinstance(limit, bool) and 1 <= limit <= LIMIT_MAX: + out["classes"][k]["limit"] = limit + if isinstance(prio, int) and not isinstance(prio, bool) and 0 <= prio <= PRIORITY_MAX: + out["classes"][k]["priority"] = prio + return out + + +def problem(sent: Any) -> str | None: + """设置接口写进来的这一组有没有问题(说明给人看)。只核对给了的项。""" + if not isinstance(sent, dict): + return "运行并发的设置需要是对象" + total = sent.get("max_total") + if total is not None and (not isinstance(total, int) or isinstance(total, bool) or not 1 <= total <= LIMIT_MAX): + return f"全站同时运行的上限需要是 1 到 {LIMIT_MAX} 之间的整数" + classes = sent.get("classes") + if classes is None: + return None + if not isinstance(classes, dict): + return "各类运行的设置需要是对象" + for k, item in classes.items(): + if k not in CLASSES: + return f"没有「{k}」这一类运行(只有交互、应用、定时、系统)" + if not isinstance(item, dict): + return f"「{CLASS_LABEL[k]}」类的设置需要是对象" + limit, prio = item.get("limit"), item.get("priority") + if limit is not None and (not isinstance(limit, int) or isinstance(limit, bool) or not 1 <= limit <= LIMIT_MAX): + return f"「{CLASS_LABEL[k]}」类的并发上限需要是 1 到 {LIMIT_MAX} 之间的整数" + if prio is not None and (not isinstance(prio, int) or isinstance(prio, bool) or not 0 <= prio <= PRIORITY_MAX): + return f"「{CLASS_LABEL[k]}」类的优先级需要是 0 到 {PRIORITY_MAX} 之间的整数" + return None + + +def merged(saved: Any, sent: dict[str, Any]) -> dict[str, Any]: + """只给了一部分的(只改一类的上限)补上存着的另几项。""" + out = values(saved) + if "max_total" in sent and sent["max_total"] is not None: + out["max_total"] = sent["max_total"] + for k, item in (sent.get("classes") or {}).items(): + for field_ in ("limit", "priority"): + if item.get(field_) is not None: + out["classes"][k][field_] = item[field_] + return out + + +_policy: tuple[float, dict[str, Any]] | None = None + + +async def policy(*, fresh: bool = False) -> dict[str, Any]: + """现在生效的设置(读一次管 POLICY_TTL_S 秒)。""" + global _policy + now = time.monotonic() + if not fresh and _policy is not None and now - _policy[0] < POLICY_TTL_S: + return _policy[1] + from app.db.models import Setting + + async with SessionLocal() as session: + row = await session.get(Setting, SETTING_KEY) + value = values(row.value if row is not None else None) + _policy = (now, value) + return value + + +def forget_policy() -> None: + """设置刚改了:本进程下一次重读(别的进程最迟 POLICY_TTL_S 秒)。""" + global _policy + _policy = None + + +# -------------------------------------------------------------------------- +# 本进程里要名额的登记 +# -------------------------------------------------------------------------- + + +@dataclass +class _Holder: + run_id: str + loop: asyncio.AbstractEventLoop + woke: asyncio.Event = field(default_factory=asyncio.Event) + + +_holders: dict[str, _Holder] = {} +_grant_locks: weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, asyncio.Lock] = weakref.WeakKeyDictionary() + + +def _local(run_id: str) -> bool: + """这一行在本进程里还有人在要 / 在用(别的事件循环留下的不算:那个循环已经没人会放它了)。""" + holder = _holders.get(run_id) + if holder is None: + return False + try: + return holder.loop is asyncio.get_running_loop() and not holder.loop.is_closed() + except RuntimeError: + return False + + +def wake_local() -> None: + for holder in list(_holders.values()): + if not holder.loop.is_closed(): + holder.loop.call_soon_threadsafe(holder.woke.set) + + +def _grant_lock() -> asyncio.Lock: + loop = asyncio.get_running_loop() + lock = _grant_locks.get(loop) + if lock is None: + lock = _grant_locks[loop] = asyncio.Lock() + return lock + + +async def grant() -> list[str]: + """把等着的按规矩过一遍,够格的改成已发;属主不在了的行删掉。交回这一次发出去的。全站串行(见模块说明)。""" + from app.cluster import state + from app.db.models_cluster import RunSlot + + pol = await policy() + async with _grant_lock(): + async with SessionLocal() as session: + if session.get_bind().dialect.name == "postgresql": + await session.execute(text("SELECT pg_advisory_xact_lock(:k)"), {"k": _GRANT_KEY}) + rows = list((await session.execute(select(RunSlot))).scalars()) + live = await state.live_for([r.owner for r in rows]) + dead = [r.run_id for r in rows if not live.alive(r.owner, local=_local(r.run_id))] + if dead: + await session.execute(delete(RunSlot).where(RunSlot.run_id.in_(dead))) + rows = [r for r in rows if r.run_id not in set(dead)] + used = Counter(r.klass for r in rows if r.granted) + total = sum(used.values()) + limits = {k: int(v["limit"]) for k, v in pol["classes"].items()} + granted: list[str] = [] + waiting = sorted((r for r in rows if not r.granted), + key=lambda r: (-int(r.priority or 0), r.requested_at, r.run_id)) + now = utcnow() + for row in waiting: + if total >= int(pol["max_total"]): + break + if used[row.klass] >= limits.get(row.klass, int(pol["max_total"])): + continue + used[row.klass] += 1 + total += 1 + granted.append(row.run_id) + if granted: + await session.execute(update(RunSlot).where(RunSlot.run_id.in_(granted)) + .values(granted=True, granted_at=now).execution_options(synchronize_session=False)) + await session.commit() + if dead: + logger.info("运行名额:收回了 %d 个属主已经不在的名额", len(dead)) + if granted: + wake_local() + signals.publish("slot") + return granted + + +async def _granted(run_id: str) -> bool | None: + """这一行发了没有:True 发了,False 还在等,None 这一行不见了(被当成属主不在了删掉)。""" + from app.db.models_cluster import RunSlot + + async with SessionLocal() as session: + return await session.scalar(select(RunSlot.granted).where(RunSlot.run_id == run_id)) + + +async def _request(run_id: str, klass: str, priority: int) -> None: + from app.db.models_cluster import RunSlot + + async with SessionLocal() as session: + await session.execute(delete(RunSlot).where(RunSlot.run_id == run_id)) + session.add(RunSlot(run_id=run_id, klass=klass, priority=priority, owner=identity.owner(), granted=False, + requested_at=utcnow())) + await session.commit() + + +async def _release(run_id: str) -> None: + from app.db.models_cluster import RunSlot + + async with SessionLocal() as session: + await session.execute(delete(RunSlot).where(RunSlot.run_id == run_id, RunSlot.owner == identity.owner())) + await session.commit() + + +@contextlib.asynccontextmanager +async def hold(run_id: str, origin_kind: str | None) -> AsyncIterator[str]: + """要一个名额(等到发给自己),出来时放掉。交回类别。取消(停止运行)在等的时候也照常生效。""" + klass = class_of(origin_kind) + pol = await policy() + priority = int(pol["classes"][klass]["priority"]) + holder = _Holder(run_id, asyncio.get_running_loop()) + _holders[run_id] = holder + poll = POLL_S.get(settings.db_dialect, 5.0) + try: + await _request(run_id, klass, priority) + while True: + holder.woke.clear() + await grant() + got = await _granted(run_id) + if got: + break + if got is None: + await _request(run_id, klass, priority) + continue + with contextlib.suppress(asyncio.TimeoutError): + await asyncio.wait_for(holder.woke.wait(), timeout=poll) + yield klass + finally: + if _holders.get(run_id) is holder: + _holders.pop(run_id, None) + try: + await asyncio.shield(_release(run_id)) + except Exception: # noqa: BLE001 - 放不掉的名额下次发的时候按「本进程里已经没人用」收回 + logger.exception("运行 %s 的名额没放掉,下次发名额时收回", run_id) + wake_local() + signals.publish("slot") + + +async def snapshot() -> dict[str, Any]: + """现在各类在跑、在等几个(系统信息、验收脚本看)。""" + from app.db.models_cluster import RunSlot + + async with SessionLocal() as session: + rows = list((await session.execute(select(RunSlot.klass, RunSlot.granted))).all()) + out = {k: {"running": 0, "waiting": 0} for k in CLASSES} + for klass, granted in rows: + out.setdefault(klass, {"running": 0, "waiting": 0})["running" if granted else "waiting"] += 1 + return {"policy": await policy(), "classes": out} + + +__all__ = ["CLASSES", "CLASS_LABEL", "ORIGIN_CLASS", "SETTING_KEY", "class_of", "defaults", "forget_policy", "grant", + "hold", "merged", "policy", "problem", "snapshot", "values", "wake_local"] diff --git a/backend/app/cluster/state.py b/backend/app/cluster/state.py new file mode 100644 index 00000000..5b4f58e2 --- /dev/null +++ b/backend/app/cluster/state.py @@ -0,0 +1,41 @@ +"""本进程手上的「别的进程谁还活着」:心跳每一拍刷新一次;碰到不认识的属主(刚起来的进程)当场再查一次库。""" +from __future__ import annotations + +import time +from collections.abc import Iterable + +from app.cluster import identity +from app.cluster.workers import Liveness, heartbeat_s, liveness + +_snapshot: tuple[float, Liveness] | None = None + + +def remember(live: Liveness) -> None: + global _snapshot + _snapshot = (time.monotonic(), live) + + +def forget() -> None: + global _snapshot + _snapshot = None + + +async def current(*, fresh: bool = False) -> Liveness: + """最近的一份(不超过一拍心跳),fresh 时当场查库。""" + if not fresh and _snapshot is not None and time.monotonic() - _snapshot[0] < heartbeat_s(): + return _snapshot[1] + live = await liveness() + remember(live) + return live + + +async def live_for(owners: Iterable[str | None]) -> Liveness: + """判断这几个属主用的一份:里面有手上这份不认识的别的进程,就当场查一次库(它可能刚起来)。""" + live = await current() + me = identity.owner() + if any(o and o != me and o not in live.others for o in owners): + live = await current(fresh=True) + return live + + +__all__ = ["current", "forget", "live_for", "remember"] diff --git a/backend/app/cluster/workers.py b/backend/app/cluster/workers.py new file mode 100644 index 00000000..d3b7066a --- /dev/null +++ b/backend/app/cluster/workers.py @@ -0,0 +1,145 @@ +"""工作进程的登记和租约(cluster_workers),以及「这一行的属主还在不在」。 + +- register / renew / unregister:启动时登记、隔一拍心跳续租约、平稳关停时删掉自己那一行(别的进程马上就知道它不在了, + 不必等租约到期)。 +- liveness():现在哪些别的进程还活着(租约没过期)。各种「启动整理」「接手」都只碰属主不在了的行: + Liveness.alive(owner, local=…) —— 属主是别的进程:看它的租约;属主是自己:看这一行在本进程里是不是真的在跑(local), + 同一个进程里前一次起停留下的(测试里一个进程起停多次服务)照样算不在了;没有属主(升级前的行)算不在了。 + +时间按应用的 utcnow(各台机器要对时,NTP):租约以十秒计,几百毫秒的偏差无碍。 +""" +from __future__ import annotations + +import logging +import os +from dataclasses import dataclass +from datetime import datetime, timedelta +from typing import Any + +from sqlalchemy import delete, select, update + +from app.cluster import identity +from app.core.config import settings +from app.db.base import SessionLocal, utcnow + +logger = logging.getLogger(__name__) + + +def lease_s() -> float: + """租约多长(秒):至少是心跳间隔的三倍——库一时慢了、一拍没续上,不算进程没了。""" + return max(float(settings.worker_lease_s), 3.0 * float(settings.worker_heartbeat_s)) + + +def heartbeat_s() -> float: + return float(settings.worker_heartbeat_s) + + +@dataclass(frozen=True) +class Liveness: + """某一刻还活着的别的进程。""" + + others: frozenset[str] = frozenset() + #: 属主是自己的行一律算活着(当班进程隔一会儿接手时用:自己刚建、还没交给后台任务的行不能当成没人管)。启动时的那一次 + #: 不信:那时本进程什么都还没在跑,属主是自己的只会是同一个进程里前一次起停留下的 + trust_mine: bool = False + + def alive(self, owner: str | None, *, local: bool = False) -> bool: + """这一行的属主还在不在。local:这一行在本进程里是不是真的在跑(属主是自己时只看它)。""" + if not owner: + return False + if owner == identity.owner(): + return local or self.trust_mine + return owner in self.others + + def trusting(self) -> Liveness: + return Liveness(self.others, trust_mine=True) + + +async def liveness(now: datetime | None = None) -> Liveness: + """现在还活着(租约没过期)的别的进程。""" + from app.db.models_cluster import ClusterWorker + + now = now or utcnow() + me = identity.owner() + async with SessionLocal() as session: + ids = (await session.execute(select(ClusterWorker.id).where( + ClusterWorker.lease_until > now, ClusterWorker.id != me))).scalars().all() + return Liveness(frozenset(ids)) + + +async def alive(owner: str | None, *, local: bool = False) -> bool: + """单独问一个属主(停止时找收件人用)。""" + if not owner or owner == identity.owner(): + return bool(owner) and local + return (await liveness()).alive(owner) + + +def _info() -> dict[str, Any]: + return {"data_dir": str(settings.data_dir), "workers": settings.workers} + + +async def register(*, exclusive: bool = False) -> None: + """登记自己(已有就续上)。exclusive:库里别的进程一律当成已经不在了,删掉它们的登记(SQLite:只有一个进程, + 守卫已经确认没有别的进程在用这个数据目录,见 cluster/guard.py)。""" + from app.db.models_cluster import ClusterWorker + + me = identity.owner() + now = utcnow() + async with SessionLocal() as session: + if exclusive: + await session.execute(delete(ClusterWorker).where(ClusterWorker.id != me)) + row = await session.get(ClusterWorker, me) + if row is None: + session.add(ClusterWorker(id=me, host=identity.host(), pid=os.getpid(), started_at=now, heartbeat_at=now, + lease_until=now + timedelta(seconds=lease_s()), info=_info())) + else: + row.heartbeat_at, row.lease_until = now, now + timedelta(seconds=lease_s()) + await session.commit() + + +async def renew() -> bool: + """续一拍租约。自己那一行不见了(别的进程以为我没了、清掉了)返回 False,调用方重新登记。""" + from app.db.models_cluster import ClusterWorker + + now = utcnow() + async with SessionLocal() as session: + done = await session.execute(update(ClusterWorker).where(ClusterWorker.id == identity.owner()).values( + heartbeat_at=now, lease_until=now + timedelta(seconds=lease_s())) + .execution_options(synchronize_session=False)) + await session.commit() + return done.rowcount == 1 + + +async def unregister() -> None: + from app.db.models_cluster import ClusterWorker + + async with SessionLocal() as session: + await session.execute(delete(ClusterWorker).where(ClusterWorker.id == identity.owner())) + await session.commit() + + +async def prune(now: datetime | None = None) -> int: + """删掉早就过期的登记(当班进程做):租约过期超过十倍租约时长的。它们认领的行早已被接手。""" + from app.db.models_cluster import ClusterWorker + + floor = (now or utcnow()) - timedelta(seconds=10 * lease_s()) + async with SessionLocal() as session: + done = await session.execute(delete(ClusterWorker).where(ClusterWorker.lease_until < floor)) + await session.commit() + return int(done.rowcount or 0) + + +async def listing() -> list[dict[str, Any]]: + """现在登记着的进程(系统信息页、验收脚本看)。""" + from app.db.models_cluster import ClusterWorker + + now = utcnow() + async with SessionLocal() as session: + rows = (await session.execute(select(ClusterWorker).order_by(ClusterWorker.started_at))).scalars().all() + return [{"id": r.id, "host": r.host, "pid": r.pid, "alive": r.lease_until > now, "me": r.id == identity.owner(), + "started_at": r.started_at.isoformat() if r.started_at else None, + "heartbeat_at": r.heartbeat_at.isoformat() if r.heartbeat_at else None} for r in rows] + + +__all__ = ["Liveness", "alive", "heartbeat_s", "lease_s", "listing", "liveness", "prune", "register", "renew", + "unregister"] diff --git a/backend/app/core/bus.py b/backend/app/core/bus.py index a94713a1..732e21d1 100644 --- a/backend/app/core/bus.py +++ b/backend/app/core/bus.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio import contextlib from collections import defaultdict +from collections.abc import Callable from typing import AsyncIterator from app.core.events import RunEventModel @@ -23,12 +24,16 @@ class EventBus: 只负责实时分发;历史事件由 DB 承担,订阅者接入时先读库补齐再接实时流, 这样刷新页面或中途接入都能看到完整时间线。 + + 多进程(app/cluster/relay.py):本进程发布的每一条也交给 tap(转发器据此给别的进程发通知、给在看的进程转流式增量); + 别的进程执行的运行,转发器从库里按 seq 读出来再在本进程发布一遍(relayed=True,不再交给 tap,免得来回转)。 """ def __init__(self) -> None: self._subs: dict[str, set[_Subscriber]] = defaultdict(set) self._seq: dict[str, int] = defaultdict(int) self._lock = asyncio.Lock() + self.tap: Callable[[RunEventModel], None] | None = None def next_seq(self, run_id: str) -> int: self._seq[run_id] += 1 @@ -38,7 +43,12 @@ class EventBus: """恢复一个已存在的 run 时,把序号接上,保证前端排序不乱。""" self._seq[run_id] = max(self._seq[run_id], value) - async def publish(self, event: RunEventModel) -> None: + async def publish(self, event: RunEventModel, *, relayed: bool = False) -> None: + if not relayed and self.tap is not None: + try: + self.tap(event) + except Exception: # noqa: BLE001 - 转发出错不能拦住本进程的推送 + pass for sub in list(self._subs.get(event.run_id, ())): try: sub.queue.put_nowait(event) diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 2ddb6629..96fd430f 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -59,6 +59,15 @@ class Settings(BaseSettings): db_pool_timeout: int = 30 db_pool_recycle: int = 1800 + # --- 多进程(app/cluster,docs/design/orchestration-round §5 W2-b)--- + # 几个工作进程(scripts/prod.sh、Docker 的入口按它给 uvicorn --workers)。只有应用库是 PostgreSQL 时才能大于 1: + # SQLite 只支持单进程,启动时拒绝(app/cluster/guard.py) + workers: int = 1 + # 每个工作进程在库里登记一行:隔 worker_heartbeat_s 秒续一次租约,租约长 worker_lease_s 秒。进程没了(强杀、断电) + # 租约到期后,别的进程接手它认领的运行、问数据、后台任务(各按各的规矩)。租约要比心跳长几倍:库一时慢了不算进程没了 + worker_lease_s: float = 30.0 + worker_heartbeat_s: float = 5.0 + # --- 前端构建产物(生产部署用)--- # 设了就由服务端一并托管页面(见 app/web.py),页面和接口同源、只起一个进程。 # 不设(开发模式)什么都不变:页面由 Vite 开发服务器提供。相对路径按仓库根目录算 @@ -223,6 +232,20 @@ class Settings(BaseSettings): except ValueError as e: raise ValueError(f"{name}:{e}") from None + @field_validator("workers") + @classmethod + def _valid_workers(cls, value: int) -> int: + if value < 1: + raise ValueError("AGENTLAB_WORKERS 至少是 1") + return value + + @field_validator("worker_heartbeat_s", "worker_lease_s") + @classmethod + def _positive_seconds(cls, value: float, info: Any) -> float: + if value <= 0: + raise ValueError(f"AGENTLAB_{(info.field_name or '').upper()} 要大于 0") + return value + @field_validator("web_dist", mode="before") @classmethod def _blank_is_unset(cls, value: object) -> object: diff --git a/backend/app/core/job_runtime.py b/backend/app/core/job_runtime.py index f1f55b10..e860e0d7 100644 --- a/backend/app/core/job_runtime.py +++ b/backend/app/core/job_runtime.py @@ -27,13 +27,16 @@ 表记 cancelled。**不自动续跑**:复制出来的数据目录(检查脚本、验收副本)启动时不能 自己去调模型、查业务库,界面让人点「重试」接着做。 -进程内登记、进程内的队:服务端是单进程部署,多进程部署时要换成库里的锁。 +**多进程**(docs/design/orchestration-round §5 W2-b,app/cluster):任务表的 owner 是认领它的进程(建任务时缺省就是建它的 +进程),执行在那个进程里;同源队记在库里(lane_tickets),几个进程共用一条;启动整理只接手属主不在了的任务 +(interrupt_on_startup,多进程时当班进程隔一会儿再来一次);停止落在别的进程上时给属主发停止请求(JobManager.stop)。 """ from __future__ import annotations import asyncio import contextlib import logging +import weakref from collections.abc import AsyncIterator, Awaitable, Callable, Iterator from typing import Any @@ -123,117 +126,266 @@ def lane_label(kind: str) -> str: class LaneTicket: - """在一个源的队里的一个位置。kind 是占着它的是什么(登记过的一种,见 register_lane_kind);job_id 是任务的(同步写 - 操作没有)。轮到它时 granted 置位。""" + """在一个源的队里的一个位置(库里 lane_tickets 的一行)。kind 是占着它的是什么(登记过的一种,见 register_lane_kind); + job_id 是任务的(同步写操作没有)。轮到它时 granted 置位。""" - def __init__(self, lanes: SourceLanes, source_id: str, kind: str, job_id: str | None, + def __init__(self, lanes: SourceLanes, row_id: int, source_id: str, kind: str, job_id: str | None, loop: asyncio.AbstractEventLoop) -> None: self._lanes = lanes + self.id = row_id self.source_id = source_id self.kind = kind self.job_id = job_id self.loop = loop self.granted = asyncio.Event() + self.woke = asyncio.Event() self.released = False async def wait(self) -> None: - """等轮到自己。""" - await self.granted.wait() + """等轮到自己:本进程里有人出队时叫醒,PostgreSQL 上别的进程出队时 NOTIFY 叫醒,另隔一会儿自己回库里看一次。""" + from app.core.config import settings - def release(self) -> None: + poll = LANE_POLL_S.get(settings.db_dialect, 5.0) + while not self.granted.is_set(): + self.woke.clear() + got = await self._lanes._advance(self.source_id, me=self) + if got or self.granted.is_set(): + self.granted.set() + return + with contextlib.suppress(asyncio.TimeoutError): + await asyncio.wait_for(self.woke.wait(), timeout=poll) + + async def release(self) -> None: """出队(轮到了、没轮到都行,重复调用无害):排在后面的第一个接着开始。""" - self._lanes._release(self) + await self._lanes._release(self) + + +class LaneHolder: + """队里的一位(holders / LaneBusy 里给出的样子):kind、job_id、属主。""" + + def __init__(self, kind: str, job_id: str | None, owner: str | None) -> None: + self.kind = kind + self.job_id = job_id + self.owner = owner + + def __repr__(self) -> str: + return f"LaneHolder({self.kind!r}, {self.job_id!r})" class LaneBusy(Exception): - """不阻塞地试占时这个源的队里已经有人:holder 是排在最前面(正在做)的那一个。""" + """不阻塞地试占时这个源的队里已经有人:holder 是排在最前面(正在做)的那一位。""" - def __init__(self, holder: LaneTicket) -> None: + def __init__(self, holder: LaneHolder) -> None: super().__init__(holder.kind) self.holder = holder +#: 等轮到自己时隔多久回库里看一次(秒):PostgreSQL 上别的进程出队靠 NOTIFY 叫醒,这是兜底;SQLite 只有本进程 +LANE_POLL_S = {"postgresql": 1.0, "sqlite": 5.0} + + class SourceLanes: - """每个数据源一条先来先到的队。""" + """每个数据源一条先来先到的队,记在库里(lane_tickets),几个进程共用一条(docs/design/orchestration-round §5 W2-b)。 + + 排队 = 插一行(id 自增,先来的号小);轮到谁 = 这个源上没有已经轮到的、活着的那一行时,号最小的那一行改成「轮到了」。改的那 + 一下按源串行:PostgreSQL 上在事务里取这个源的咨询锁,SQLite 只有一个进程、本进程的锁就够。属主不在了的行(进程没了;本进程 + 里登记已经没了——测试里事件循环关了留下的)改的时候顺手删掉,后面的接着轮到。""" def __init__(self) -> None: - self._queues: dict[str, list[LaneTicket]] = {} + self._tickets: dict[int, LaneTicket] = {} + #: 本进程里正在排进某个源的队(插了行、还没登记好 LaneTicket)的个数:这期间属主是本进程、登记里还没有的那一行不能 + #: 当成没人占着删掉 + self._entering: dict[str, int] = {} + self._locks: weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, asyncio.Lock] = weakref.WeakKeyDictionary() + self._listening = False - def _queue(self, source_id: str) -> list[LaneTicket]: - """这个源的队。别的事件循环留下的(测试每个用例一个循环)去掉:那个循环里没人会再出队了。""" - queue = self._queues.get(source_id) or [] + def _local_alive(self, row_id: int) -> bool: + ticket = self._tickets.get(row_id) + if ticket is None or ticket.released: + return False + try: + return ticket.loop is asyncio.get_running_loop() and not ticket.loop.is_closed() + except RuntimeError: + return False + + def _lock(self) -> asyncio.Lock: loop = asyncio.get_running_loop() - alive = [t for t in queue if t.loop is loop and not t.released] - if alive: - self._queues[source_id] = alive - alive[0].granted.set() - else: - self._queues.pop(source_id, None) - return alive + lock = self._locks.get(loop) + if lock is None: + lock = self._locks[loop] = asyncio.Lock() + return lock - def enter(self, source_id: str, kind: str, job_id: str | None = None) -> LaneTicket: + def _listen(self) -> None: + if self._listening: + return + from app.cluster.signal import signals + + signals.on("lane", lambda parts: self._wake(parts[0] if parts else None)) + self._listening = True + + def _wake(self, source_id: str | None) -> None: + for ticket in list(self._tickets.values()): + if (source_id is None or ticket.source_id == source_id) and not ticket.loop.is_closed(): + ticket.loop.call_soon_threadsafe(ticket.woke.set) + + async def _locked(self, session: AsyncSession, source_id: str) -> None: + if session.get_bind().dialect.name == "postgresql": + from sqlalchemy import text + + from app.cluster.mutex import advisory_key + + await session.execute(text("SELECT pg_advisory_xact_lock(:k)"), {"k": advisory_key(f"lane:{source_id}")}) + + async def _rows(self, session: AsyncSession, source_id: str) -> list[Any]: + """这个源队里活着的各行(号小的在前);属主不在了的删掉(不提交)。""" + from app.cluster import state + from app.db.models_cluster import LaneTicket as Row + + rows = list((await session.execute(select(Row).where(Row.source_id == source_id).order_by(Row.id))).scalars()) + live = await state.live_for([r.owner for r in rows]) + if self._entering.get(source_id): + live = live.trusting() + dead = [r.id for r in rows if not live.alive(r.owner, local=self._local_alive(r.id))] + if dead: + from sqlalchemy import delete + + await session.execute(delete(Row).where(Row.id.in_(dead))) + return [r for r in rows if r.id not in dead] + + async def _advance(self, source_id: str, *, me: LaneTicket | None = None) -> bool: + """这个源上没有轮到的就让号最小的轮到。交回 me 是不是轮到了。""" + from app.db.models_cluster import LaneTicket as Row + + granted_now: int | None = None + async with self._lock(), SessionLocal() as session: + await self._locked(session, source_id) + rows = await self._rows(session, source_id) + head = next((r for r in rows if r.granted), None) + if head is None and rows: + head = rows[0] + await session.execute(update(Row).where(Row.id == head.id).values(granted=True) + .execution_options(synchronize_session=False)) + granted_now = head.id + await session.commit() + if granted_now is not None: + ticket = self._tickets.get(granted_now) + if ticket is not None: + ticket.granted.set() + ticket.woke.set() + from app.cluster.signal import signals + + signals.publish("lane", source_id) + return me is not None and head is not None and head.id == me.id + + async def enter(self, source_id: str, kind: str, job_id: str | None = None) -> LaneTicket: """排进队尾,马上返回(不等)。排在最前面的立刻 granted;调用方 await ticket.wait() 等轮到自己。kind 要登记过。""" + from app.cluster import identity + from app.db.models_cluster import LaneTicket as Row + if kind not in _LANE_KINDS: raise ValueError(f"同源队里没有登记「{kind}」这种事,先 register_lane_kind") - queue = self._queue(source_id) - ticket = LaneTicket(self, source_id, kind, job_id, asyncio.get_running_loop()) - queue.append(ticket) - self._queues[source_id] = queue - if len(queue) == 1: + self._listen() + self._entering[source_id] = self._entering.get(source_id, 0) + 1 + try: + async with SessionLocal() as session: + row = Row(source_id=source_id, kind=kind, job_id=job_id, owner=identity.owner(), granted=False, + created_at=utcnow()) + session.add(row) + await session.commit() + row_id = int(row.id) + ticket = LaneTicket(self, row_id, source_id, kind, job_id, asyncio.get_running_loop()) + self._tickets[row_id] = ticket + finally: + left = self._entering.get(source_id, 1) - 1 + if left > 0: + self._entering[source_id] = left + else: + self._entering.pop(source_id, None) + if await self._advance(source_id, me=ticket): ticket.granted.set() return ticket - def try_enter(self, source_id: str, kind: str) -> LaneTicket: - """队空着才占(马上就轮到);有人在队里抛 LaneBusy。""" - queue = self._queue(source_id) - if queue: - raise LaneBusy(queue[0]) - return self.enter(source_id, kind) + async def try_enter(self, source_id: str, kind: str) -> LaneTicket: + """队空着才占(马上就轮到);有人在队里抛 LaneBusy。查和占在同一个事务里、按源串行。""" + from app.cluster import identity + from app.db.models_cluster import LaneTicket as Row - def holders(self, source_id: str) -> list[LaneTicket]: + if kind not in _LANE_KINDS: + raise ValueError(f"同源队里没有登记「{kind}」这种事,先 register_lane_kind") + self._listen() + async with self._lock(), SessionLocal() as session: + await self._locked(session, source_id) + rows = await self._rows(session, source_id) + if rows: + head = next((r for r in rows if r.granted), rows[0]) + await session.commit() + raise LaneBusy(LaneHolder(head.kind, head.job_id, head.owner)) + row = Row(source_id=source_id, kind=kind, job_id=None, owner=identity.owner(), granted=True, + created_at=utcnow()) + session.add(row) + await session.commit() + row_id = int(row.id) + ticket = LaneTicket(self, row_id, source_id, kind, None, asyncio.get_running_loop()) + ticket.granted.set() + self._tickets[row_id] = ticket + return ticket + + async def holders(self, source_id: str) -> list[LaneHolder]: """队里的各位,排在最前面(正在做)的在前。""" - return list(self._queue(source_id)) + async with SessionLocal() as session: + rows = await self._rows(session, source_id) + await session.commit() + rows.sort(key=lambda r: (not r.granted, r.id)) + return [LaneHolder(r.kind, r.job_id, r.owner) for r in rows] - def ahead(self, source_id: str, job_id: str) -> list[str] | None: + async def ahead(self, source_id: str, job_id: str) -> list[str] | None: """排在这个任务前面的各位是什么(kind,正在做的在前);这个任务不在队里(刚建、后台还没排进来)返回 None。""" - queue = self._queue(source_id) - for i, ticket in enumerate(queue): - if ticket.job_id == job_id: - return [t.kind for t in queue[:i]] + holders = await self.holders(source_id) + for i, holder in enumerate(holders): + if holder.job_id == job_id: + return [h.kind for h in holders[:i]] return None - def _release(self, ticket: LaneTicket) -> None: + async def _release(self, ticket: LaneTicket) -> None: + from sqlalchemy import delete + + from app.db.models_cluster import LaneTicket as Row + if ticket.released: return ticket.released = True - queue = self._queues.get(ticket.source_id) - if not queue or ticket not in queue: - return - head = queue[0] is ticket - queue.remove(ticket) - if not queue: - self._queues.pop(ticket.source_id, None) - elif head: - queue[0].granted.set() + self._tickets.pop(ticket.id, None) + + async def go() -> None: + async with SessionLocal() as session: + await session.execute(delete(Row).where(Row.id == ticket.id)) + await session.commit() + await self._advance(ticket.source_id) + + try: + await asyncio.shield(go()) + except Exception: # noqa: BLE001 - 删不掉的那一行下次排队时按「本进程里已经没人占着」收回 + logger.exception("同源队里的一位没能出队(源 %s),下次排队时收回", ticket.source_id) + self._wake(ticket.source_id) @contextlib.asynccontextmanager async def hold(self, source_id: str, kind: str, job_id: str | None = None) -> AsyncIterator[LaneTicket]: """阻塞地占:排队,轮到了再进去,出来时出队。""" - ticket = self.enter(source_id, kind, job_id) + ticket = await self.enter(source_id, kind, job_id) try: await ticket.wait() yield ticket finally: - ticket.release() + await ticket.release() - @contextlib.contextmanager - def try_hold(self, source_id: str, kind: str) -> Iterator[LaneTicket]: + @contextlib.asynccontextmanager + async def try_hold(self, source_id: str, kind: str) -> AsyncIterator[LaneTicket]: """不阻塞地试占(同步起草、导入目录):占不到抛 LaneBusy;占到了,执行期间新建的任务排在后面。""" - ticket = self.try_enter(source_id, kind) + ticket = await self.try_enter(source_id, kind) try: yield ticket finally: - ticket.release() + await ticket.release() lanes = SourceLanes() @@ -287,26 +439,48 @@ async def cancel_queued(session: AsyncSession, job_model: Any, task_model: Any, async def claim(session: AsyncSession, job_model: Any, job_id: str) -> bool: - """认领:排队中 → 进行中。开始之前就被停止了返回 False。""" + """认领:排队中 → 进行中,属主记成本进程(多进程:执行就在这里)。开始之前就被停止了返回 False。""" + from app.db.base import current_owner + + values: dict[str, Any] = {"status": "running", "started_at": utcnow()} + if hasattr(job_model, "owner"): + values["owner"] = current_owner() result = await session.execute( update(job_model).where(job_model.id == job_id, job_model.status == "queued") - .values(status="running", started_at=utcnow()).execution_options(synchronize_session=False)) + .values(**values).execution_options(synchronize_session=False)) await session.commit() return result.rowcount == 1 async def interrupt_on_startup(job_model: Any, close: Callable[[AsyncSession, str], Awaitable[None]], *, - running: Callable[[str], bool]) -> int: - """服务启动时:排队、进行中、而且不在这个进程里跑的任务逐个 close(各自记成 interrupted、没结束的表记 cancelled)。 - 不自动续跑。返回整理了几个。""" + running: Callable[[str], bool], live: Any = None) -> int: + """接手属主不在了的任务(服务启动时;多进程时当班进程隔一会儿再来一次,见 app/cluster/reaper.py):排队、进行中、属主 + 不在了(进程没了,或者就是本进程、却不在本进程里跑)的任务认领过来、逐个 close(各自记成 interrupted、没结束的表记 + cancelled)。别的活着的进程在跑的一概不碰。不自动续跑。返回整理了几个。 + + live:判断属主在不在的那一份(cluster.workers.Liveness),不给就当场查库(启动时的口径:属主是自己的只看本进程)。""" + from app.cluster import claims, state + + live = live if live is not None else await state.current(fresh=True) + ids = [job_id for job_id, _ in await claims.orphans(job_model, live, running=running, statuses=ACTIVE)] async with SessionLocal() as session: - ids = [job_id for job_id in (await session.execute( - select(job_model.id).where(job_model.status.in_(ACTIVE)))).scalars() if not running(job_id)] for job_id in ids: await close(session, job_id) return len(ids) +async def stale(rows: Any, *, running: Callable[[str], bool]) -> list[Any]: + """读接口当场整理用:rows 里记着排队、进行中,属主却不在了的(属主是本进程的看 running——同以前「这个进程里却没在跑」, + 别的进程看租约)。""" + from app.cluster import state + + rows = [r for r in rows if r.status in ACTIVE] + if not rows: + return [] + live = await state.live_for([getattr(r, "owner", None) for r in rows]) + return [r for r in rows if not live.alive(getattr(r, "owner", None), local=running(r.id))] + + # ========================================================================== # 在后台执行 # ========================================================================== @@ -351,7 +525,7 @@ class JobRun: finally: # 收尾写完才出队:排在后面的任务开始时,这个任务已经记成结束了 if self.ticket is not None: - self.ticket.release() + await self.ticket.release() def lane_of(self, job: Any) -> str | None: """这个任务排在哪条队里:缺省是任务的数据源(job.source_id)。不占源的任务(以后的评测:开始时拍目录快照,不挡 @@ -365,7 +539,7 @@ class JobRun: source_id = self.lane_of(job) if job is not None and job.status == "queued" else None if source_id is None: return - self.ticket = lanes.enter(source_id, self.kind, self.job_id) + self.ticket = await lanes.enter(source_id, self.kind, self.job_id) await self.ticket.wait() async def _run(self) -> None: @@ -382,14 +556,29 @@ class JobRun: class JobManager: - """进程内的一种后台任务:一个 asyncio 任务一件事,停的时候给宽限。factory 按任务编号造一次执行(JobRun);label 是 - 日志里的叫法(「数据目录起草任务」),name 是 asyncio 任务名的前缀。""" + """一种后台任务:一个 asyncio 任务一件事,停的时候给宽限。factory 按任务编号造一次执行(JobRun);label 是日志里的叫法 + (「数据目录起草任务」),name 是 asyncio 任务名的前缀,也是跨进程停止请求里的 target。 + + 执行在认领它的那个进程里(任务表的 owner 列,建任务时缺省就是建它的进程)。停止落在别的进程上时(多进程,见 + app/cluster):stop 给属主发一条停止请求,属主照本进程的规矩停、收尾,这里等它收完(最多 grace 秒);属主不在了的交给 + 调用方按「不在这个进程里跑」处理(同以前「记录对不上:直接记停止」)。""" def __init__(self, factory: Callable[[str], JobRun], *, label: str, name: str) -> None: self._runs: dict[str, JobRun] = {} self._factory = factory self.label = label self.name = name + from app.cluster import requests + + requests.handle("stop", name)(self._stop_requested) + + def model(self) -> Any: + """任务表(停止时读属主用):缺省取 factory 上的 model。""" + return getattr(self._factory, "model", None) + + async def _stop_requested(self, job_id: str, actor: str | None, payload: dict[str, Any]) -> None: + await self.stop(job_id, str(payload.get("reason") or "user"), grace=float(payload.get("grace") or STOP_GRACE_S), + remote=False) def _alive(self, job_id: str) -> JobRun | None: run = self._runs.get(job_id) @@ -418,17 +607,40 @@ class JobManager: def running(self, job_id: str) -> bool: return self._alive(job_id) is not None - async def stop(self, job_id: str, reason: str, *, grace: float = STOP_GRACE_S) -> bool: - """停掉这个进程里在跑(或在排队等)的任务,等它收尾(最多 grace 秒)。不在这个进程里返回 False。""" + async def stop(self, job_id: str, reason: str, *, grace: float = STOP_GRACE_S, remote: bool = True) -> bool: + """停掉这个任务,等它收尾(最多 grace 秒)。在本进程里跑的直接停;remote 时在别的活着的进程里跑的给它发停止请求、 + 等它收完。都不是(不在跑、属主不在了)返回 False。""" run = self._alive(job_id) if run is None: - return False + return await self._stop_remote(job_id, reason, grace) if remote else False if run.stop_reason is None: run.stop_reason = reason run.task.cancel() # type: ignore[union-attr] await asyncio.wait([run.task], timeout=grace) # type: ignore[list-item] return True + async def _stop_remote(self, job_id: str, reason: str, grace: float) -> bool: + from app.cluster import identity, requests, workers + + model = self.model() + if model is None or not hasattr(model, "owner"): + return False + async with SessionLocal() as session: + row = await session.get(model, job_id) + owner, status = (row.owner, row.status) if row is not None else (None, None) + if status not in ACTIVE or not owner or owner == identity.owner() or not await workers.alive(owner): + return False + await requests.send(owner, "stop", self.name, job_id, payload={"reason": reason, "grace": grace}) + loop = asyncio.get_running_loop() + deadline = loop.time() + grace + workers.heartbeat_s() + while loop.time() < deadline: + await asyncio.sleep(0.1) + async with SessionLocal() as session: + status = await session.scalar(select(model.status).where(model.id == job_id)) + if status not in ACTIVE: + break + return True + async def wait(self, job_id: str, timeout: float | None = None) -> None: """等这个任务做完(测试、调试用)。不在跑直接返回。""" run = self._alive(job_id) @@ -449,4 +661,4 @@ class JobManager: __all__ = ["ACTIVE", "FINISHED", "JobFailed", "JobManager", "JobRun", "LaneBusy", "LaneTicket", "OPEN", "RETRYABLE", "SHUTDOWN_GRACE_S", "STOP_GRACE_S", "SerialWriter", "SourceLanes", "TASK_DONE", "TASK_STATUSES", "cancel_queued", - "claim", "close_job", "interrupt_on_startup", "lane_kinds", "lane_label", "lanes", "register_lane_kind"] + "claim", "close_job", "interrupt_on_startup", "lane_kinds", "lane_label", "lanes", "register_lane_kind", "stale"] diff --git a/backend/app/data/catalog_jobs.py b/backend/app/data/catalog_jobs.py index 415b4e5d..fbff46b3 100644 --- a/backend/app/data/catalog_jobs.py +++ b/backend/app/data/catalog_jobs.py @@ -246,7 +246,7 @@ async def waiting_for(session: AsyncSession, job: CatalogDraftJob) -> str | None 任务。""" if job.status != "queued": return None - ahead = runtime.lanes.ahead(job.source_id, job.id) + ahead = await runtime.lanes.ahead(job.source_id, job.id) if ahead is not None: if "profile" in ahead: return "profile" @@ -347,19 +347,22 @@ def import_busy_message(row: Any) -> str: return f"数据源「{getattr(row, 'name', '')}」正在导入数据目录,请等导入完成后再试" -#: 正在同步起草(POST /catalog/draft)的数据源。进程内登记:服务端是单进程部署,多进程部署时要换成库里的锁 +#: 本进程里正在同步起草(POST /catalog/draft)的数据源:先登记、再查库(见 sync_draft)。别的进程上的同步起草看同源队里 +#: kind=sync_draft 的那一位(队记在库里,多进程共用) _SYNC: set[str] = set() -def sync_drafting(source_id: str) -> bool: - """这个源正在同步起草。""" - return source_id in _SYNC +async def sync_drafting(source_id: str) -> bool: + """这个源正在同步起草(本进程里,或者别的进程占着同源队做同步起草)。""" + if source_id in _SYNC: + return True + return any(h.kind == "sync_draft" for h in await runtime.lanes.holders(source_id)) async def ensure_draft_free(session: AsyncSession, row: Any, *, then: str = THEN_DRAFT) -> None: """同种任务互斥:这个源没有在起草——没有排队或进行中的起草任务,也没有在同步起草。有就抛 DraftBusy。建起草任务、 重试之前调它。剖析任务不算:同源有剖析任务时起草任务照建,排在它后面。then 是说法最后一句要做的事。""" - if row.id in _SYNC: + if await sync_drafting(row.id): raise DraftBusy(sync_busy_message(row, then)) if await active_job(session, row.id) is not None: raise DraftBusy(busy_message(row, then)) @@ -375,7 +378,7 @@ async def ensure_idle(session: AsyncSession, row: Any, kind: str = "import") -> await catalog_profile_jobs.ensure_free(session, row, then=_then(kind, "profile")) -def _lane_busy(row: Any, holder: runtime.LaneTicket, kind: str) -> Exception: +def _lane_busy(row: Any, holder: runtime.LaneHolder, kind: str) -> Exception: """库里查过没有任务、这个源的队却被占着(同步起草、导入目录正在做,或者一个任务刚好在收尾):按占着的是谁说,最后 一句按这次要做的事(kind)说。""" if holder.kind == "profile": @@ -400,13 +403,13 @@ async def _sync_write(row: Any, kind: str, check: Callable[[], Awaitable[None]]) 再占队——别的同步写操作正在做时占不到,同样抛忙,什么都不做。占到了,执行期间新建的任务排在它后面,做完才开始。""" await check() try: - ticket = runtime.lanes.try_enter(row.id, kind) + ticket = await runtime.lanes.try_enter(row.id, kind) except runtime.LaneBusy as e: raise _lane_busy(row, e.holder, kind) from e try: yield finally: - ticket.release() + await ticket.release() async def _ensure_idle_but_sync(session: AsyncSession, row: Any) -> None: @@ -423,7 +426,7 @@ async def sync_draft(session: AsyncSession, row: Any) -> AsyncIterator[None]: """同步起草期间登记这个源、占着它的队:同一个源建起草任务、重试、导入、再开一次同步起草都抛 DraftBusy,期间新建的 剖析任务排在它后面;有排队或进行中的起草任务时这里抛 DraftBusy,有剖析任务时抛 ProfileBusy。先登记、再查库:查库要 让出事件循环,先登记才不会和同时建的任务互相看不见(建任务那边写库之后也再看一次,见 create_job)。""" - if row.id in _SYNC: + if await sync_drafting(row.id): raise DraftBusy(sync_busy_message(row)) _SYNC.add(row.id) try: @@ -484,7 +487,7 @@ async def create_job(session: AsyncSession, row: Any, source: Any, *, tables: li # 部分唯一索引:另一个请求同时给这个源建了任务 await session.rollback() raise DraftBusy(busy_message(row)) from e - if row.id in _SYNC: + if await sync_drafting(row.id): # 写库的时候同一个源开始了同步起草(它查库时还没看到这个任务):这个任务不要了 await session.execute(delete(CatalogDraftTask).where(CatalogDraftTask.job_id == job.id)) await session.execute(delete(CatalogDraftJob).where(CatalogDraftJob.id == job.id)) @@ -541,12 +544,12 @@ async def _close(session: AsyncSession, job_id: str, job_status: str, job_error: task_error) -async def startup() -> int: +async def startup(*, live: Any = None) -> int: """服务启动时:排队、进行中的任务标成 interrupted,没结束的表记 cancelled、写明服务重启。不自动续跑。返回标了几个。""" async def interrupt(session: AsyncSession, job_id: str) -> None: await _close(session, job_id, "interrupted", INTERRUPTED_ERROR, "cancelled", INTERRUPTED_ERROR) - n = await runtime.interrupt_on_startup(CatalogDraftJob, interrupt, running=manager.running) + n = await runtime.interrupt_on_startup(CatalogDraftJob, interrupt, running=manager.running, live=live) if n: logger.warning("服务重启:%d 个数据目录起草任务没有做完,已标为中断,可在数据目录页重试", n) return n diff --git a/backend/app/data/catalog_profile_jobs.py b/backend/app/data/catalog_profile_jobs.py index 810c3758..e3f7ec15 100644 --- a/backend/app/data/catalog_profile_jobs.py +++ b/backend/app/data/catalog_profile_jobs.py @@ -279,7 +279,7 @@ async def waiting_for(session: AsyncSession, job: CatalogProfileJob) -> str | No 任务,或者正在同步起草。""" if job.status != "queued": return None - ahead = runtime.lanes.ahead(job.source_id, job.id) + ahead = await runtime.lanes.ahead(job.source_id, job.id) if ahead is not None: if any(kind in ("draft", "sync_draft") for kind in ahead): return "draft" @@ -287,7 +287,7 @@ async def waiting_for(session: AsyncSession, job: CatalogProfileJob) -> str | No return "introspect" if "introspect" in ahead else None from app.data import catalog_jobs - if catalog_jobs.sync_drafting(job.source_id): + if await catalog_jobs.sync_drafting(job.source_id): return "draft" earlier = (await session.execute( select(CatalogDraftJob.id).where(CatalogDraftJob.source_id == job.source_id, @@ -690,7 +690,7 @@ async def _behind_draft(session: AsyncSession, source_id: str) -> bool: """这个源有排队、进行中的起草任务,或者正在同步起草:新建的剖析任务要排在它后面。""" from app.data import catalog_jobs - return catalog_jobs.sync_drafting(source_id) or await catalog_jobs.active_job(session, source_id) is not None + return await catalog_jobs.sync_drafting(source_id) or await catalog_jobs.active_job(session, source_id) is not None async def nothing_reason(session: AsyncSession, row: Any, source: Any, scope: str) -> str | None: @@ -859,13 +859,13 @@ async def cancel(session: AsyncSession, job: CatalogProfileJob) -> None: await _close(session, job_id, "cancelled", None, "cancelled", None) -async def startup() -> int: +async def startup(*, live: Any = None) -> int: """服务启动时:排队、进行中的剖析任务标成 interrupted,没结束的表记 cancelled、写明服务重启,不自动续跑;再把剖析设置的 旧键迁移掉。返回标了几个任务。""" async def interrupt(session: AsyncSession, job_id: str) -> None: await _close(session, job_id, "interrupted", INTERRUPTED_ERROR, "cancelled", INTERRUPTED_ERROR) - n = await runtime.interrupt_on_startup(CatalogProfileJob, interrupt, running=manager.running) + n = await runtime.interrupt_on_startup(CatalogProfileJob, interrupt, running=manager.running, live=live) if n: logger.warning("服务重启:%d 个数据剖析任务没有做完,已标为中断,可在数据目录页重试", n) async with SessionLocal() as session: diff --git a/backend/app/data/engine.py b/backend/app/data/engine.py index 417302ad..a5db6903 100644 --- a/backend/app/data/engine.py +++ b/backend/app/data/engine.py @@ -646,8 +646,13 @@ class EngineCache: if self._verifying.get(token) is done: del self._verifying[token] - async def invalidate(self, source_id: str) -> None: + async def invalidate(self, source_id: str, *, broadcast: bool = True) -> None: sid = str(source_id) + if broadcast: + # 多进程:别的进程手上这个源的连接池也要扔掉(换了地址、口令、只读开关),见 app/cluster/caches.py + from app.cluster import caches + + await caches.broadcast("datasource_engine", sid) # 正在进行的核对也撇开:失效之后来的请求重新核对,不搭失效之前那一次的便车 for token in [t for t in self._verifying if _source_of(t[0]) == sid]: del self._verifying[token] diff --git a/backend/app/data/introspect_jobs.py b/backend/app/data/introspect_jobs.py index 0f730765..974f4e96 100644 --- a/backend/app/data/introspect_jobs.py +++ b/backend/app/data/introspect_jobs.py @@ -132,7 +132,7 @@ async def waiting_for(session: AsyncSession, job: IntrospectJob) -> str | None: 起草);任务刚建、后台还没排进队时看库:同源有比它先建的、排队或进行中的起草、剖析任务。""" if job.status != "queued": return None - ahead = runtime.lanes.ahead(job.source_id, job.id) + ahead = await runtime.lanes.ahead(job.source_id, job.id) if ahead is not None: return ahead[0] if ahead else None from app.db.models import CatalogDraftJob, CatalogProfileJob @@ -512,12 +512,12 @@ async def cancel(session: AsyncSession, job: IntrospectJob) -> None: await _close(session, job_id, "cancelled", None, STOPPED_ERROR) -async def startup() -> int: +async def startup(*, live: Any = None) -> int: """服务启动时:排队、进行中的探查任务标成 interrupted,没读的表 cancelled,读完的物化进去;不自动续跑。""" async def interrupt(session: AsyncSession, job_id: str) -> None: await _close(session, job_id, "interrupted", INTERRUPTED_ERROR, INTERRUPTED_ERROR) - n = await runtime.interrupt_on_startup(IntrospectJob, interrupt, running=manager.running) + n = await runtime.interrupt_on_startup(IntrospectJob, interrupt, running=manager.running, live=live) if n: logger.warning("服务重启:%d 个探查任务没有做完,已标为中断,可在数据源卡片上重新探查", n) return n diff --git a/backend/app/data/source_maintenance.py b/backend/app/data/source_maintenance.py index aa8597bf..f9c3c7a3 100644 --- a/backend/app/data/source_maintenance.py +++ b/backend/app/data/source_maintenance.py @@ -121,7 +121,7 @@ async def _entered(source_id: str, job_id: str) -> None: loop = asyncio.get_running_loop() deadline = loop.time() + LANE_WAIT_S while loop.time() < deadline: - if runtime.lanes.ahead(source_id, job_id) is not None or not introspect_jobs.manager.running(job_id): + if await runtime.lanes.ahead(source_id, job_id) is not None or not introspect_jobs.manager.running(job_id): return await asyncio.sleep(0.01) logger.warning("定时维护:探查任务 %s 过了 %.0f 秒还没进同源队,剖析任务照建", job_id, LANE_WAIT_S) diff --git a/backend/app/data/source_objects.py b/backend/app/data/source_objects.py index 65d78ee5..5dfcab0e 100644 --- a/backend/app/data/source_objects.py +++ b/backend/app/data/source_objects.py @@ -405,20 +405,13 @@ def target_schema(row: Any) -> str | None: return index_schema(row) or None -_locks: "weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, dict[str, asyncio.Lock]]" = weakref.WeakKeyDictionary() +def source_lock(source_id: str) -> Any: + """写一个源的对象索引和 schema_cache 时持的锁:后台探查的开头和收尾、按需加入、加入范围都持它。跨进程的锁 + (app/cluster/mutex:本进程按事件循环各一把 asyncio 锁,PostgreSQL 时另取咨询锁):多进程时按需探查、加入范围也是 + 一个接一个,schema_cache、结构版本不会被两个进程各写一半。""" + from app.cluster import mutex - -def source_lock(source_id: str) -> asyncio.Lock: - """写一个源的对象索引和 schema_cache 时持的锁(进程内,按事件循环各一份:asyncio.Lock 不能跨循环用)。后台探查的开头 - 和收尾、按需加入、加入范围都持它。""" - loop = asyncio.get_running_loop() - by_source = _locks.get(loop) - if by_source is None: - by_source = _locks[loop] = {} - lock = by_source.get(source_id) - if lock is None: - lock = by_source[source_id] = asyncio.Lock() - return lock + return mutex.lock(f"source-objects:{source_id}") def _scope_filter(source_id: str, schema: str) -> tuple[Any, ...]: diff --git a/backend/app/data/table_versions.py b/backend/app/data/table_versions.py index 39230169..e31d8f37 100644 --- a/backend/app/data/table_versions.py +++ b/backend/app/data/table_versions.py @@ -681,21 +681,17 @@ async def resolve_source( # 锁:发布、回收、启动清理彼此串行 # -------------------------------------------------------------------------- -_locks: "weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, asyncio.Lock]" = weakref.WeakKeyDictionary() - - -def store_lock() -> asyncio.Lock: - """版本存储的写锁,按事件循环各一把。不可重入:持着它时别再调会自己取锁的函数。 +def store_lock() -> Any: + """版本存储的写锁(app/cluster/mutex:本进程一把 asyncio 锁,按事件循环各一把;应用库是 PostgreSQL 时另取一把会话级咨询锁, + 几个进程之间也串行)。不可重入:持着它时别再调会自己取锁的函数。用法同 asyncio.Lock(async with)。 回收要先看「哪些文件有记录」再删:发布正处在「文件已 link、记录未提交」之间时,回收 会把它当孤儿删掉;复用一个刚被判为无人引用的构建,也会撞上正在删它的回收。上传不频繁, 全部串行最省心。按循环分开是因为 asyncio.Lock 一旦在某个循环里等过,就不能换循环用。 """ - loop = asyncio.get_running_loop() - lock = _locks.get(loop) - if lock is None: - lock = _locks[loop] = asyncio.Lock() - return lock + from app.cluster import mutex + + return mutex.lock("upload-store") _store_lock = store_lock @@ -747,6 +743,10 @@ def acquire_store_guard() -> bool: 同一进程里重复调用是幂等的;数据目录换了就放掉旧的、对新文件重新加锁。上次没拿到的,再调一次会重试 (占着它的进程退出后就能拿到)。没有 fcntl 的平台照常可写,只记一条警告。 """ + if settings.db_dialect == "postgresql": + # 应用库是 PostgreSQL 时可以多进程:发布、回收、启动清理之间靠跨进程的锁(store_lock 的咨询锁)互斥,不再靠文件锁 + # 独占数据目录(flock 在网络文件系统上也靠不住) + return True path = _guard_path() if _guard.path == path and _guard.fd is not None: return True @@ -776,7 +776,10 @@ def acquire_store_guard() -> bool: def store_writable() -> bool: - """本进程能不能写版本存储。没对当前数据目录取过守卫(没调过 startup)的按能写处理:守卫只在启动时取。""" + """本进程能不能写版本存储。没对当前数据目录取过守卫(没调过 startup)的按能写处理:守卫只在启动时取。PostgreSQL 时总能写 + (见 acquire_store_guard)。""" + if settings.db_dialect == "postgresql": + return True return _guard.writable or _guard.path != _guard_path() @@ -2445,16 +2448,27 @@ def _same_path(a: str | os.PathLike[str], b: str | os.PathLike[str]) -> bool: return False -def _sweep_tmp() -> int: - """删掉上次崩溃留下的 *.tmp-* 半成品(库和原件都是先写临时文件再 link)。""" +#: 多进程时只清多久以前的半成品(秒):别的进程正在解析、写的临时文件不在锁里(解析在锁外做),新的不能删 +TMP_STALE_S = 6 * 3600.0 + + +def _sweep_tmp(min_age_s: float = 0.0) -> int: + """删掉上次崩溃留下的 *.tmp-* 半成品(库和原件都是先写临时文件再 link)。min_age_s:只删改动时间早于这么多秒以前的 + (多进程时别的进程可能正在写,见 startup)。""" + import time as _time + removed = 0 + floor = _time.time() - min_age_s for root in (tables_root(), raw_store.raw_root()): if not root.is_dir(): continue for path in root.rglob(f"*{_TMP}*"): - if path.is_file(): - path.unlink(missing_ok=True) - removed += 1 + try: + if path.is_file() and (not min_age_s or path.stat().st_mtime < floor): + path.unlink(missing_ok=True) + removed += 1 + except OSError: + continue return removed @@ -2677,7 +2691,9 @@ async def startup(session: AsyncSession) -> None: logger.error("上传表格的版本存储被另一个进程占用,本进程跳过启动清理、迁移和回收") return async with _store_lock(): - swept = await asyncio.to_thread(_sweep_tmp) + # 半成品:单进程(SQLite,守卫保证只有本进程)启动时一个都不会有人在写,全清;PostgreSQL 可以多进程,别的进程正在解析的 + # 临时文件写在锁外,只清 TMP_STALE_S 以前的 + swept = await asyncio.to_thread(_sweep_tmp, TMP_STALE_S if settings.db_dialect == "postgresql" else 0.0) if swept: logger.info("清理了 %d 个上次未完成的上传临时文件", swept) migrated = await _migrate_legacy(session) diff --git a/backend/app/db/base.py b/backend/app/db/base.py index 7e741218..a018ff05 100644 --- a/backend/app/db/base.py +++ b/backend/app/db/base.py @@ -34,6 +34,14 @@ def new_id() -> str: return uuid.uuid4().hex +def current_owner() -> str: + """认领一行时记的属主:这个工作进程(app/cluster/identity.OWNER)。运行、问数据的一轮、各种后台任务表的 owner 列缺省取它: + 建这一行的进程就是认领它的进程,和这一行同一个事务写下,不会有「行建好了、还没人认领」的空档(多进程,W2-b)。""" + from app.cluster.identity import owner + + return owner() + + class UTCDateTime(TypeDecorator): """存 UTC、读出来也是带时区的 UTC。 @@ -362,6 +370,19 @@ _COLUMN_MIGRATIONS: list[tuple[str, str, str]] = [ ("catalog_profile_tasks", "snapshot_id", "VARCHAR(32)"), # 第 3 波:发布时这一版引用的表的结构版本 ("workflow_versions", "schema_versions", "JSON"), + # 编排完善第 2 波 W2-b(多进程协调):认领这一行的工作进程(app/cluster)。存量行为空:升级前的进程认领的,按「属主不在了」 + # 处理(升级要停掉全部进程再起) + ("runs", "owner", "VARCHAR(120)"), + ("chat_turn_jobs", "owner", "VARCHAR(120)"), + ("catalog_draft_jobs", "owner", "VARCHAR(120)"), + ("catalog_profile_jobs", "owner", "VARCHAR(120)"), + ("introspect_jobs", "owner", "VARCHAR(120)"), + ("eval_runs", "owner", "VARCHAR(120)"), + ("eval_catalog_snapshots", "owner", "VARCHAR(120)"), + ("documents", "owner", "VARCHAR(120)"), + ("publish_plans", "owner", "VARCHAR(120)"), + # 问数据:等不及的那次复核还在后台跑(实时流在它回来之前不关;多进程时别的进程上的流也看得到) + ("chat_turn_jobs", "late_review_at", "DATETIME"), ] #: 迁移补上的列要建的索引:(索引名, 表, 列或多列)。create_all 不碰已有的表,表上新列的索引也就不会建。 diff --git a/backend/app/db/models.py b/backend/app/db/models.py index f10efdea..b010432a 100644 --- a/backend/app/db/models.py +++ b/backend/app/db/models.py @@ -8,7 +8,7 @@ from sqlalchemy import ( ) from sqlalchemy.orm import Mapped, mapped_column, relationship -from app.db.base import Base, TimestampMixin, UTCDateTime, new_id, utcnow +from app.db.base import Base, current_owner, TimestampMixin, UTCDateTime, new_id, utcnow # -------------------------------------------------------------------------- # 设置:模型供应商 / 用户偏好 / MCP / 自定义工具 @@ -370,6 +370,8 @@ class CatalogDraftJob(Base, TimestampMixin): source_id: Mapped[str] = mapped_column(ForeignKey("data_sources.id", ondelete="CASCADE")) # queued | running | succeeded | partial | failed | cancelled | interrupted status: Mapped[str] = mapped_column(String(20), default="queued") + #: 认领这一行的工作进程(app/cluster;缺省是建它的进程)。多进程时启动整理、接手只碰属主不在了的行 + owner: Mapped[str | None] = mapped_column(String(120), default=current_owner) use_model: Mapped[bool] = mapped_column(default=False) # 实际用的模型 id(用模型时开始起草后写上) model: Mapped[str | None] = mapped_column(String(200), default=None) @@ -459,6 +461,8 @@ class CatalogProfileJob(Base, TimestampMixin): source_id: Mapped[str] = mapped_column(ForeignKey("data_sources.id", ondelete="CASCADE")) # queued | running | succeeded | partial | failed | cancelled | interrupted status: Mapped[str] = mapped_column(String(20), default="queued") + #: 认领这一行的工作进程(app/cluster;缺省是建它的进程)。多进程时启动整理、接手只碰属主不在了的行 + owner: Mapped[str | None] = mapped_column(String(120), default=current_owner) # selected 指定的表 | needed 需要剖析的表 | all 全部表 scope: Mapped[str] = mapped_column(String(20), default="selected") # 建任务时的剖析设置快照(catalog_profile.ProfileSettings.to_dict) @@ -657,6 +661,8 @@ class Run(Base, TimestampMixin): thread_id: Mapped[str] = mapped_column(String(64), index=True, default=new_id) # queued | running | interrupted | succeeded | failed | cancelled status: Mapped[str] = mapped_column(String(20), default="queued", index=True) + #: 认领这一行的工作进程(app/cluster;缺省是建它的进程)。多进程时启动整理、接手只碰属主不在了的行 + owner: Mapped[str | None] = mapped_column(String(120), default=current_owner) graph: Mapped[dict[str, Any]] = mapped_column(default=dict) # 执行时的图快照 input: Mapped[dict[str, Any]] = mapped_column(default=dict) output: Mapped[dict[str, Any]] = mapped_column(default=dict) @@ -811,6 +817,8 @@ class Document(Base, TimestampMixin): # 压在 HTTP 请求里必然超时——而后端其实还在跑,界面显示失败、数据其实成功, # 比直接拒绝还糟。ready | processing | failed status: Mapped[str] = mapped_column(String(20), default="ready") + #: 认领这一行的工作进程(app/cluster;缺省是建它的进程)。多进程时启动整理、接手只碰属主不在了的行 + owner: Mapped[str | None] = mapped_column(String(120), default=current_owner) error: Mapped[str] = mapped_column(Text, default="") chunks: Mapped[list["Chunk"]] = relationship( @@ -1099,3 +1107,6 @@ from app.db.models_blueprints import ( # noqa: E402,F401 from app.db import models_chat, models_eval, models_sources # noqa: E402,F401 # 编排完善第 1 波(docs/design/orchestration-round/00-总方案.md §4):数据事件触发的待派发事件和去抖等待 from app.db import models_triggers # noqa: E402,F401 +# 编排完善第 2 波 W2-b(docs/design/orchestration-round/00-总方案.md §5):多进程协调——工作进程的登记和租约、调度当班、 +# 跨进程的请求、运行的并发名额、同源队、按时间窗的计数 +from app.db import models_cluster # noqa: E402,F401 diff --git a/backend/app/db/models_blueprints.py b/backend/app/db/models_blueprints.py index 5f057620..e31e8fc5 100644 --- a/backend/app/db/models_blueprints.py +++ b/backend/app/db/models_blueprints.py @@ -88,6 +88,8 @@ class PublishPlan(Base, TimestampMixin): summary: Mapped[list[Any]] = mapped_column(default=list) problems: Mapped[list[Any]] = mapped_column(default=list) status: Mapped[str] = mapped_column(String(20), default="planned") + #: 执行这份发布单的工作进程(status=running 时,app/cluster):属主不在了的才按「执行中断」收拾 + owner: Mapped[str | None] = mapped_column(String(120), default=None) class StoredFile(Base): diff --git a/backend/app/db/models_chat.py b/backend/app/db/models_chat.py index f13542f7..600f33ec 100644 --- a/backend/app/db/models_chat.py +++ b/backend/app/db/models_chat.py @@ -24,7 +24,7 @@ from typing import Any from sqlalchemy import Float, ForeignKey, Integer, String, Text, UniqueConstraint from sqlalchemy.orm import Mapped, mapped_column -from app.db.base import Base, TimestampMixin, UTCDateTime, new_id +from app.db.base import Base, current_owner, TimestampMixin, UTCDateTime, new_id #: 这一轮现在在哪一步 STAGES = ("build", "run", "waiting", "review") @@ -40,6 +40,8 @@ class ChatTurnJob(Base, TimestampMixin): conversation_id: Mapped[str] = mapped_column(ForeignKey("conversations.id", ondelete="CASCADE"), index=True) #: queued | running | succeeded | failed | cancelled | interrupted(core/job_runtime 的状态词) status: Mapped[str] = mapped_column(String(20), default="queued", index=True) + #: 认领这一行的工作进程(app/cluster;缺省是建它的进程)。多进程时启动整理、接手只碰属主不在了的行 + owner: Mapped[str | None] = mapped_column(String(120), default=current_owner) stage: Mapped[str | None] = mapped_column(String(20), default=None) #: ask | retry | rerun | max_steps | run | continue | resume(app/chat/job.py 的 MODES) mode: Mapped[str] = mapped_column(String(20), default="ask") @@ -54,6 +56,9 @@ class ChatTurnJob(Base, TimestampMixin): error: Mapped[str | None] = mapped_column(Text, default=None) started_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None) finished_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None) + #: 等不及的那次复核还在后台跑(从什么时候起):实时流在它回来之前不关。多进程时别的进程上的流也按它判断(属主不在了 + #: 就不再等) + late_review_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None) class ChatTurnEvent(Base): diff --git a/backend/app/db/models_cluster.py b/backend/app/db/models_cluster.py new file mode 100644 index 00000000..9555be21 --- /dev/null +++ b/backend/app/db/models_cluster.py @@ -0,0 +1,118 @@ +"""多进程协调用的表(app/cluster,docs/design/orchestration-round/00-总方案.md §5 W2-b)。 + +由 app/db/models.py 末尾整模块导入,create_all 建表(都是新表)。认领用的 owner 列加在各自的表上(runs、chat_turn_jobs、 +各种后台任务表……,见 db/base 的迁移),这里只放协调本身的几张: + +- ClusterWorker:一个工作进程一行(id 是 app/cluster/identity.OWNER:主机 + pid + 启动随机串)。进程活着就隔一拍心跳续租约 + (lease_until);租约过了期就算这个进程没了,它认领的行由别的进程按各自的规矩接手。各行的认领只记 owner,不各自续租: + 租约按进程算,一个进程一拍只写一行,进程一没,它认领的全部一起到期。 +- ClusterLease:全站只许一个进程做的事(调度当班)的租约:name 一行,owner 是当班的进程,epoch 每换一次当班加一。 +- ClusterRequest:发给别的进程的请求(停止它在跑的运行、问数据、后台任务;往它在执行的运行里记一条事件)。owner 是收件的 + 进程:它隔一拍心跳(PostgreSQL 上另有 NOTIFY 叫醒)取走自己的请求照办,办完删掉。 +- RunSlot:运行的并发名额(app/cluster/slots.py):一次运行每段执行要一个名额,按类别(交互、应用、定时、系统)各有全站上限 + 和优先级,按库里的计数发。 +- LaneTicket:同源队(core/job_runtime 的 SourceLanes):同一个数据源上的起草、剖析、探查、同步起草、导入先来先到、一个 + 接一个;id 自增,先来的号小。 +- RateCounter:几个进程共用的计数(访问密钥每分钟限流的令牌桶、登录失败的退避),到期的由当班进程清掉。只在 PostgreSQL + (可以多进程)时用;SQLite 只有一个进程,计数照旧在进程里(app/cluster/counters.py)。 +""" +from __future__ import annotations + +from datetime import datetime +from typing import Any + +from sqlalchemy import Boolean, Float, Index, Integer, String +from sqlalchemy.orm import Mapped, mapped_column + +from app.db.base import Base, UTCDateTime, new_id, utcnow + + +class ClusterWorker(Base): + __tablename__ = "cluster_workers" + + id: Mapped[str] = mapped_column(String(120), primary_key=True) + host: Mapped[str] = mapped_column(String(255), default="") + pid: Mapped[int] = mapped_column(Integer, default=0) + started_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow) + heartbeat_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow) + lease_until: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow, index=True) + #: 版本、数据目录之类给人看的信息 + info: Mapped[dict[str, Any]] = mapped_column(default=dict) + + +class ClusterLease(Base): + __tablename__ = "cluster_leases" + + name: Mapped[str] = mapped_column(String(64), primary_key=True) + owner: Mapped[str | None] = mapped_column(String(120), default=None) + lease_until: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None) + acquired_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None) + epoch: Mapped[int] = mapped_column(Integer, default=0) + + +class ClusterRequest(Base): + __tablename__ = "cluster_requests" + + id: Mapped[str] = mapped_column(String(32), primary_key=True, default=new_id) + #: 收件的进程 + owner: Mapped[str] = mapped_column(String(120), index=True) + #: stop 停止 | note 往运行里记一条事件 + kind: Mapped[str] = mapped_column(String(20)) + #: 对什么:run | chat_turn | 后台任务的种类(draft、profile、introspect、eval、shadow_draft) + target: Mapped[str] = mapped_column(String(40)) + target_id: Mapped[str] = mapped_column(String(64)) + actor: Mapped[str | None] = mapped_column(String(100), default=None) + payload: Mapped[dict[str, Any]] = mapped_column(default=dict) + created_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow) + + +class RunSlot(Base): + __tablename__ = "run_slots" + + run_id: Mapped[str] = mapped_column(String(32), primary_key=True) + #: interactive | app | schedule | system(app/cluster/slots.CLASSES) + klass: Mapped[str] = mapped_column(String(20)) + priority: Mapped[int] = mapped_column(Integer, default=0) + owner: Mapped[str] = mapped_column(String(120), index=True) + granted: Mapped[bool] = mapped_column(Boolean, default=False, index=True) + requested_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow) + granted_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None) + + +class LaneTicket(Base): + __tablename__ = "lane_tickets" + __table_args__ = (Index("ix_lane_tickets_source_id_id", "source_id", "id"),) + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + source_id: Mapped[str] = mapped_column(String(64)) + #: 登记过的一种事(core/job_runtime.register_lane_kind) + kind: Mapped[str] = mapped_column(String(40)) + job_id: Mapped[str | None] = mapped_column(String(64), default=None) + owner: Mapped[str] = mapped_column(String(120), index=True) + granted: Mapped[bool] = mapped_column(Boolean, default=False) + created_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow) + + +class RateCounter(Base): + __tablename__ = "rate_counters" + + key: Mapped[str] = mapped_column(String(255), primary_key=True) + count: Mapped[int] = mapped_column(Integer, default=0) + #: 令牌桶里还剩几个(访问密钥限流)/ 要等到什么时刻(登录失败,Unix 秒) + value: Mapped[float] = mapped_column(Float, default=0.0) + #: 上次补令牌的时刻(Unix 秒) + at: Mapped[float] = mapped_column(Float, default=0.0) + expires_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow, index=True) + + +class CacheEvent(Base): + """「某个进程里缓存着的东西变了」(app/cluster/caches.py):改的进程写一条,别的进程隔一拍心跳(或收到 NOTIFY)按 id + 往后读,照着清掉、重读自己那一份。id 自增;当班进程清掉一个钟头以前的。""" + + __tablename__ = "cache_events" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + name: Mapped[str] = mapped_column(String(64)) + key: Mapped[str | None] = mapped_column(String(255), default=None) + owner: Mapped[str] = mapped_column(String(120)) + created_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow, index=True) diff --git a/backend/app/db/models_eval.py b/backend/app/db/models_eval.py index 9d8e4f63..38d364cf 100644 --- a/backend/app/db/models_eval.py +++ b/backend/app/db/models_eval.py @@ -29,7 +29,7 @@ from typing import Any from sqlalchemy import ForeignKey, Index, Integer, String, Text, UniqueConstraint from sqlalchemy.orm import Mapped, mapped_column, synonym -from app.db.base import Base, TimestampMixin, UTCDateTime, new_id +from app.db.base import Base, current_owner, TimestampMixin, UTCDateTime, new_id #: 题目的种类:ask 问数据题(走问数据同一条路作答)、workflow 钉版本的工作流题 CASE_KINDS = ("ask", "workflow") @@ -128,6 +128,8 @@ class EvalCatalogSnapshot(Base, TimestampMixin): origin: Mapped[str] = mapped_column(String(20), default="live") # queued | running | succeeded | partial | failed | cancelled | interrupted status: Mapped[str] = mapped_column(String(20), default="succeeded") + #: 认领这一行的工作进程(app/cluster;缺省是建它的进程)。多进程时启动整理、接手只碰属主不在了的行 + owner: Mapped[str | None] = mapped_column(String(120), default=current_owner) artifact_id: Mapped[str | None] = mapped_column(String(64), default=None) # 拍了哪些数据源:[{id, name}](名字是拍的那一刻的) sources: Mapped[list[Any]] = mapped_column(default=list) @@ -160,6 +162,8 @@ class EvalRun(Base, TimestampMixin): suite_id: Mapped[str] = mapped_column(ForeignKey("eval_suites.id", ondelete="CASCADE")) # queued | running | succeeded | partial | failed | cancelled | interrupted(core/job_runtime 的状态词) status: Mapped[str] = mapped_column(String(20), default="queued") + #: 认领这一行的工作进程(app/cluster;缺省是建它的进程)。多进程时启动整理、接手只碰属主不在了的行 + owner: Mapped[str | None] = mapped_column(String(120), default=current_owner) trigger: Mapped[str] = mapped_column(String(20), default="manual") # 定时发起的那一条(ScheduleFire.eval_run_id 反过来指向这里) schedule_id: Mapped[str | None] = mapped_column(String(32), default=None) diff --git a/backend/app/db/models_sources.py b/backend/app/db/models_sources.py index 2423fa50..9b23d379 100644 --- a/backend/app/db/models_sources.py +++ b/backend/app/db/models_sources.py @@ -22,7 +22,7 @@ from typing import Any from sqlalchemy import Boolean, ForeignKey, Index, Integer, String, Text, UniqueConstraint, text from sqlalchemy.orm import Mapped, mapped_column -from app.db.base import Base, PortableJSON, UTCDateTime, new_id, utcnow +from app.db.base import Base, current_owner, PortableJSON, UTCDateTime, new_id, utcnow class SourceObject(Base): @@ -100,6 +100,8 @@ class IntrospectJob(Base): source_id: Mapped[str] = mapped_column(ForeignKey("data_sources.id", ondelete="CASCADE")) # queued | running | succeeded | partial | failed | cancelled | interrupted status: Mapped[str] = mapped_column(String(20), default="queued") + #: 认领这一行的工作进程(app/cluster;缺省是建它的进程)。多进程时启动整理、接手只碰属主不在了的行 + owner: Mapped[str | None] = mapped_column(String(120), default=current_owner) #: manual 人点的 | schedule 定时维护(第 3 波) trigger: Mapped[str] = mapped_column(String(20), default="manual") #: 这次探查的是哪个 schema("" 是默认 schema) diff --git a/backend/app/engine/runner.py b/backend/app/engine/runner.py index b33e472a..9e2dc47a 100644 --- a/backend/app/engine/runner.py +++ b/backend/app/engine/runner.py @@ -15,6 +15,7 @@ from langgraph.errors import GraphRecursionError from langgraph.types import Command from sqlalchemy import case, func, select, update +from app.cluster import identity from app.core.bus import bus from app.core.config import settings from app.core.errors import describe_exception, raw_detail @@ -73,7 +74,8 @@ class RunManager: self._tasks: dict[str, asyncio.Task[None]] = {} self._checkpointer: BaseCheckpointSaver | None = None self._cm: Any = None - self._semaphore = asyncio.Semaphore(settings.max_concurrent_runs) + # 并发上限以前是这里一把进程内的信号量(max_concurrent_runs);现在是库里按类别发的名额(app/cluster/slots.py), + # 几个进程合起来算,评测、召回这些系统运行不再挤占问数据和试跑 #: 正在关停。此时收到的取消来自关停而不是用户,运行要记成可恢复的中断 self._closing = False #: 已经跑完、正在落状态和封存的运行。关停不去打断它们 @@ -105,44 +107,68 @@ class RunManager: update(Run).where(func.coalesce(Run.last_seq, 0) < latest) .values(last_seq=latest, updated_at=Run.updated_at) .execution_options(synchronize_session=False)) - stranded = list((await session.execute( - select(Run.id, Run.last_seq).where(Run.status == "running") - )).all()) - queued = list((await session.execute( - select(Run.id, Run.last_seq).where(Run.status == "queued") - )).all()) - await session.execute( - update(Run) - .where(Run.status == "running") - .values(status="interrupted", error="服务重启,运行已挂起,可从断点继续运行") - ) - await session.execute( - update(Run) - .where(Run.status == "queued") - .values(status="failed", error=QUEUED_AT_RESTART) - ) await session.commit() - # 进程被强杀时连 server_shutdown 日志都来不及发,事件流停在半路。补上这一条, - # 回放它的客户端才知道它是挂起了,而不是还在跑 - for run_id, last_seq in stranded: - bus.set_seq(run_id, last_seq or 0) - await self._emit(run_id, EventType.LOG, data={ - "level": "warn", "code": "server_shutdown", - "message": "服务上次未正常关闭,这次运行已中断;可从断点继续运行", - }) - # 排队中的改成了失败,也补一条 run.failed:以前只改状态,时间线里没有失败原因这一行(05-已知遗留 (h))。 - # 不写结束时间、不封存:它从没开始执行,同以前(定时通知按「没有开始时间」认出它,scheduler/webhook.py) - for run_id, last_seq in queued: - bus.set_seq(run_id, last_seq or 0) - await self._emit(run_id, EventType.RUN_FAILED, data={"error": QUEUED_AT_RESTART}) + # 上次进程被强杀时留下的记录:只碰属主不在了的(多进程时别的活着的进程在跑的一概不碰,见 reap) + from app.cluster import state + + await self.reap(await state.current(fresh=True)) # 口径账本:最近几天封存了、一次都没比过的探索运行,在后台补比一次(ledger/match.py)。只读工件和版本表,不查业务库、 # 不调模型;和调度循环同一个开关,拷来的数据目录、测试不补 from app.ledger import match as ledger_match from app.ledger import recall as ledger_recall - ledger_match.start_backfill() - # 口径召回(ledger/recall.py):最近发布、自动那一批没发完的接着发,结束了还没收尾的召回运行补上收尾。同一个开关 - ledger_recall.start_resume() + # 多进程时只由调度当班的进程做(app/cluster;单进程时本进程启动就当班):几个进程各补一遍会重复比对、重复召回 + from app.cluster.service import leading + + if leading(): + ledger_match.start_backfill() + # 口径召回(ledger/recall.py):最近发布、自动那一批没发完的接着发,结束了还没收尾的召回运行补上收尾。同一个开关 + ledger_recall.start_resume() + + async def reap(self, live: Any) -> int: + """接手属主不在了的运行(启动时;多进程时当班进程隔一会儿再来一次,app/cluster/reaper.py)。规矩同以前的启动整理: + + - running 至少跑过一步、有断点:标 interrupted(可从断点继续运行),补一条 server_shutdown 日志——进程被强杀时连这一条 + 都来不及发,事件流停在半路,回放它的客户端才知道它是挂起了而不是还在跑; + - queued 还在排队,一个断点都没写过,「恢复」它只会从 START 用默认值重跑一遍、还丢掉 run.input,那不是恢复是伪造: + 标 failed,补一条 run.failed(不写结束时间、不封存:它从没开始执行,同以前)。 + + 不自动重跑有副作用的步骤:人看了再点「继续运行」。属主还活着的(别的进程正在跑、本进程里正在跑)一概不碰;认领那一下是 + 带条件的 UPDATE(状态、属主都还是刚才看到的),几个进程同时接手同一次运行只有一个改得动。交回整理了几次运行。""" + from app.cluster.claims import same + + async with SessionLocal() as session: + rows = (await session.execute(select(Run.id, Run.owner, Run.status).where( + Run.status.in_(("running", "queued"))))).all() + done = 0 + for run_id, owner, status in rows: + if live.alive(owner, local=self.is_active(run_id)): + continue + event: RunEventModel | None = None + async with SessionLocal() as session: + values: dict[str, Any] = ( + {"status": "interrupted", "error": "服务重启,运行已挂起,可从断点继续运行"} if status == "running" + else {"status": "failed", "error": QUEUED_AT_RESTART}) + claimed = await session.execute(update(Run).where(Run.id == run_id, Run.status == status, + same(Run.owner, owner)) + .values(owner=None, **values).execution_options(synchronize_session=False)) + if claimed.rowcount != 1: + await session.rollback() + continue + run = await session.get(Run, run_id) + bus.set_seq(run_id, await _seq_floor(session, run_id, run.last_seq if run else 0)) + event = self._event(run_id, EventType.LOG, data={ + "level": "warn", "code": "server_shutdown", + "message": "服务上次未正常关闭,这次运行已中断;可从断点继续运行", + }) if status == "running" else self._event(run_id, EventType.RUN_FAILED, data={"error": QUEUED_AT_RESTART}) + await _stage(session, event, run) + await session.commit() + await bus.publish(event) + await bus.close(run_id) + done += 1 + if done: + logger.warning("接手了 %d 次属主已经不在的运行(在跑的标中断、排队的标失败)", done) + return done async def shutdown(self) -> None: self._closing = True @@ -224,7 +250,20 @@ class RunManager: self, run_id: str, event_type: EventType | str, *, node_id: str | None = None, **data: Any, ) -> None: - """给运行之外的代码往运行里记一条事件。发起时就定下的事件不走这里,交给 start 的 notes。""" + """给运行之外的代码往运行里记一条事件。发起时就定下的事件不走这里,交给 start 的 notes。 + + 执行中的运行,序号由执行它的进程发(core/bus 的进程内计数):运行在别的活着的进程里执行时交给它记(cluster/requests), + 不在这里取号——这里的计数和那边的不是同一个,直接写会撞号。""" + if not self.is_active(run_id): + from app.cluster import requests, workers + + async with SessionLocal() as session: + row = (await session.execute(select(Run.status, Run.owner).where(Run.id == run_id))).first() + if row is not None and row.status in ("running", "queued") and row.owner \ + and row.owner != identity.owner() and await workers.alive(row.owner): + await requests.send(row.owner, "note", "run", run_id, payload={ + "type": str(event_type), "node_id": node_id, "data": data}) + return await self._emit(run_id, event_type, node_id=node_id, data=data) # ---------------- 启动 ---------------- @@ -334,6 +373,7 @@ class RunManager: agent_limits=agent_limits, data_versions=data_versions, options=options, + origin_kind=origin_kind, ) ) return run @@ -644,6 +684,19 @@ class RunManager: self._stopped_by[run_id] = actor task.cancel() return "stopping" + # 不在本进程里执行(多进程,app/cluster):在别的活着的进程里跑(或排队)的给它发停止请求,它照本进程的规矩停、发 + # run.cancelled(带上是谁停的),界面上同样是「正在停止」;属主不在了的照租约过期接手(标中断),再按停在断点上的放弃 + from app.cluster import requests, workers + + async with SessionLocal() as session: + row = (await session.execute(select(Run.status, Run.owner).where(Run.id == run_id))).first() + if row is not None and row.status in ("running", "queued") and row.owner and row.owner != identity.owner(): + if await workers.alive(row.owner): + await requests.send(row.owner, "stop", "run", run_id, actor=actor) + return "stopping" + from app.cluster import state + + await self.reap((await state.current(fresh=True)).trusting()) return "cancelled" if await self._abandon(run_id, actor) else None async def _abandon(self, run_id: str, actor: str | None) -> bool: @@ -717,7 +770,10 @@ class RunManager: options: dict[str, Any] | None = None, resumed: list[str] | tuple[str, ...] = (), actors: dict[str, str | None] | None = None, + origin_kind: str | None = None, ) -> None: + from app.cluster import slots + started = time.perf_counter() status = "succeeded" error: str | None = None @@ -735,12 +791,13 @@ class RunManager: # 等待从 run.interrupted 算起——以前执行一直计到 finally,收尾那一截执行和等待两边都算, # active + wait 比墙钟还长(机器一忙就差出几十毫秒) paused: float | None = None - async with self._semaphore: + async with slots.hold(run_id, origin_kind): try: async with SessionLocal() as session: row = await session.get(Run, run_id) if row is not None: row.status = "running" + row.owner = identity.owner() # started_at 只在第一次开始时写。以前每段 _drive 都覆盖它, # 审批恢复、接着跑之后,运行的"开始时间"就成了最后一段的开始 if row.started_at is None: @@ -1061,6 +1118,10 @@ class RunManager: run.end_reason = end_reason if status != "interrupted": run.finished_at = now + else: + # 停在审批上(或挂起)的运行不属于任何进程:谁收到决定谁接着跑(resume / continue_failed 的条件 UPDATE + # 认领,同一个决定只会被一个进程执行) + run.owner = None if status in ("failed", "cancelled"): # 并行支路上已经停下来等人的那张审批卡,运行结束了它也就没有意义了; # 留着的话它一直挂在全局待办里,批了也只会被告知运行不在断点上 @@ -1247,6 +1308,8 @@ def _carried(run: Run) -> dict[str, Any]: "data_versions": run.data_versions, # 运行选项同理:照发起时的快照,不重新读设置 "options": run.options or {}, + # 并发名额按来源归类(app/cluster/slots.py):恢复、接着跑的那一段和第一段同一类 + "origin_kind": run.origin_kind, } @@ -1494,7 +1557,7 @@ async def _claim( """只在状态仍是 expected 时把运行改成 running;已经被别人改过就回滚、说明原因。""" claimed = await session.execute( update(Run).where(Run.id == run_id, Run.status.in_(expected)) - .values(status="running", error=None, **values) + .values(status="running", error=None, owner=identity.owner(), **values) ) if claimed.rowcount != 1: await session.rollback() @@ -1666,3 +1729,28 @@ async def _timing(session: Any, run: Run, elapsed_ms: int, now: datetime) -> dic run_manager = RunManager() + + +def _register_requests() -> None: + """别的进程发来的:停止本进程在执行的运行、往它里面记一条事件(app/cluster/requests.py)。""" + from app.cluster import requests + + @requests.handle("stop", "run") + async def _stop(run_id: str, actor: str | None, _payload: dict[str, Any]) -> None: + if run_manager.is_active(run_id): + await run_manager.cancel(run_id, actor=actor) + + @requests.handle("note", "run") + async def _note(run_id: str, _actor: str | None, payload: dict[str, Any]) -> None: + kind, node_id, data = payload.get("type") or EventType.LOG, payload.get("node_id"), dict(payload.get("data") or {}) + if run_manager.is_active(run_id): + await run_manager._emit(run_id, kind, node_id=node_id, data=data) + return + # 收到时运行已经停下了:停在审批上的接在库里最后一条后面记(同 process/events.record),别的不记 + from app.engine.run_append import append_events + + if await append_events(run_id, [(kind, node_id, data)], require="interrupted") is None: + logger.info("运行 %s 已经不在执行,别的进程要记的一条 %s 没有记", run_id, kind) + + +_register_requests() diff --git a/backend/app/engine/spend.py b/backend/app/engine/spend.py index 87510ba7..f709b4d1 100644 --- a/backend/app/engine/spend.py +++ b/backend/app/engine/spend.py @@ -15,6 +15,10 @@ 各管各的上限,互不挤占。三本按同一个本地日期滚动(_today)。 目录里没有价格的模型估不出金额:只计次数(unpriced_calls),金额上限对它不起作用,由调用方照实说明。 + +**多进程。** 先读后写以前只靠进程内的锁:两个进程各读到同一个余额,会一起越过上限。现在每次记账先占住设置表里这本账的那一行 +(_locked:PostgreSQL 上是行锁,SQLite 上拿到写锁),读、判断、写在同一个事务里,几个进程一个接一个;这本账还没有那一行时 +插一行,撞了说明别的进程刚插上,重来一次。进程内的锁留着:同一个进程里的几个并行核查先在本进程排队,少占库的锁。 """ from __future__ import annotations @@ -43,6 +47,16 @@ def _lock(key: str) -> asyncio.Lock: return lock +async def _locked(session: Any, key: str) -> Any: + """占住设置表里 key 这一行再读它(见模块说明「多进程」):没有这一行交回 None。updated_at 不动。""" + from sqlalchemy import update + + await session.execute(update(Setting).where(Setting.key == key).values(value=Setting.value, + updated_at=Setting.updated_at) + .execution_options(synchronize_session=False)) + return await session.get(Setting, key, populate_existing=True) + + def _today() -> str: """本地日期:每日上限按它滚动。""" return datetime.now().astimezone().date().isoformat() @@ -80,21 +94,31 @@ class Ledger: limit 为 None 表示不限。估不出金额的调用(priced=False)照记次数,金额记 0,调用方应传 limit=None: 拿 0 去比上限,等于说它不花钱。 """ + from sqlalchemy.exc import IntegrityError + async with _lock(self.key): - async with SessionLocal() as session: - row = await session.get(Setting, self.key) - spend = self._spend_of(row) - if limit is not None and spend["usd"] + amount > limit + 1e-12: - return None - spend["usd"] = round(spend["usd"] + amount, 8) - spend["calls"] += 1 - spend["unpriced_calls"] += 0 if priced else 1 - if row is None: - session.add(Setting(key=self.key, value=spend)) - else: - row.value = spend - await session.commit() - return spend["date"] + for _ in range(3): + async with SessionLocal() as session: + row = await _locked(session, self.key) + spend = self._spend_of(row) + if limit is not None and spend["usd"] + amount > limit + 1e-12: + await session.rollback() + return None + spend["usd"] = round(spend["usd"] + amount, 8) + spend["calls"] += 1 + spend["unpriced_calls"] += 0 if priced else 1 + if row is None: + session.add(Setting(key=self.key, value=spend)) + else: + row.value = spend + try: + await session.commit() + except IntegrityError: + # 这本账的那一行刚被别的进程插上:重来一次,这回占得住它 + await session.rollback() + continue + return spend["date"] + raise RuntimeError(f"记账「{self.key}」失败:设置表里这一行一直插不进去") async def settle(self, date: str, delta: float) -> None: """调用结束后按实际用量找补预留的差额。跨了天就不找补:预留记在前一天,今天从零算。""" @@ -102,9 +126,10 @@ class Ledger: return async with _lock(self.key): async with SessionLocal() as session: - row = await session.get(Setting, self.key) + row = await _locked(session, self.key) spend = self._spend_of(row) if row is None or spend["date"] != date: + await session.rollback() return spend["usd"] = max(0.0, round(spend["usd"] + delta, 8)) row.value = spend diff --git a/backend/app/eval/runner.py b/backend/app/eval/runner.py index dd340d9f..3f179db4 100644 --- a/backend/app/eval/runner.py +++ b/backend/app/eval/runner.py @@ -43,7 +43,7 @@ from sqlalchemy import select, update from sqlalchemy.ext.asyncio import AsyncSession from app.core import job_runtime as runtime -from app.db.base import SessionLocal, utcnow +from app.db.base import current_owner, SessionLocal, utcnow from app.db.models_eval import EvalCase, EvalResult, EvalRun, EvalSuite from app.eval.cases import CaseInvalid @@ -963,22 +963,30 @@ def running(run: Any) -> bool: return manager.running(leader_of(run)) +def leader_of_id(rows: Sequence[EvalRun], job_id: str) -> str: + """rows 里 id 是 job_id 的那一臂由哪一臂的任务在跑(A/B 几臂共用领头那一臂的后台任务)。""" + row = next((r for r in rows if r.id == job_id), None) + return leader_of(row) if row is not None else job_id + + async def reconcile_stale(session: AsyncSession, rows: Sequence[EvalRun]) -> None: """读接口当场整理:记着排队、进行中,这个进程里却没在跑(服务重启过)的,照实标中断。""" - stale = [r for r in rows if r.status in runtime.ACTIVE and not running(r)] + # 属主不在了才算:多进程时别的进程正在跑的评测,这里看不到它的任务,却不能标中断(runtime.stale 按租约判断) + stale = await runtime.stale(list(rows), running=lambda job_id: manager.running(leader_of_id(rows, job_id))) for row in stale: await runtime.close_job(session, EvalRun, EvalResult, row.id, "interrupted", None, "cancelled", INTERRUPTED, finish=_closing(row.id, INTERRUPTED)) await session.refresh(row) -async def startup() -> int: - """服务启动时:没结束的评测标中断(没结束的项记停止),不自动续跑。""" +async def startup(*, live: Any = None) -> int: + """服务启动时(多进程时当班进程隔一会儿再来一次):属主不在了的评测标中断(没结束的项记停止),不自动续跑。""" async def close(session: AsyncSession, job_id: str) -> None: await runtime.close_job(session, EvalRun, EvalResult, job_id, "interrupted", None, "cancelled", INTERRUPTED, finish=_closing(job_id, INTERRUPTED)) - return await runtime.interrupt_on_startup(EvalRun, close, running=manager.running) + # 几臂共用领头那一臂的后台任务,各臂的属主是同一个进程:按属主判断就够了(running 只在属主是本进程时看) + return await runtime.interrupt_on_startup(EvalRun, close, running=manager.running, live=live) async def stop(run_id: str) -> bool: @@ -1035,6 +1043,7 @@ async def retry(run_id: str) -> list[str]: setattr(r, key, value) for arm in arms: arm.status, arm.error, arm.stop_reason, arm.finished_at = "queued", None, None, None + arm.owner = current_owner() # 重新排队的这一次由本进程认领(多进程:别把它当成原来那个进程的) await session.flush() for arm in arms: await _recount(session, arm.id) diff --git a/backend/app/eval/snapshots.py b/backend/app/eval/snapshots.py index 3aa29f8c..93a0db47 100644 --- a/backend/app/eval/snapshots.py +++ b/backend/app/eval/snapshots.py @@ -562,8 +562,8 @@ class ShadowDrafts(runtime.JobManager): manager = ShadowDrafts() -async def startup() -> int: - """服务启动时:没结束的影子起草标中断(不自动续跑)。""" +async def startup(*, live: Any = None) -> int: + """服务启动时(多进程时当班进程隔一会儿再来一次):属主不在了的影子起草标中断(不自动续跑)。""" async def close(session: AsyncSession, job_id: str) -> None: await session.execute(update(EvalCatalogSnapshot).where( EvalCatalogSnapshot.id == job_id, EvalCatalogSnapshot.status.in_(runtime.ACTIVE)) @@ -571,12 +571,13 @@ async def startup() -> int: .execution_options(synchronize_session=False)) await session.commit() - return await runtime.interrupt_on_startup(EvalCatalogSnapshot, close, running=manager.running) + return await runtime.interrupt_on_startup(EvalCatalogSnapshot, close, running=manager.running, live=live) async def reconcile_stale(session: AsyncSession, rows: Iterable[EvalCatalogSnapshot]) -> None: """读接口当场整理:记着排队、进行中,这个进程里却没在跑(服务重启过)的,照实标中断。""" - stale = [r for r in rows if r.status in runtime.ACTIVE and not manager.running(r.id)] + # 属主不在了才算:多进程时别的进程正在跑的,这里看不到它的任务,却不能标中断(runtime.stale 按租约判断) + stale = await runtime.stale(list(rows), running=manager.running) for row in stale: row.status, row.error, row.finished_at = "interrupted", INTERRUPTED, utcnow() if stale: diff --git a/backend/app/ledger/issue.py b/backend/app/ledger/issue.py index 0c451172..ee94a0a5 100644 --- a/backend/app/ledger/issue.py +++ b/backend/app/ledger/issue.py @@ -328,9 +328,11 @@ _locks: "weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, dict[tuple[str, st weakref.WeakKeyDictionary() -def _lock(key: tuple[str, str, int]) -> asyncio.Lock: - group = _locks.setdefault(asyncio.get_running_loop(), {}) - return group.setdefault(key, asyncio.Lock()) +def _lock(key: tuple[str, str, int]) -> Any: + """跨进程的锁(app/cluster/mutex):多进程时「有没有出具过、今天出了几次」的核对和发起照样一个接一个。""" + from app.cluster import mutex + + return mutex.lock(f"ledger-issue:{key[0]}:{key[1]}:{key[2]}") async def _hider(run_id: str) -> Any: diff --git a/backend/app/ledger/match.py b/backend/app/ledger/match.py index 3a4b12c4..1761ec9b 100644 --- a/backend/app/ledger/match.py +++ b/backend/app/ledger/match.py @@ -721,6 +721,18 @@ async def match_run(run_id: str, *, trigger: str = "sealed", timeout: float | No 运行一封存就比的(trigger=sealed)比完接着看能不能按口径自动正式出具(ledger/issue.auto_issue,拍板 L1):事后补比、 按需比的不自动出具——那是按现在的口径回看老答案,不该替人补发正式运行。""" + from app.cluster import mutex + + # 多进程:封存时比(执行它的进程)和补比(当班进程)可能同时来;跨进程的锁里一次一个,补比、封存时比先看比过没有 + async with mutex.lock(f"ledger-match:{run_id}"): + if trigger in ("sealed", "backfill"): + async with SessionLocal() as session: + if await store.matched_run_ids(session, [run_id]): + return None + return await _match_run(run_id, trigger=trigger, timeout=timeout) + + +async def _match_run(run_id: str, *, trigger: str, timeout: float | None) -> str | None: from app.api.evidence import _sealed from app.api.settings import ledger_settings diff --git a/backend/app/ledger/recall.py b/backend/app/ledger/recall.py index 0a9738cd..5cc96e12 100644 --- a/backend/app/ledger/recall.py +++ b/backend/app/ledger/recall.py @@ -446,9 +446,11 @@ _locks: "weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, dict[tuple[str, in weakref.WeakKeyDictionary() -def _lock(key: tuple[str, int]) -> asyncio.Lock: - group = _locks.setdefault(asyncio.get_running_loop(), {}) - return group.setdefault(key, asyncio.Lock()) +def _lock(key: tuple[str, int]) -> Any: + """跨进程的锁(app/cluster/mutex):同一(W、N)的计划和发起多进程时也串行,「已经自动发过几次」读得准。""" + from app.cluster import mutex + + return mutex.lock(f"ledger-recall:{key[0]}:{key[1]}") async def launch(workflow_id: str, new_version: int, *, batch: str = "auto", actor: str | None = None) -> list[str]: @@ -689,46 +691,54 @@ async def settle(run_id: str) -> str | None: return None _settling.add(run_id) try: - async with SessionLocal() as session: - run = await session.get(Run, run_id) - if run is None or run.origin_kind != ORIGIN or run.status not in ("succeeded", "failed", "cancelled") \ - or not run.manifest_hash: - return None - if await _settled(session, run_id) is not None: - return None - origin = run.origin if isinstance(run.origin, dict) else {} - if origin.get("probe"): - return await _settle_probe(run) - base = {"role": "recall", "workflow_id": origin.get("workflow_id"), "from_version": origin.get("from_version"), - "to_version": origin.get("to_version")} - if run.status != "succeeded": - await _mark(run_id, {**base, "outcome": "failed", "changed": None, - "error": str(run.error or "运行没有成功结束")[:300]}) - return "failed" - meta = origin.get("metrics") or [] - values = await card_values(run_id) - # 读不出召回卡、少了哪个比较的旧值或新值:两边都是空,按渲染比会被当成「没变」,给每个原答案盖上「已核对,不影响」 - # ——照实记重算失败,原答案一个不动(收过尾就不再核对,盖错了收不回来) - if not meta or any(f"{side}_{m.get('k')}" not in values for m in meta for side in ("old", "new")): - await _mark(run_id, {**base, "outcome": "failed", "changed": None, "error": CARD_UNREADABLE}) - return "failed" - results = compare(meta, values) - changed = any(r["changed"] for r in results) - answers = list(dict.fromkeys(str(u.get("run_id")) for u in origin.get("uses") or [] if u.get("run_id"))) - tasks: list[str] = [] - if changed: - tasks = await open_tasks(run, reason="caliber") - else: - for answer in answers: - await _mark_answer(answer, run) - await _mark(run_id, {**base, "outcome": "changed" if changed else "unchanged", "changed": changed, - "answers": len(answers), "tasks": tasks, - "metrics": [{"k": r["k"], "name": r["name"], "changed": r["changed"]} for r in results]}) - return "changed" if changed else "unchanged" + # 多进程:收尾的进程(runner 收尾时)和当班进程启动时的「补上收尾」可能同时来,跨进程的锁里先看收过没有 + from app.cluster import mutex + + async with mutex.lock(f"ledger-settle:{run_id}"): + return await _settle(run_id) finally: _settling.discard(run_id) +async def _settle(run_id: str) -> str | None: + async with SessionLocal() as session: + run = await session.get(Run, run_id) + if run is None or run.origin_kind != ORIGIN or run.status not in ("succeeded", "failed", "cancelled") \ + or not run.manifest_hash: + return None + if await _settled(session, run_id) is not None: + return None + origin = run.origin if isinstance(run.origin, dict) else {} + if origin.get("probe"): + return await _settle_probe(run) + base = {"role": "recall", "workflow_id": origin.get("workflow_id"), "from_version": origin.get("from_version"), + "to_version": origin.get("to_version")} + if run.status != "succeeded": + await _mark(run_id, {**base, "outcome": "failed", "changed": None, + "error": str(run.error or "运行没有成功结束")[:300]}) + return "failed" + meta = origin.get("metrics") or [] + values = await card_values(run_id) + # 读不出召回卡、少了哪个比较的旧值或新值:两边都是空,按渲染比会被当成「没变」,给每个原答案盖上「已核对,不影响」 + # ——照实记重算失败,原答案一个不动(收过尾就不再核对,盖错了收不回来) + if not meta or any(f"{side}_{m.get('k')}" not in values for m in meta for side in ("old", "new")): + await _mark(run_id, {**base, "outcome": "failed", "changed": None, "error": CARD_UNREADABLE}) + return "failed" + results = compare(meta, values) + changed = any(r["changed"] for r in results) + answers = list(dict.fromkeys(str(u.get("run_id")) for u in origin.get("uses") or [] if u.get("run_id"))) + tasks: list[str] = [] + if changed: + tasks = await open_tasks(run, reason="caliber") + else: + for answer in answers: + await _mark_answer(answer, run) + await _mark(run_id, {**base, "outcome": "changed" if changed else "unchanged", "changed": changed, + "answers": len(answers), "tasks": tasks, + "metrics": [{"k": r["k"], "name": r["name"], "changed": r["changed"]} for r in results]}) + return "changed" if changed else "unchanged" + + async def _mark_answer(answer_id: str, recall_run: Run) -> None: """数没变:在原答案的运行上记一条 ledger.recalled(已按 vN 核对,不影响)。这次召回记过的不再记。""" origin = recall_run.origin if isinstance(recall_run.origin, dict) else {} @@ -1299,8 +1309,10 @@ _catalog_locks: "weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, dict[str, weakref.WeakKeyDictionary() -def _catalog_lock(source_id: str) -> asyncio.Lock: - return _catalog_locks.setdefault(asyncio.get_running_loop(), {}).setdefault(source_id, asyncio.Lock()) +def _catalog_lock(source_id: str) -> Any: + from app.cluster import mutex + + return mutex.lock(f"ledger-catalog:{source_id}") async def catalog_pass(source_id: str, edits: Sequence[CatalogEdit]) -> int: @@ -1649,26 +1661,34 @@ async def consume_source_changes() -> int: return 0 _consuming.add(loop) try: - count = 0 - while True: - async with SessionLocal() as session: - since = await _cursor(session) - batch = await source_changes.list_changes(session, since_seq=since, limit=200) - if not batch: - return count - for change in batch: - try: - if basis_kind(change) is not None: - await process_change(change) - count += 1 - except Exception: # noqa: BLE001 - 一条出错不挡后面的 - logger.exception("召回处理变化记录 #%s 出错", change.get("seq")) - async with SessionLocal() as session: - await _set_cursor(session, int(batch[-1]["seq"])) + # 多进程:几个进程的探查、剖析结束时都会来拉,游标是设置表里先读后写的一行——跨进程的锁里一份接一份,后来的接着游标往下拉 + from app.cluster import mutex + + async with mutex.lock("ledger-consume-changes"): + return await _consume(source_changes) finally: _consuming.discard(loop) +async def _consume(source_changes: Any) -> int: + count = 0 + while True: + async with SessionLocal() as session: + since = await _cursor(session) + batch = await source_changes.list_changes(session, since_seq=since, limit=200) + if not batch: + return count + for change in batch: + try: + if basis_kind(change) is not None: + await process_change(change) + count += 1 + except Exception: # noqa: BLE001 - 一条出错不挡后面的 + logger.exception("召回处理变化记录 #%s 出错", change.get("seq")) + async with SessionLocal() as session: + await _set_cursor(session, int(batch[-1]["seq"])) + + # ========================================================================== # 发布钩子、启动时接着做、测试用的 drain # ========================================================================== diff --git a/backend/app/main.py b/backend/app/main.py index c96663a3..4b2b96bb 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -2,7 +2,7 @@ from __future__ import annotations import logging from contextlib import asynccontextmanager -from typing import AsyncIterator +from typing import Any, AsyncIterator from fastapi import Depends, FastAPI, Request from fastapi.middleware.cors import CORSMiddleware @@ -49,27 +49,54 @@ logger = logging.getLogger("agentlab") @asynccontextmanager async def lifespan(_: FastAPI) -> AsyncIterator[None]: + from app import cluster + from app.cluster import guard, mutex from app.data.engine import sqlite_dqs_self_check sqlite_dqs_self_check() # 写错的双引号列名必须报错;做不到就记日志并让启动失败 settings.ensure_dirs() + # SQLite 只支持单进程:同一个数据目录已经有一个进程在用(uvicorn --workers、两个服务指着同一个目录)就拒绝启动、照实说 + # (app/cluster/guard.py)。PostgreSQL 可以多进程,什么都不做 + guard.acquire() await init_db() - await seed_defaults() + # 多进程协调(app/cluster):登记这个工作进程、开始心跳和进程间通知、争调度当班。要在下面的启动整理之前:整理只碰属主不在了的 + # 行,要先知道别的进程谁还活着 + await cluster.start() + from app.cluster.relay import relay - # 记级别之前发布过的老版本:有正式运行凭据的补记为已发布,免得子工作流、口径卡钉住它们的正式运行被误判成「没发布过」 - from app.apps.versions import backfill_legacy_levels - from app.db.base import SessionLocal as _Sessions + await relay.start() + _register_reapers() + _register_cluster_hooks() - async with _Sessions() as _s: - if levelled := await backfill_legacy_levels(_s): - logger.info("给 %d 个老的发布版本补记了级别(有正式运行凭据)", levelled) - await _s.commit() - await _platform_startup() + # 下面这一段是全站的一次性整理(补默认数据、补记级别、存量密钥加密、上传版本的清理和迁移……),各项都是幂等的;多进程同时 + # 启动时一个接一个做(跨进程的锁),后起的看得到先起的做完的样子,不会重复插、互相删 + async with mutex.lock("startup"): + await seed_defaults() + + # 记级别之前发布过的老版本:有正式运行凭据的补记为已发布,免得子工作流、口径卡钉住它们的正式运行被误判成「没发布过」 + from app.apps.versions import backfill_legacy_levels + from app.db.base import SessionLocal as _Sessions + + async with _Sessions() as _s: + if levelled := await backfill_legacy_levels(_s): + logger.info("给 %d 个老的发布版本补记了级别(有正式运行凭据)", levelled) + await _s.commit() + await _platform_startup() + + # 上传表格的版本底座:清理上次没发布完的文件、给老上传补初始版本、回收没人用的版本。 + # 出错只记日志:最坏是留下几个多余的文件,不该让服务起不来 + from app.data import table_versions + from app.db.base import SessionLocal + + async with SessionLocal() as _s: + try: + await table_versions.startup(_s) + except Exception: # noqa: BLE001 + logger.exception("上传表格的版本整理失败,服务照常启动") # 把存的 embedding 配置读进来。不读的话重启后会悄悄退回本地哈希向量, # 而库里的向量是上次那个模型建的——检索会整体退回关键词, # 用户只看到"重启之后搜得不准了" - from app.db.base import SessionLocal from app.memory.embeddings import load_setting async with SessionLocal() as _session: @@ -78,88 +105,20 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: except Exception as e: # noqa: BLE001 - key 没了不该拦住整个服务起不来 logger.warning("embedding 配置没能生效,暂用本地向量:%s", e) - # 上次被强杀时留下的"处理中"文档。后台任务不会自己回来,留着它只会在 - # 界面上永远转圈——和 runner 对 running 的 run 是同一个道理 - from sqlalchemy import update as _update - - from app.db.models import Document as _Doc - - async with SessionLocal() as _s: - await _s.execute( - _update(_Doc).where(_Doc.status == "processing") - .values(status="failed", error="服务重启,这份文档没有处理完,请重新上传") - ) - await _s.commit() - - # 上传表格的版本底座:清理上次没发布完的文件、给老上传补初始版本、回收没人用的版本。 - # 出错只记日志:最坏是留下几个多余的文件,不该让服务起不来 - from app.data import table_versions - - async with SessionLocal() as _s: - try: - await table_versions.startup(_s) - except Exception: # noqa: BLE001 - logger.exception("上传表格的版本整理失败,服务照常启动") - - # 上次没做完的数据目录起草、剖析任务:标成中断(没结束的表记停止),不自动续跑——拷来的数据目录(检查脚本、验收副本) - # 启动时不能替人去调模型、查业务库。界面上点「重试」接着做没做完的表。剖析设置里按整次剖析算的旧预算顺带迁移掉 - from app.data import catalog_jobs, catalog_profile_jobs - - try: - await catalog_jobs.startup() - except Exception: # noqa: BLE001 - logger.exception("数据目录起草任务的启动整理失败,服务照常启动") - try: - await catalog_profile_jobs.startup() - except Exception: # noqa: BLE001 - logger.exception("数据剖析任务的启动整理失败,服务照常启动") - # 后台探查任务(next-round 03 A1)同一个规矩:没做完的标中断,读完的表物化进表结构,不自动续跑 - from app.data import introspect_jobs - - try: - await introspect_jobs.startup() - except Exception: # noqa: BLE001 - logger.exception("探查任务的启动整理失败,服务照常启动") - # 评测(next-round 01 做法 6、9):上次没做完的评测、影子起草同一个规矩——标中断(没做完的题记停止),不自动续跑: - # 拷来的数据目录启动时不能替人去调模型、查业务库。界面上点「重试」只把没做完的题接着做 - from app.eval import runner as eval_runner, snapshots as eval_snapshots - - try: - await eval_snapshots.startup() - except Exception: # noqa: BLE001 - logger.exception("影子起草的启动整理失败,服务照常启动") - try: - await eval_runner.startup() - except Exception: # noqa: BLE001 - logger.exception("评测任务的启动整理失败,服务照常启动") - + # 打开断点库;上次被强杀时留下的运行(只碰属主不在了的:多进程时别的活着的进程在跑的一概不碰)——在跑的标中断、排队的标失败 await run_manager.setup() - # 问数据的一轮(app/chat/job.py):在 run_manager 之后整理——服务重启时在跑的运行这时已经标成挂起。还在建图的标中断; - # 已经发起了运行的接着等它、照常复核收尾(和调度循环同一个开关:拷来的数据目录不替人调模型,等人打开对话时再接着收尾) - from app.chat import job as chat_job - - try: - await chat_job.startup() - except Exception: # noqa: BLE001 - logger.exception("问数据的启动整理失败,服务照常启动") - # 流程:存量的待审批补建待办(派给任何人,幂等);上次进程没了时已发出、没拿到结果的写入收成结果未知、安排核实 - from app.process import actions as _actions, tasks as _tasks - - await _tasks.startup() - await _actions.startup() - # 发布单:上次进程没了时还在执行的单改成停下(停在没做完的那一步、写明执行中断),这样能接着做 - from app.blueprints import publish_plan as _plans - - try: - if stopped := await _plans.startup(): - logger.info("%d 份发布单执行中断,已改成停下,可以接着做", stopped) - except Exception: # noqa: BLE001 - logger.exception("发布单的启动整理失败,服务照常启动") + # 接手属主不在了的:处理中的文档标失败;数据目录起草、剖析、探查、评测、影子起草标中断(不自动续跑:拷来的数据目录启动时不能 + # 替人去调模型、查业务库,界面上点「重试」接着做);问数据的一轮(在 run_manager 之后:在跑的运行这时已经标成挂起)还在建图的 + # 标中断,发起了运行的接着等它、照常复核收尾;待审批补建待办、停在 resolving 的收拾;结果未知的写入安排核实;执行中断的 + # 发布单改成停下。各项的规矩见 _register_reapers 登记的各模块。启动时每个进程都做一次(同样一个接一个),之后当班进程隔一会儿 + # 再做一次(别的进程没了的时候) + async with mutex.lock("startup"): + await cluster.service.startup_reap() from app.sandbox.manager import sandbox_manager as _sbm await _sbm.start_reaper() # 定时任务的调度循环放在 run_manager 之后起:它到点要发起运行。拷来的数据目录(检查脚本的 - # --stacks)用 AGENTLAB_SCHEDULER=0 关掉,免得替原库把定时任务再执行一遍 + # --stacks)用 AGENTLAB_SCHEDULER=0 关掉,免得替原库把定时任务再执行一遍。每个进程都起,只有调度当班的进程真的做 from app import scheduler if scheduler.scheduler_enabled(): @@ -170,10 +129,14 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: finally: # 先停调度,再停运行:停运行的途中不该再有新运行被发起。起草、剖析任务停下来标成中断,下次启动不会再续 await scheduler.stop() + from app.data import catalog_jobs, catalog_profile_jobs, introspect_jobs + from app.chat import job as chat_job + from app.eval import runner as eval_runner, snapshots as eval_snapshots + await catalog_jobs.manager.shutdown() await catalog_profile_jobs.manager.shutdown() await introspect_jobs.manager.shutdown() - # 问数据的一轮停下不改那几行(下次启动照重启的规矩整理),也要在运行之前停 + # 问数据的一轮停下不改那几行(下次启动、或者别的进程照「属主不在了」的规矩接手),也要在运行之前停 await chat_job.shutdown() # 评测要在运行之前停:它停下时会取消自己发起的作答、算期望的运行,再标中断 await eval_runner.shutdown() @@ -182,9 +145,136 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: from app.sandbox.manager import sandbox_manager await sandbox_manager.close() + # 最后交班、删掉本进程的登记:别的进程马上就知道本进程不在了,接手它留下的行 + await relay.stop() + await cluster.stop() logger.info("已停止") +def _register_cluster_hooks() -> None: + """心跳里本进程自己的活、接班之后要补做的事(app/cluster/service.py)。""" + from app import cluster + from app.auth import keys + + # 访问密钥的请求记录攒在每个进程自己的内存里(每个请求同步写一行会和运行抢 SQLite 的写锁):每个进程隔一拍心跳写自己的。 + # 过了保留期的清理只在当班进程(调度循环的那一拍) + cluster.on_beat("访问密钥的请求记录", keys.flush) + + def on_lead(now_leading: bool) -> None: + if not now_leading: + return + from app.ledger import match as ledger_match, recall as ledger_recall + from app.scheduler.loop import runner + + # 接了调度班:马上补一拍;口径比对的补比、自动召回的续发也归当班进程(同 run_manager.setup 里启动时的那一次) + runner.poke() + ledger_match.start_backfill() + ledger_recall.start_resume() + + cluster.on_leadership(on_lead) + _register_cache_handlers() + + +def _register_cache_handlers() -> None: + """别的进程改了、本进程缓存着的东西怎么重读(app/cluster/caches.py:改的进程 broadcast,这里照着清掉、重读)。""" + from app.cluster import caches + + async def auth_settings(_key: str | None) -> None: + from app.auth import load_settings + + await load_settings(force=True) + + async def public_address(_key: str | None) -> None: + from app.core import public_url + + await public_url.refresh() + + async def embeddings(_key: str | None) -> None: + from app.db.base import SessionLocal + from app.memory.embeddings import load_setting + + async with SessionLocal() as session: + await load_setting(session) + + async def capabilities(_key: str | None) -> None: + from app.capabilities import registry + from app.db.base import SessionLocal + + async with SessionLocal() as session: + await registry.refresh_cache(session, broadcast=False) + + async def datasource_engine(key: str | None) -> None: + from app.data.engine import engines + + if key: + await engines.invalidate(key, broadcast=False) + + def app_pages(_key: str | None) -> None: + from app.apps.resolve import clear_cache + + clear_cache() + + for name, fn in (("auth_settings", auth_settings), ("public_url", public_address), ("embeddings", embeddings), + ("capabilities", capabilities), ("datasource_engine", datasource_engine), ("app_pages", app_pages)): + caches.on(name, fn) + + +def _register_reapers() -> None: + """接手属主不在了的行的各种规矩(app/cluster/reaper.py),按这个先后:运行在问数据之前(问数据接手时要看运行的状态)。""" + from app.blueprints import publish_plan + from app.chat import job as chat_job + from app.cluster import reaper + from app.data import catalog_jobs, catalog_profile_jobs, introspect_jobs + from app.eval import runner as eval_runner, snapshots as eval_snapshots + from app.process import actions, tasks + + async def documents(live: Any) -> int: + return await _reap_documents(live) + + for name, fn in ( + ("运行", run_manager.reap), + ("处理中的文档", documents), + ("数据目录起草任务", lambda live: catalog_jobs.startup(live=live)), + ("数据剖析任务", lambda live: catalog_profile_jobs.startup(live=live)), + ("探查任务", lambda live: introspect_jobs.startup(live=live)), + ("影子起草", lambda live: eval_snapshots.startup(live=live)), + ("评测任务", lambda live: eval_runner.startup(live=live)), + ("问数据的一轮", _chat_reaper(chat_job)), + ("待办", lambda live: tasks.startup(live=live)), + ("结果未知的写入", lambda live: actions.startup(live=live)), + ("发布单", lambda live: publish_plan.startup(live=live)), + ): + reaper.register(name, fn) + + +def _chat_reaper(chat_job: Any) -> Any: + async def reap(live: Any) -> Any: + counts = await chat_job.startup(live=live) + return {k: v for k, v in counts.items() if v} or 0 + return reap + + +async def _reap_documents(live: Any) -> int: + """上次被强杀时留下的「处理中」文档(只看属主不在了的)。后台任务不会自己回来,留着它只会在界面上永远转圈——和 runner + 对 running 的 run 是同一个道理。""" + from sqlalchemy import update as _update + + from app.cluster import claims + from app.db.base import SessionLocal + from app.db.models import Document as _Doc + + ids = [doc_id for doc_id, _ in await claims.orphans(_Doc, live, running=lambda _id: False, + statuses=("processing",))] + if ids: + async with SessionLocal() as _s: + await _s.execute( + _update(_Doc).where(_Doc.id.in_(ids), _Doc.status == "processing") + .values(status="failed", error="服务重启,这份文档没有处理完,请重新上传") + ) + await _s.commit() + return len(ids) + + async def _platform_startup() -> None: """应用平台的启动整理(BE-1):登录设置、对外访问地址读进缓存;自定义工具请求头、MCP env 和请求头里的存量明文加密 一遍(幂等);存量的内网 MCP 服务改为绑定登记端点(只在升级后第一次启动时做);能力登记同步一次。后三样出错只记 diff --git a/backend/app/notify/service.py b/backend/app/notify/service.py index 4c401397..d412c33e 100644 --- a/backend/app/notify/service.py +++ b/backend/app/notify/service.py @@ -262,7 +262,11 @@ async def _where(session: Any, run: Run | None) -> str: #: 邮件渠道:设置表里一行 {用户 id: 建的时间}(通知自己记的账,键的开头同 notify/digest.STATE_PREFIX,设置接口不列)。只在 #: 第一次派到待办时建一次——本人删掉了就是不要,不再替他建回来 MAIL_AUTO_KEY = "notify_state:mail_auto" -_mail_auto_lock = asyncio.Lock() +#: 先读后写那一行:跨进程的锁(app/cluster/mutex) +def _mail_auto_lock() -> Any: + from app.cluster import mutex + + return mutex.lock("notify-mail-auto") async def ensure_assignee_mail(users: list[str]) -> int: @@ -276,7 +280,7 @@ async def ensure_assignee_mail(users: list[str]) -> int: if not users or current_mode() == "off": return 0 - async with _mail_auto_lock, SessionLocal() as session: + async with _mail_auto_lock(), SessionLocal() as session: cfg = await mail.load_config(session) if not (cfg.enabled and cfg.default_for_assignees): return 0 diff --git a/backend/app/process/actions.py b/backend/app/process/actions.py index 14b4658b..0f1d6edd 100644 --- a/backend/app/process/actions.py +++ b/backend/app/process/actions.py @@ -668,12 +668,22 @@ async def _lookup(action_id: str, claimed_at: datetime, now: datetime) -> int: return 1 -async def startup() -> int: +async def startup(*, live: Any = None) -> int: """启动时:上次进程没了的时候已经发出(sent)、还没拿到结果的动作收成结果未知,安排核实;结果未知却还没安排核实的 - (挂起之后、安排之前进程没了)补上。返回处理的条数。""" + (挂起之后、安排之前进程没了)补上。返回处理的条数。 + + 多进程:所属的运行还在别的活着的进程里执行的不碰(那边正在发、正在等结果)。""" + from app.cluster import state + from app.db.models import Run + from app.engine.runner import run_manager + + live = live if live is not None else await state.current(fresh=True) async with SessionLocal() as session: - ids = list((await session.execute(select(ActionRecord.id).where( - ActionRecord.state.in_(("sent", "unknown")), ActionRecord.resolution.is_(None)))).scalars()) + rows = (await session.execute(select(ActionRecord.id, ActionRecord.run_id, Run.status, Run.owner) + .outerjoin(Run, Run.id == ActionRecord.run_id).where( + ActionRecord.state.in_(("sent", "unknown")), ActionRecord.resolution.is_(None)))).all() + ids = [action_id for action_id, run_id, status, owner in rows + if not (status in ("running", "queued") and live.alive(owner, local=run_manager.is_active(run_id)))] for action_id in ids: try: await on_unknown(action_id) diff --git a/backend/app/process/tasks.py b/backend/app/process/tasks.py index 4d62c7eb..c82baa5d 100644 --- a/backend/app/process/tasks.py +++ b/backend/app/process/tasks.py @@ -1023,16 +1023,26 @@ def _aware(moment: datetime) -> datetime: # -------------------------------------------------------------------------- -async def startup() -> int: +async def startup(*, live: Any = None) -> int: """启动时:存量的待审批补建 Task(派给任何人,幂等);上次停在 resolving 的(判定之后、恢复之前进程没了)按 Approval - 的状态收拾——还在等的退回 pending、撤回那一票,已经交给引擎的记成已处理。返回补建的条数。""" + 的状态收拾——还在等的退回 pending、撤回那一票,已经交给引擎的记成已处理。返回补建的条数。 + + 多进程:别的进程活着时,resolving 可能正是它在判定、恢复(几秒的事),只收拾停在 resolving 超过两个租约时长的。""" + from datetime import timedelta + + from app.cluster import state + from app.cluster.workers import lease_s + + live = live if live is not None else await state.current(fresh=True) + stale_before = utcnow() - timedelta(seconds=2 * lease_s()) if live.others else None async with SessionLocal() as session: orphans = list((await session.execute( select(Approval).outerjoin(Task, Task.approval_id == Approval.id) .where(Approval.status == "pending", Task.id.is_(None)))).scalars()) for approval in orphans: await open_task(session, approval) - stuck = list((await session.execute(select(Task).where(Task.status == "resolving"))).scalars()) + stuck = list((await session.execute(select(Task).where( + Task.status == "resolving", *((Task.updated_at < stale_before,) if stale_before is not None else ())))).scalars()) for task in stuck: waiting = task.approval_id and await session.scalar( select(Approval.status).where(Approval.id == task.approval_id)) == "pending" diff --git a/backend/app/records/actions.py b/backend/app/records/actions.py index 1b94218a..aa4fb6da 100644 --- a/backend/app/records/actions.py +++ b/backend/app/records/actions.py @@ -79,9 +79,11 @@ async def append_action(args: dict[str, Any], *, idem_key: str | None, run_id: s environment=environment) try: columns = columns_of(table) - done = await asyncio.to_thread( - store.append, path_of(table), columns, values, version=table.version, run_id=run_id, - actor=actor, action_key=action_key_of(idem_key), supersedes=supersedes) + # 多进程时几个进程会往同一个记录表文件里追加:库里的锁(store.write_lock)一个接一个,进程里的 _WRITE_LOCK 照旧 + async with store.write_lock(): + done = await asyncio.to_thread( + store.append, path_of(table), columns, values, version=table.version, run_id=run_id, + actor=actor, action_key=action_key_of(idem_key), supersedes=supersedes) except store.RecordsRefused as e: return _result(None, error=e.message, environment=environment) if not done.duplicate: diff --git a/backend/app/records/store.py b/backend/app/records/store.py index 1a370754..5cd17f62 100644 --- a/backend/app/records/store.py +++ b/backend/app/records/store.py @@ -4,7 +4,8 @@ 业务列(「填什么」卡片生成),外加一列不对外的 _action_key(幂等键,唯一)。查询走连接上的同名临时视图 (view_select),视图不露 _action_key。 -**写。** 只有 append 一条路:单进程锁(_WRITE_LOCK)里开一个短事务,先按 action_key 查——同一个动作第二次来交回 +**写。** 只有 append 一条路:写锁里开一个短事务(进程里的 _WRITE_LOCK;多进程时调用方另持库里的锁 write_lock(),几个进程 +往同一个文件里写一个接一个——SQLite 文件自己的文件锁在网络文件系统上靠不住),先按 action_key 查——同一个动作第二次来交回 第一次写下的那一行,不再写;更正是一次新的追加,带 _supersedes 指向被更正的行,原行不改。_seq 是 SQLite 的 rowid (INTEGER PRIMARY KEY),只增不减:从不删行,也就不会复用。 @@ -49,6 +50,14 @@ _BAD_NAME = re.compile(r"[\"`\[\]\x00-\x1f\x7f]") _WRITE_LOCK = threading.Lock() +def write_lock() -> Any: + """跨进程的写锁(app/cluster/mutex:PostgreSQL 时是咨询锁,SQLite 只有一个进程、就是本进程的锁)。异步的调用方在把写入 + 放进线程之前持着它(async with)。""" + from app.cluster import mutex + + return mutex.lock("records-store") + + class RecordsRefused(ValueError): """这一次写入不收:code 是问题编号(records.*,白话在 app/plain.py),message 直接给人看。""" diff --git a/backend/app/records/tables.py b/backend/app/records/tables.py index 878c7896..b42c805c 100644 --- a/backend/app/records/tables.py +++ b/backend/app/records/tables.py @@ -187,7 +187,8 @@ async def _free_name(session: AsyncSession, base: str) -> str: async def _register(session: AsyncSession, *, key: str, version: int, columns: list[Column], label: str, app_id: str | None, blueprint_id: str | None) -> RecordsTable: path = store.table_path(key, version) - await asyncio.to_thread(store.create_file, path, columns) + async with store.write_lock(): + await asyncio.to_thread(store.create_file, path, columns) base = f"{SOURCE_PREFIX}{re.sub(r'[^a-z0-9]', '', key.lower())[:12] or 'table'}" name = await _free_name(session, base if version == 1 else f"{base}_v{version}") source = DataSource(name=name, kind="sqlite", database=str(path), readonly=True, origin=RECORDS_ORIGIN, @@ -371,7 +372,8 @@ async def _sync_registration(session: AsyncSession, row: RecordsTable, wanted: l have = columns_of(row) known = {h.name.lower() for h in have} added = [c for c in wanted if c.name.lower() not in known] - await asyncio.to_thread(store.add_columns, store.table_path(row.table_key, row.version), added) + async with store.write_lock(): + await asyncio.to_thread(store.add_columns, store.table_path(row.table_key, row.version), added) by_name = {w.name.lower(): w for w in wanted} # 列的顺序:原有的在前(照原来的顺序),新加的在后——和文件里的列序一致。删掉的列(只在不兼容时出现)这里不会有 kept = [h if draft else by_name.get(h.name.lower(), h) for h in have] diff --git a/backend/app/scheduler/loop.py b/backend/app/scheduler/loop.py index 6d231434..7a7fba52 100644 --- a/backend/app/scheduler/loop.py +++ b/backend/app/scheduler/loop.py @@ -9,6 +9,10 @@ skipped_misfire;更早的积压只记一条跳过,不补。下一次一律 事件触发的任务(docs/design/orchestration-round/00-总方案.md §4)不在 fire_due 里:它们没有 next_run_at。每一拍发完到点的, 接着派待派发事件(scheduler/outbox.dispatch_due)。调度关着时这一拍不存在,事件留在表里。 + +**多进程只有一个进程当班**(app/cluster/leader.py):循环在每个进程里都起,每一拍先看本进程当不当班,不当班这一拍什么都 +不做(定时认领、事件派发、流程的时限和核实、Webhook 投递、各类周期清理都只在当班的进程里跑)。当班的进程没了,待命的接班 +之后马上补一拍。切换的那一会儿两边都以为自己当班也重复不了:认领是条件 UPDATE、触发记录和投递记录有唯一约束。 """ from __future__ import annotations @@ -178,6 +182,7 @@ class _Runner: #: 通知推送另起任务:Webhook 超时加重试最长要几十秒,不能拖着下一拍的发起。同一时刻只跑一份 self.notify: asyncio.Task[None] | None = None self._stopping: asyncio.Event | None = None + self._poked: asyncio.Event | None = None def running(self) -> bool: return self.task is not None and not self.task.done() @@ -189,10 +194,21 @@ class _Runner: self.task = asyncio.create_task(self._loop(interval, self._stopping), name="agentlab-scheduler") logger.info("定时任务调度已启动,每 %s 秒检查一次", interval) + def poke(self) -> None: + """马上补一拍(刚接了调度班)。""" + if self._poked is not None: + self._poked.set() + async def _loop(self, interval: float, stopping: asyncio.Event) -> None: + from app.cluster import leading from app.scheduler import webhook + self._poked = asyncio.Event() while not stopping.is_set(): + if not leading(): + # 不当班:这一拍什么都不做,等下一拍(或者接班时被叫醒) + await self._wait(stopping, interval) + continue try: await fire_due() await dispatch_events() @@ -204,8 +220,20 @@ class _Runner: raise except Exception: # noqa: BLE001 - 这一拍出错不该让循环停下 logger.exception("定时任务调度这一拍出错,下一拍照常进行") - with contextlib.suppress(TimeoutError): - await asyncio.wait_for(stopping.wait(), timeout=interval) + await self._wait(stopping, interval) + + async def _wait(self, stopping: asyncio.Event, interval: float) -> None: + poked = self._poked + waits = [asyncio.ensure_future(stopping.wait())] + if poked is not None: + waits.append(asyncio.ensure_future(poked.wait())) + try: + await asyncio.wait(waits, timeout=interval, return_when=asyncio.FIRST_COMPLETED) + finally: + for w in waits: + w.cancel() + if poked is not None: + poked.clear() async def stop(self) -> None: """让循环收完手上这一拍再停、正在推送的通知推完;各等一个宽限(STOP_GRACE_S、NOTIFY_GRACE_S),两件事并行 diff --git a/backend/app/scheduler/service.py b/backend/app/scheduler/service.py index 4b0aa041..9539b94b 100644 --- a/backend/app/scheduler/service.py +++ b/backend/app/scheduler/service.py @@ -380,26 +380,33 @@ def check_fields(template: dict[str, str], fields: list[dict[str, Any]], workflo # 发起一次触发 # -------------------------------------------------------------------------- -#: 每个事件循环一把锁:发起(查重叠 → 记触发 → 发起运行 → 改记录)和通知扫描查库互斥。单进程部署,进程内 -#: 互斥就够;按事件循环分开是因为测试每个用例一个新循环,跨循环用同一把 asyncio.Lock 会报错 -_LOCKS: "weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, asyncio.Lock]" = weakref.WeakKeyDictionary() +def launch_lock() -> Any: + """发起(查重叠 → 记触发 → 发起运行 → 改记录)和通知扫描查库互斥。跨进程的锁(app/cluster/mutex:本进程按事件循环各一把, + PostgreSQL 时另取咨询锁):多进程时到点的发起在当班进程、「立即运行」在接到请求的进程,照样互斥。""" + from app.cluster import mutex - -def launch_lock() -> asyncio.Lock: - loop = asyncio.get_running_loop() - lock = _LOCKS.get(loop) - if lock is None: - lock = _LOCKS[loop] = asyncio.Lock() - return lock + return mutex.lock("schedule-launch") #: 正在发起的触发:占位记录已写进库、launch_run 还没返回。通知扫描有 launch_lock 挡着,接口的读(任务列表的 -#: last_fire、触发历史)没有:它们据此按「已发起」给,不把还在发起的说成「未能发起」(单进程部署,进程内的集合就够) +#: last_fire、触发历史)没有:它们据此按「已发起」给,不把还在发起的说成「未能发起」 _LAUNCHING: set[str] = set() -def launching(fire_id: str) -> bool: - return fire_id in _LAUNCHING +def launching(fire_id: str, fire: Any = None) -> bool: + """这一次触发是不是还在发起:本进程里在发起;多进程时也可能在别的进程里——占位记录(「未能发起:中断」、还没有运行)写下 + 不到一个租约时长的,按还在发起算(进程真没了的,过了租约就照实显示)。""" + if fire_id in _LAUNCHING: + return True + if fire is None or getattr(fire, "status", None) != "failed_to_start" or getattr(fire, "run_id", None) \ + or getattr(fire, "reason", None) != INTERRUPTED_REASON or getattr(fire, "created_at", None) is None: + return False + from datetime import timedelta + + from app.cluster.workers import lease_s + + created = fire.created_at if fire.created_at.tzinfo else fire.created_at.replace(tzinfo=timezone.utc) + return created > utcnow() - timedelta(seconds=lease_s()) def actor_label(name: str) -> str: diff --git a/backend/tests/test_eval_snapshots.py b/backend/tests/test_eval_snapshots.py index a6702b06..955cfcb1 100644 --- a/backend/tests/test_eval_snapshots.py +++ b/backend/tests/test_eval_snapshots.py @@ -120,7 +120,7 @@ async def test_shadow_drafting_writes_only_the_copy(source, monkeypatch): assert snap.status in ("queued", "running") # 不占数据源的队:同一个源照样能马上起草、剖析 await asyncio.sleep(0) - assert job_runtime.lanes.holders(source["id"]) == [] + assert await job_runtime.lanes.holders(source["id"]) == [] done = await wait_done(snap.id) assert done.status == "succeeded", (done.error, done.progress) # 线上目录一项不动 diff --git a/backend/tests/test_introspect_jobs.py b/backend/tests/test_introspect_jobs.py index feb03bda..62734cf7 100644 --- a/backend/tests/test_introspect_jobs.py +++ b/backend/tests/test_introspect_jobs.py @@ -211,7 +211,7 @@ async def test_stop_keeps_what_was_read_and_restart_marks_interrupted(client, bi async def test_introspect_queues_behind_profile_and_blocks_sync_writes(client, small, monkeypatch): sid, _ = small - held = runtime.lanes.enter(sid, "profile", "someone-else") + held = await runtime.lanes.enter(sid, "profile", "someone-else") try: job = (await client.post(f"/api/datasources/{sid}/introspect")).json()["job"] await asyncio.sleep(0.1) @@ -220,7 +220,7 @@ async def test_introspect_queues_behind_profile_and_blocks_sync_writes(client, s r = await client.post(f"/api/datasources/{sid}/introspect") assert r.status_code == 409 and r.json()["code"] == "introspect_busy" finally: - held.release() + await held.release() await introspect_jobs.manager.wait(job["id"], timeout=30) assert (await client.get(f"/api/datasources/{sid}/introspect-jobs/{job['id']}")).json()["job"]["status"] \ == "succeeded" @@ -237,7 +237,8 @@ async def test_introspect_queues_behind_profile_and_blocks_sync_writes(client, s job = (await client.post(f"/api/datasources/{sid}/introspect")).json()["job"] for _ in range(100): await asyncio.sleep(0.02) - if runtime.lanes.holders(sid) and runtime.lanes.holders(sid)[0].kind == "introspect": + holders = await runtime.lanes.holders(sid) + if holders and holders[0].kind == "introspect": break r = await client.post(f"/api/datasources/{sid}/catalog/draft", json={"tables": ["alpha"]}) assert r.status_code == 409 and r.json()["code"] == "introspect_busy" and "探查" in r.json()["detail"] @@ -436,14 +437,14 @@ async def test_draft_and_profile_jobs_say_they_wait_for_introspect(small): try: for kind, job, waiting in (("profile", profile, catalog_profile_jobs.waiting_for), ("draft", draft, catalog_jobs.waiting_for)): - held = runtime.lanes.enter(sid, "introspect", "an-introspect-job") - mine = runtime.lanes.enter(sid, kind, job.id) + held = await runtime.lanes.enter(sid, "introspect", "an-introspect-job") + mine = await runtime.lanes.enter(sid, kind, job.id) try: async with SessionLocal() as session: assert await waiting(session, job) == "introspect", kind finally: - held.release() - mine.release() + await held.release() + await mine.release() finally: async with SessionLocal() as session: for model, job_id in ((CatalogProfileJob, profile.id), (CatalogDraftJob, draft.id)): @@ -644,7 +645,7 @@ async def test_tables_added_while_the_job_is_queued_stay_in_scope(client, small) got = await so.add_to_scope(sid, ["eta"], trail=so.Trail(via=so.VIA_LOOK, actor="tester")) assert got.added == ["eta"] await _scope(client, sid, mode="rules", include=["alpha"], removed=["eta"]) - held = runtime.lanes.enter(sid, "profile", "someone-else") + held = await runtime.lanes.enter(sid, "profile", "someone-else") try: job = (await client.post(f"/api/datasources/{sid}/introspect")).json()["job"] assert job["status"] == "queued" and job["total"] == 1 @@ -655,7 +656,7 @@ async def test_tables_added_while_the_job_is_queued_stay_in_scope(client, small) # 任务还没开始:范围还是上一次的(eta 还在),新加的两张已经进了表结构 assert set((await _cache(sid))["tables"]) == {"alpha", "beta", "gamma", "eta"} finally: - held.release() + await held.release() await introspect_jobs.manager.wait(job["id"], timeout=30) assert (await client.get(f"/api/datasources/{sid}/introspect-jobs/{job['id']}")).json()["job"]["status"] \ == "succeeded" diff --git a/backend/tests/test_job_runtime.py b/backend/tests/test_job_runtime.py index ffda8c90..742e3fe2 100644 --- a/backend/tests/test_job_runtime.py +++ b/backend/tests/test_job_runtime.py @@ -32,7 +32,7 @@ def test_an_unregistered_kind_is_refused(): async def go() -> None: with pytest.raises(ValueError, match="没有登记"): - lanes.enter("src", "nonsense-kind") + await lanes.enter("src", "nonsense-kind") asyncio.run(go()) @@ -52,16 +52,16 @@ def test_a_new_kind_queues_behind_the_existing_ones_on_the_same_source(): lanes = job_runtime.SourceLanes() async def go() -> None: - first = lanes.enter("src-a", "draft", "job-1") - second = lanes.enter("src-a", "test-introspect", "job-2") - other = lanes.enter("src-b", "test-introspect", "job-3") + first = await lanes.enter("src-a", "draft", "job-1") + second = await lanes.enter("src-a", "test-introspect", "job-2") + other = await lanes.enter("src-b", "test-introspect", "job-3") assert first.granted.is_set() and not second.granted.is_set() assert other.granted.is_set(), "不同源并行" - assert lanes.ahead("src-a", "job-2") == ["draft"] - first.release() + assert await lanes.ahead("src-a", "job-2") == ["draft"] + await first.release() await asyncio.wait_for(second.wait(), timeout=1) - second.release() - other.release() + await second.release() + await other.release() asyncio.run(go()) diff --git a/backend/tests/test_run_append.py b/backend/tests/test_run_append.py index 2032c090..7c5103e2 100644 --- a/backend/tests/test_run_append.py +++ b/backend/tests/test_run_append.py @@ -622,10 +622,10 @@ def _judged_args(run: Run) -> tuple: async def test_a_judge_interleaved_with_a_post_seal_append_survives_a_restart(flaky, monkeypatch): - """按需裁判记判定(_append_judged)的条件更新不带 last_seq 条件:它取了号、还没写,封存后追加抢先提交了, - 它再提交时把 last_seq 写回自己那个更小的号。同一进程里序号计数只增不减,不会撞号;进程重启后计数从 0 起, - 各处又按 last_seq 接号——裁判自己也是。所以启动时按库里实际的最后一条改正 last_seq:之后再裁判、继续运行, - 都接在实际的最后一条后面。""" + """按需裁判记判定(_append_judged)和封存后追加交错:裁判取了号、还没写,封存后追加抢先提交了。以前裁判的条件更新不带 + last_seq 条件,照样提交、把 last_seq 写回自己那个更小的号(同一进程里计数只增不减,不撞号;多进程时两个进程各自的计数 + 会撞号)。现在同 append_events:last_seq 也要还是读到的那个,后到的裁判这一次不记(判定照样交回给点的人),last_seq + 停在追加的那一条上。启动时照旧按库里实际的最后一条改正 last_seq:之后再裁判、继续运行,都接在实际的最后一条后面。""" from app.api import evidence from app.engine.run_append import append_events @@ -662,9 +662,9 @@ async def test_a_judge_interleaved_with_a_post_seal_append_survives_a_restart(fl await asyncio.sleep(0.01) written = await append_events(first.id, [inquiry("这次为什么失败")], require="sealed") held.set() - assert await judging == {"seq": first.last_seq + 1} + assert await judging is None, "追加先提交了:后到的裁判不记,免得两条撞号" assert written is not None and written[0].seq == first.last_seq + 2 - assert (await _row(first.id)).last_seq == first.last_seq + 1, "裁判后提交,last_seq 被写回了小的号" + assert (await _row(first.id)).last_seq == written[0].seq monkeypatch.setattr(evidence, "SessionLocal", opened) # 进程重启:内存里的序号计数没了,启动时 setup 照常执行一遍。只改落后的那条,别的运行原样不动 diff --git a/backend/tests/test_source_upkeep.py b/backend/tests/test_source_upkeep.py index 0aaab038..ef51710f 100644 --- a/backend/tests/test_source_upkeep.py +++ b/backend/tests/test_source_upkeep.py @@ -243,7 +243,8 @@ async def test_background_jobs_follow_the_lane_order(client, shop): await session.commit() ids = {"introspect": intro.id, "draft": draft.id, "profile": profile.id} # 队里是 起草(在做)→ 剖析(排着);探查刚建、还没进队:排在队里的后面 - tickets = [runtime.lanes.enter(sid, "draft", ids["draft"]), runtime.lanes.enter(sid, "profile", ids["profile"])] + tickets = [await runtime.lanes.enter(sid, "draft", ids["draft"]), + await runtime.lanes.enter(sid, "profile", ids["profile"])] try: items = (await client.get(f"/api/datasources/{sid}/background-jobs")).json()["items"] assert [i["kind"] for i in items] == ["draft", "profile", "introspect"], items @@ -253,7 +254,7 @@ async def test_background_jobs_follow_the_lane_order(client, shop): assert {"total", "done", "status"} <= set(items[0]) finally: for t in tickets: - t.release() + await t.release() async with SessionLocal() as session: for model, key in ((IntrospectJob, "introspect"), (CatalogDraftJob, "draft"), (CatalogProfileJob, "profile")): await session.execute(delete(model).where(model.id == ids[key])) diff --git a/frontend/src/lib/termsConcurrency.ts b/frontend/src/lib/termsConcurrency.ts new file mode 100644 index 00000000..335f1b8c --- /dev/null +++ b/frontend/src/lib/termsConcurrency.ts @@ -0,0 +1,29 @@ +/** + * 设置页「运行并发」一节的叫法(服务端 app/cluster/slots.py:交互、应用、定时、系统四类,各有全站上限和优先级,另有全站总上限)。 + * 叫法同服务端的 CLASS_LABEL;取值范围、合不合规只在服务端判(422 的原话落在这一节下面),界面不另判一套。 + */ +export const CONCURRENCY_TEXT = { + title: '运行并发', + hint: '全站同时执行的运行按来源分四类,各有上限和优先级:名额满了时先给优先级高的一类,已经在执行的不打断。多进程部署时几个进程合起来算', + maxTotal: '全站同时运行的上限', + limit: '上限', + priority: '优先级', + priorityHint: '数字大的先拿到空出来的名额(0 到 100)', + classes: { + interactive: { label: '交互', hint: '问数据、画布试运行、手动发起的正式运行、蓝图试运行、个人应用的刷新' }, + app: { label: '应用', hint: '应用入口、按需视图、外部系统的钩子、接口调用' }, + schedule: { label: '定时', hint: '定时任务和事件触发' }, + system: { label: '系统', hint: '评测、口径召回、对账、补算、按口径正式出具' }, + } as Record, + now: (running: number, waiting: number) => `现在 ${running} 个在执行${waiting ? `、${waiting} 个在等名额` : ''}`, + save: '保存运行并发设置', + saving: '正在保存…', + saved: '已保存运行并发设置', + discard: '放弃修改', + readOnly: '只有管理员能修改运行并发设置', + leaveTitle: (n: number) => `运行并发设置有 ${n} 项还没保存`, + leaveConsequences: ['离开后这些修改会丢失'], +} + +/** 四类的先后(同服务端 CLASSES) */ +export const CONCURRENCY_CLASSES = ['interactive', 'app', 'schedule', 'system'] as const