mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 21:01:59 +00:00
ThearchyHelios
45fe01effe
- 只比成功结束的运行，基准也只取成功的：失败、取消的运行指标不全，和上一次比只会得出「移除 N 个」，拿失败的那次当 基准每个指标都成了「新增」。基准改成自己查同一定时任务、同一工作流、比这次先发起的运行里成功并封存的最近一次， 不再用对比接口的 default_base（按结束时间挑：老的失败运行事后被接着跑完，会拿后一次触发当它的「上一次」）。 中间隔着失败、取消的运行时，通知写「与上一次成功的运行（#短编号，日期）相比」，Fire.compare 多一个 previous - 和封存绑定：运行的封存要盖住最后一条结束事件才算封存完（sealed_now）。runner 先写终态、再另起会话封存，失败后 接着跑完的运行在两步之间还带着失败时的清单哈希，以前这一拍会直接认领、按旧封存比、通知里写旧的清单哈希；现在 等新的封存，等不来就照常推送但不比、不写哈希。停在审批上的通知也不再写失败时留下的哈希。对比工件连同两边的清单 哈希一起存，触发历史发现任何一边不在终态或清单哈希变了就给 null；对比途中封存变了不记（条件写） - 时机和时限：锁里只挑出状态有变化的触发，认领挪到对比之后、锁外按条件 UPDATE 做。对比最多等 20 秒（含排队）， 同一时刻最多 2 个；超时只记警告，通知不带对比照常推送。停机时扫描在对比途中被取消，这次触发还没认领，下次照常 比、照常推（以前先认领、后比，这条通知会永远丢掉）。同一份封存已经比过的直接用记下的结果 - 失败通知：从没开始执行的，原因刚写过，处理行不再让人「查看原因」，任务在排期上写「下一次触发会照常发起」，停用 的写怎么重新发起；开始执行过的改成「请在 AgentLab 的「记录」中查看失败原因」，和挂起那句指向同一个导航入口 - 新增 test_scheduler_compare_guards.py（封存绑定、时限、取消、并发、复用），test_scheduler_compare.py 补真实缺 指标的失败运行、上一次失败、补跑老运行几种用例；去掉实现逐项验证过会变红
846 lines
45 KiB
Python
846 lines
45 KiB
Python
"""定时任务的通知：运行结束、失败、停在审批（或因服务重启挂起）时按任务的设置推送 Webhook。
|
||
|
||
四种格式：通用 JSON、飞书（文本消息）、企业微信（markdown）、钉钉（markdown）。配了签名密钥时：通用格式在请求头
|
||
里带 HMAC-SHA256 签名，飞书、钉钉按各自平台的算法签（飞书放在请求体，钉钉放在地址上）；企业微信的 key 本来就在
|
||
地址里，没有另外的签名。
|
||
|
||
Webhook 地址本身就是凭证（企业微信的 key、钉钉的 access_token 都在地址里，飞书的在路径里），和签名密钥一样
|
||
加密落库，日志、推送记录、接口返回里只出现主机名。httpx 每发一个请求都会在 INFO 级别记一行完整地址，这里装了
|
||
一个过滤器把推送期间的那几行换成主机名。
|
||
|
||
通知里只有任务名、状态、出具档位、缺口条数、运行链接、封存哈希（界面上叫清单哈希）前 12 位和起止时间，失败时
|
||
另写失败节点的名字和下一步（不许诺能继续运行）；运行成功结束后和同一任务上一次成功的运行的对比（auto_compare）只写
|
||
各类指标的个数和对比页链接。不放报告里的具体数字，也不放失败时的原始报错——报错里可能带着查询到的数据，Webhook
|
||
推去的群不一定有权看。
|
||
|
||
三个平台出错时都回 HTTP 200、错误码放在响应体里，所以送没送达要看响应体：按地址的主机认平台，不看选的格式——
|
||
格式选错了（飞书的地址选了通用格式）正是最需要认出来的情形。群机器人的地址只认自己平台的格式，保存时就拦下
|
||
对不上的（见 bot_format）。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import base64
|
||
import contextvars
|
||
import hashlib
|
||
import hmac
|
||
import json
|
||
import logging
|
||
import unicodedata
|
||
from dataclasses import dataclass
|
||
from datetime import datetime, timedelta, timezone
|
||
from typing import Any
|
||
from urllib.parse import parse_qsl, quote_plus, urlsplit
|
||
|
||
import httpx
|
||
from sqlalchemy import and_, func, or_, select, update
|
||
|
||
from app.core.crypto import decrypt
|
||
from app.core.events import EventType
|
||
from app.db.base import SessionLocal, utcnow
|
||
from app.db.models import Approval, Run, RunEvent, Schedule, ScheduleFire, WebhookDelivery
|
||
from app.engine.labels import ISSUANCE_LABEL
|
||
from app.scheduler import auto_compare, service
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
FORMATS = ("generic", "feishu", "wecom", "dingtalk")
|
||
EVENTS = ("finished", "failed", "awaiting_approval")
|
||
#: 格式、事件在报错里的叫法，和界面上的选项一致
|
||
FORMAT_LABEL = {"generic": "通用", "feishu": "飞书", "wecom": "企业微信", "dingtalk": "钉钉"}
|
||
EVENT_LABEL = {"finished": "运行结束", "failed": "运行失败", "awaiting_approval": "等待审批"}
|
||
#: 单次请求的超时（秒）
|
||
TIMEOUT_S = 10.0
|
||
#: 失败后再试 3 次，每次之前等这么久（秒）。4xx（429 除外）不重试：地址或参数错了，重试也一样
|
||
RETRY_DELAYS: tuple[float, ...] = (1.0, 2.0, 4.0)
|
||
#: 只看这么多天内的触发：更早结束的运行状态不会再变，不用每一拍都扫。例外见 notify_due：停在审批上、有待审批的不受
|
||
#: 这个限制，运行记录这些天内有过变化的也照扫——审批可以等上一个多月，批完接着执行、停到下一个审批点或跑完都要推
|
||
SCAN_DAYS = 30
|
||
#: 运行结束后等封存（清单哈希）最多等多久。runner 先写终态再封存，两步之间被这一拍撞上就等下一拍
|
||
SEAL_WAIT_S = 60
|
||
#: 测试用：换成 httpx.MockTransport 接住请求
|
||
_transport: httpx.AsyncBaseTransport | None = None
|
||
|
||
_HEADLINE = {"finished": "运行完成", "failed": "运行失败", "awaiting_approval": "等待审批"}
|
||
#: 运行状态在通知里的叫法，和界面一致。interrupted 分两种，见 _status_label
|
||
_STATUS_LABEL = {"succeeded": "已完成", "failed": "失败", "cancelled": "已取消", "failed_to_start": "未能发起"}
|
||
#: 挂起的运行要人去处理：不处理，这个任务之后的触发都按「上一次还没结束」跳过
|
||
SUSPENDED_REASON = "运行停在断点上、没有待审批，通常是服务重启所致"
|
||
SUSPENDED_ACTION = "请在 AgentLab 的「记录」中继续运行或放弃这次运行；它结束之前，这个定时任务之后的触发都会跳过"
|
||
#: 失败的运行怎么处理：不许诺「修正后可继续运行」。开始执行过的也可能没有断点（执行第一个节点前就失败）或已不兼容
|
||
#: （系统升级），能不能继续以记录页的提示为准。「记录」是导航里那一页的名字，和挂起那句一致
|
||
FAILED_ACTION = "请在 AgentLab 的「记录」中查看失败原因"
|
||
#: 从没开始执行就失败的（服务重启时还在排队，被改成失败、没有开始时间）：没有可继续的进度。不照抄运行记录上的报错——
|
||
#: 通知一律不放原始报错，这一句是按「没有开始时间」认出来的，自己写
|
||
NOT_STARTED_REASON = "运行在开始执行前就已中止（通常是服务重启时仍在排队），没有可继续的进度"
|
||
#: 失败的运行不挡之后的触发（挂起的才挡）：任务还在排期上时补这一句，免得人以为这个定时任务就此停了
|
||
NEXT_AS_USUAL = "下一次触发会照常发起"
|
||
#: 从没开始执行、任务又不在排期上（停用了）：原因刚写过，记录页上也没有更多可看的，下一步是重新发起。「立即运行」
|
||
#: 是定时任务页上那个按钮的名字，停用的任务也能点
|
||
RERUN_ACTION = "如需重新发起，请在 AgentLab 的「定时任务」中点击「立即运行」"
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 地址：校验、提示、脱敏
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
_URL_FORMAT = ("Webhook 地址需要是 http:// 或 https:// 开头的完整地址，例如飞书机器人的 "
|
||
"https://open.feishu.cn/open-apis/bot/v2/hook/…")
|
||
|
||
|
||
def check_url(url: Any) -> str:
|
||
"""校验 Webhook 地址，返回去掉首尾空白的地址。不对抛 ValueError（中文说明，不回显地址）。
|
||
|
||
除了 http(s)、有主机名，还要 httpx 真发得出去：端口写错、夹着不可见的字符（从聊天软件复制时常带上零宽空格）、
|
||
主机名不合 IDNA 规则的，保存时就拦下。以前这些照样存进去，到「测试 Webhook」才在 httpx 里抛出 500 或英文报错。
|
||
"""
|
||
text = url.strip() if isinstance(url, str) else ""
|
||
if not text:
|
||
raise ValueError(_URL_FORMAT)
|
||
if any(c.isspace() or unicodedata.category(c) in ("Cc", "Cf") for c in text):
|
||
raise ValueError("Webhook 地址中有空格或不可见的字符（从聊天软件复制时常会带上），请删除后重新填写")
|
||
if len(text) > 2000:
|
||
raise ValueError("Webhook 地址过长（超过 2000 个字符），请检查是否粘贴了多余的内容")
|
||
try:
|
||
parts = urlsplit(text)
|
||
except ValueError:
|
||
raise ValueError(_URL_FORMAT) from None
|
||
if parts.scheme not in ("http", "https") or not parts.hostname:
|
||
raise ValueError(_URL_FORMAT)
|
||
try:
|
||
port = parts.port
|
||
except ValueError: # 不是数字，或者超出 65535
|
||
port = -1
|
||
if port is not None and not 1 <= port <= 65535:
|
||
raise ValueError("Webhook 地址中的端口需要是 1 到 65535 之间的数字")
|
||
try:
|
||
# 照发送时的样子建一个请求（不发出去）：httpx 到这一步才按 IDNA 解码主机名
|
||
httpx.Request("POST", text)
|
||
except (httpx.InvalidURL, ValueError): # IDNA 的错误是 UnicodeError，也是 ValueError
|
||
raise ValueError("Webhook 地址的主机名不符合规范，无法发送，请检查后重新填写") from None
|
||
return text
|
||
|
||
|
||
#: 群机器人的地址（主机，路径前缀）→ 格式。这些地址只认自己平台的消息格式：格式选错了，平台回错误码，群里收不到
|
||
_BOTS = (
|
||
("open.feishu.cn", "/open-apis/bot/", "feishu"),
|
||
("open.larksuite.com", "/open-apis/bot/", "feishu"),
|
||
("qyapi.weixin.qq.com", "/cgi-bin/webhook/", "wecom"),
|
||
("oapi.dingtalk.com", "/robot/", "dingtalk"),
|
||
)
|
||
#: 平台的主机 → 按哪个平台读 HTTP 200 里的错误码。同一主机上不是群机器人的接口（飞书集成平台的 Webhook 触发器
|
||
#: 收任意 JSON）也是同一套错误码约定
|
||
_PLATFORM_HOSTS = {host: fmt for host, _, fmt in _BOTS}
|
||
|
||
|
||
def bot_format(url: str) -> str | None:
|
||
"""这条地址是哪个平台的群机器人（feishu / wecom / dingtalk），不是群机器人的返回 None。"""
|
||
try:
|
||
parts = urlsplit(url)
|
||
except ValueError:
|
||
return None
|
||
host = (parts.hostname or "").lower()
|
||
return next((fmt for bot_host, prefix, fmt in _BOTS if host == bot_host and parts.path.startswith(prefix)), None)
|
||
|
||
|
||
def format_problem(fmt: str, url: str) -> str | None:
|
||
"""格式和地址对不上时的说明（群机器人的地址选了别的格式），对得上返回 None。"""
|
||
expected = bot_format(url)
|
||
if expected is None or expected == fmt:
|
||
return None
|
||
label = FORMAT_LABEL[expected]
|
||
return f"这是{label}群机器人的地址，只接收{label}的消息格式，Webhook 格式请选择「{label}」"
|
||
|
||
|
||
def host_of(url: str) -> str:
|
||
"""主机名（带非默认端口）。日志、报错里只写它。端口写错的地址也照样给出主机名。"""
|
||
try:
|
||
parts = urlsplit(url)
|
||
host = parts.hostname or "?"
|
||
except ValueError:
|
||
return "?"
|
||
try:
|
||
port = parts.port
|
||
except ValueError:
|
||
return host
|
||
return f"{host}:{port}" if port else host
|
||
|
||
|
||
def url_hint(url: str) -> str:
|
||
"""给界面看的地址提示：只露主机和末 4 位（open.feishu.cn/…ab12）。"""
|
||
return f"{host_of(url)}/…{url[-4:]}"
|
||
|
||
|
||
def _sensitive(url: str, secret: str | None) -> list[str]:
|
||
"""一条地址里不能出现在日志、报错里的片段：整条地址、查询参数的值（key、access_token）、路径里的长片段（飞书
|
||
的令牌在路径里）、签名密钥。长的在前，免得先换掉短的、长的就认不出来了。"""
|
||
out = {url, secret or ""}
|
||
try:
|
||
parts = urlsplit(url)
|
||
out |= {value for _, value in parse_qsl(parts.query)}
|
||
out |= {seg for seg in parts.path.split("/") if len(seg) >= 12}
|
||
out.add(f"{parts.netloc}{parts.path}")
|
||
except ValueError:
|
||
pass
|
||
return sorted((s for s in out if len(s) >= 4), key=len, reverse=True)
|
||
|
||
|
||
def _scrub(text: str, url: str, secret: str | None) -> str:
|
||
for piece in _sensitive(url, secret):
|
||
text = text.replace(piece, "…")
|
||
return text
|
||
|
||
|
||
#: 正在推送哪条地址、用的哪个密钥。httpx 记请求日志时还在同一个任务里，过滤器据此只改推送的那几行
|
||
_SENDING: contextvars.ContextVar[tuple[str, str | None] | None] = contextvars.ContextVar("webhook_sending", default=None)
|
||
|
||
|
||
class _RedactWebhookUrls(logging.Filter):
|
||
"""把 httpx 记下的推送请求的地址换成主机名。别的请求（模型接入、数据源工具）原样放过。"""
|
||
|
||
def filter(self, record: logging.LogRecord) -> bool:
|
||
sending = _SENDING.get()
|
||
if sending and isinstance(record.args, tuple):
|
||
url, secret = sending
|
||
record.args = tuple(
|
||
url_hint(str(arg)) if isinstance(arg, httpx.URL)
|
||
else _scrub(arg, url, secret) if isinstance(arg, str) else arg
|
||
for arg in record.args)
|
||
return True
|
||
|
||
|
||
def _install_filter() -> None:
|
||
target = logging.getLogger("httpx")
|
||
if not any(isinstance(f, _RedactWebhookUrls) for f in target.filters):
|
||
target.addFilter(_RedactWebhookUrls())
|
||
|
||
|
||
_install_filter()
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 消息内容
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
@dataclass
|
||
class Notice:
|
||
"""一条通知要说的事。event 是 finished / failed / awaiting_approval / test；status 是运行状态，没能发起的是
|
||
failed_to_start。没有的项为 None。
|
||
|
||
suspended：运行停在断点上、没有待审批（通常是服务重启挂起的），按 failed 事件推。error_node：失败节点的名字。
|
||
next_fire：任务启用着、往后还会触发（失败通知据此补「下一次触发会照常发起」）。
|
||
compare：和同一定时任务上一次成功的运行的对比摘要（auto_compare.compare_fire 的返回：基准是哪一次、只有个数的
|
||
summary、基准是不是紧挨着的上一次、基准开始的时间），compare_url：对比页的链接。
|
||
fmt、signed 只给测试消息用：说明里只说这条消息证明得了的事。"""
|
||
|
||
event: str
|
||
schedule_id: str
|
||
schedule_name: str
|
||
timezone: str
|
||
fire_id: str | None = None
|
||
fire_at: datetime | None = None
|
||
fire_kind: str | None = None
|
||
run_id: str | None = None
|
||
status: str | None = None
|
||
tier: str | None = None
|
||
gaps: int | None = None
|
||
run_url: str | None = None
|
||
manifest_hash: str | None = None
|
||
started_at: datetime | None = None
|
||
finished_at: datetime | None = None
|
||
reason: str | None = None
|
||
pending_approvals: int | None = None
|
||
suspended: bool = False
|
||
error_node: str | None = None
|
||
next_fire: bool = False
|
||
compare: dict[str, Any] | None = None
|
||
compare_url: str | None = None
|
||
fmt: str | None = None
|
||
signed: bool = False
|
||
|
||
@property
|
||
def never_started(self) -> bool:
|
||
"""失败的运行从没开始执行（没有开始时间）：服务重启时还在排队、被改成失败的就是这种，没有断点可续。"""
|
||
return self.status == "failed" and self.run_id is not None and self.started_at is None
|
||
|
||
|
||
def _local(moment: datetime | None, tz: str, *, seconds: bool = True) -> str | None:
|
||
if moment is None:
|
||
return None
|
||
try:
|
||
from app.scheduler import cron
|
||
|
||
local = moment.astimezone(cron.zone(tz))
|
||
except ValueError:
|
||
local = moment.astimezone(timezone.utc)
|
||
return local.strftime("%Y-%m-%d %H:%M:%S" if seconds else "%Y-%m-%d %H:%M")
|
||
|
||
|
||
def message(notice: Notice) -> tuple[str, list[tuple[str, str]], str | None]:
|
||
"""(标题, [(项, 内容)…], 运行链接)。三种文本格式都从这一份排出来。"""
|
||
if notice.event == "test":
|
||
return f"定时任务「{notice.schedule_name}」Webhook 测试", [("说明", _test_text(notice))], None
|
||
if notice.status == "failed_to_start":
|
||
headline = "未能发起运行"
|
||
elif notice.suspended:
|
||
headline = "运行已挂起"
|
||
else:
|
||
headline = _HEADLINE.get(notice.event, notice.event)
|
||
title = f"定时任务「{notice.schedule_name}」{headline}"
|
||
lines: list[tuple[str, str]] = []
|
||
if notice.status:
|
||
lines.append(("状态", _status_label(notice)))
|
||
if notice.tier:
|
||
gaps = f"（缺口 {notice.gaps} 条）" if notice.gaps is not None else ""
|
||
lines.append(("出具档位", f"{ISSUANCE_LABEL.get(notice.tier, notice.tier)}{gaps}"))
|
||
if notice.compare:
|
||
# 只给各类指标的个数：具体的数在对比页上，跟着运行的权限走，不随消息推进群里
|
||
lines.append((_compare_head(notice), auto_compare.summary_text(notice.compare.get("summary") or {})))
|
||
if notice.pending_approvals:
|
||
lines.append(("待审批", f"{notice.pending_approvals} 项，请到 AgentLab 处理；运行结束之前，"
|
||
"这个定时任务之后的触发都会跳过"))
|
||
if notice.status == "failed_to_start" and notice.reason:
|
||
lines.append(("原因", notice.reason))
|
||
elif notice.suspended:
|
||
lines += [("原因", SUSPENDED_REASON), ("处理", SUSPENDED_ACTION)]
|
||
elif notice.status == "failed":
|
||
# 不写原始报错（可能带着查询到的数据），写失败节点的名字和下一步
|
||
if notice.error_node:
|
||
lines.append(("失败节点", f"「{notice.error_node}」"))
|
||
if notice.never_started:
|
||
# 原因刚写过，也没有断点可续：不再让人去查看原因，说下一次照常，不在排期上的说怎么重新发起
|
||
lines += [("原因", NOT_STARTED_REASON), ("处理", NEXT_AS_USUAL if notice.next_fire else RERUN_ACTION)]
|
||
else:
|
||
lines.append(("处理", f"{FAILED_ACTION}；{NEXT_AS_USUAL}" if notice.next_fire else FAILED_ACTION))
|
||
if notice.fire_at is not None:
|
||
when = f"{_local(notice.fire_at, notice.timezone, seconds=False)}（{notice.timezone}）"
|
||
# 立即运行不写「计划触发」：群里的人会以为报告按时出了
|
||
if notice.fire_kind == "manual":
|
||
lines.append(("手动触发", when))
|
||
elif notice.fire_kind == "misfire":
|
||
lines.append(("计划触发", f"{when}，错过后补发"))
|
||
else:
|
||
lines.append(("计划触发", when))
|
||
if notice.started_at is not None:
|
||
lines.append(("开始", _local(notice.started_at, notice.timezone) or ""))
|
||
if notice.finished_at is not None:
|
||
lines.append(("结束", _local(notice.finished_at, notice.timezone) or ""))
|
||
if notice.manifest_hash:
|
||
lines.append(("清单哈希", notice.manifest_hash[:12]))
|
||
if notice.run_id:
|
||
# 和界面上的短编号一样写（#前 6 位）；完整 id 在运行链接里
|
||
lines.append(("运行", f"#{notice.run_id[:6]}"))
|
||
return title, lines, notice.run_url
|
||
|
||
|
||
def _compare_head(notice: Notice) -> str:
|
||
"""对比那一行的项名。基准是紧挨着的上一次时写「与上一次相比」；不是（上一次失败了、取消了）时写明比的是哪一次：
|
||
短编号（同界面，#前 6 位）和开始的日期（按定时任务的时区）。"""
|
||
compare = notice.compare or {}
|
||
if compare.get("previous", True):
|
||
return "与上一次相比"
|
||
when = _date_of(compare.get("base_started_at"), notice.timezone)
|
||
return f"与上一次成功的运行（#{str(compare.get('base_run_id') or '')[:6]}{f'，{when}' if when else ''}）相比"
|
||
|
||
|
||
def _date_of(text: Any, tz: str) -> str | None:
|
||
"""ISO 时间（对比结果里运行的开始时间）按定时任务的时区写成日期，读不出来的返回 None。"""
|
||
try:
|
||
moment = datetime.fromisoformat(str(text))
|
||
except ValueError:
|
||
return None
|
||
if moment.tzinfo is None:
|
||
moment = moment.replace(tzinfo=timezone.utc)
|
||
return (_local(moment, tz, seconds=False) or "")[:10] or None
|
||
|
||
|
||
def _status_label(notice: Notice) -> str:
|
||
if notice.status == "interrupted":
|
||
return "等待审批" if notice.pending_approvals else "已挂起，可继续运行"
|
||
return _STATUS_LABEL.get(notice.status or "", notice.status or "")
|
||
|
||
|
||
def _test_text(notice: Notice) -> str:
|
||
"""测试消息的说明只说收到它证明得了的事：飞书、钉钉的机器人开了签名校验时，验签不过就不收；通用格式的接收端验没
|
||
验签这边不知道；企业微信不签名，填了密钥也不提。"""
|
||
if notice.signed and notice.fmt in ("feishu", "dingtalk"):
|
||
return "这是一条测试消息。收到即表示 Webhook 地址可用；机器人开启了签名校验的，签名密钥也配置正确"
|
||
if notice.signed:
|
||
return "这是一条测试消息。收到即表示 Webhook 地址可用；请求头带有签名，接收端可据此核对签名密钥"
|
||
return "这是一条测试消息。收到即表示 Webhook 地址配置正确"
|
||
|
||
|
||
def _links(notice: Notice) -> list[tuple[str, str]]:
|
||
"""消息末尾的链接：运行，有对比时再加对比页。没设对外访问地址时都没有。"""
|
||
return [(label, url) for label, url in (("查看运行", notice.run_url), ("查看对比", notice.compare_url)) if url]
|
||
|
||
|
||
def _text(notice: Notice) -> str:
|
||
title, lines, _ = message(notice)
|
||
out = [f"【AgentLab】{title}", *(f"{k}：{v}" for k, v in lines)]
|
||
out += [f"{label}：{url}" for label, url in _links(notice)]
|
||
return "\n".join(out)
|
||
|
||
|
||
def _wecom_markdown(notice: Notice) -> str:
|
||
title, lines, _ = message(notice)
|
||
color = "info" if notice.event in ("finished", "test") else "warning"
|
||
out = [f"**【AgentLab】{title}**"]
|
||
for key, value in lines:
|
||
shown = f'<font color="{color}">{value}</font>' if key == "状态" else value
|
||
out.append(f"> {key}：{shown}")
|
||
out += [f"[{label}]({url})" for label, url in _links(notice)]
|
||
return "\n".join(out)
|
||
|
||
|
||
def _dingtalk_markdown(notice: Notice) -> tuple[str, str]:
|
||
title, lines, _ = message(notice)
|
||
out = [f"#### 【AgentLab】{title}", "", *(f"- {k}：{v}" for k, v in lines)]
|
||
for label, url in _links(notice):
|
||
out += ["", f"[{label}]({url})"]
|
||
return title, "\n".join(out)
|
||
|
||
|
||
def _iso(moment: datetime | None) -> str | None:
|
||
return service.as_utc(moment).isoformat() if moment is not None else None
|
||
|
||
|
||
def _generic(notice: Notice, now: datetime) -> dict[str, Any]:
|
||
title, _, _ = message(notice)
|
||
return {
|
||
"source": "agentlab",
|
||
"event": notice.event,
|
||
"title": title,
|
||
"text": _text(notice),
|
||
"schedule_id": notice.schedule_id,
|
||
"schedule_name": notice.schedule_name,
|
||
"fire_id": notice.fire_id,
|
||
"fire_at": _iso(notice.fire_at),
|
||
"fire_kind": notice.fire_kind,
|
||
"run_id": notice.run_id,
|
||
"status": notice.status,
|
||
"tier": notice.tier,
|
||
"tier_label": ISSUANCE_LABEL.get(notice.tier) if notice.tier else None,
|
||
"gaps": notice.gaps,
|
||
"pending_approvals": notice.pending_approvals,
|
||
"run_url": notice.run_url,
|
||
"manifest_hash": notice.manifest_hash[:12] if notice.manifest_hash else None,
|
||
"started_at": _iso(notice.started_at),
|
||
"finished_at": _iso(notice.finished_at),
|
||
"reason": (notice.reason if notice.status == "failed_to_start"
|
||
else f"{SUSPENDED_REASON}。{SUSPENDED_ACTION}" if notice.suspended
|
||
else NOT_STARTED_REASON if notice.never_started else None),
|
||
"error_node": notice.error_node,
|
||
# 对比只给基准是哪一次（是不是紧挨着的上一次）、各类指标的个数和对比页链接
|
||
"compare": ({"base_run_id": notice.compare.get("base_run_id"),
|
||
"summary": dict(notice.compare.get("summary") or {}),
|
||
"previous": notice.compare.get("previous", True) is not False, "url": notice.compare_url}
|
||
if notice.compare else None),
|
||
"sent_at": _iso(now),
|
||
}
|
||
|
||
|
||
def build_request(fmt: str, url: str, secret: str | None, notice: Notice, *,
|
||
now: datetime | None = None) -> tuple[str, dict[str, str], bytes]:
|
||
"""(请求地址, 请求头, 请求体)。签名里的时间戳按 now 取，每次重试重新算：飞书要求时间戳在一小时内。"""
|
||
now = now or utcnow()
|
||
headers = {"Content-Type": "application/json"}
|
||
if fmt == "feishu":
|
||
body: dict[str, Any] = {"msg_type": "text", "content": {"text": _text(notice)}}
|
||
if secret:
|
||
ts = str(int(now.timestamp()))
|
||
# 飞书自定义机器人的签名：以「timestamp + 换行 + 密钥」为键，对空串做 HMAC-SHA256，再 base64
|
||
key = f"{ts}\n{secret}".encode()
|
||
body = {"timestamp": ts, "sign": base64.b64encode(hmac.new(key, b"", hashlib.sha256).digest()).decode(),
|
||
**body}
|
||
return url, headers, _json(body)
|
||
if fmt == "wecom":
|
||
return url, headers, _json({"msgtype": "markdown", "markdown": {"content": _wecom_markdown(notice)}})
|
||
if fmt == "dingtalk":
|
||
title, text = _dingtalk_markdown(notice)
|
||
target = url
|
||
if secret:
|
||
# 钉钉自定义机器人的签名：毫秒时间戳，以密钥为键对「timestamp + 换行 + 密钥」做 HMAC-SHA256，base64 后
|
||
# 再做 URL 编码，和 timestamp 一起加在地址上
|
||
ts = str(int(now.timestamp() * 1000))
|
||
digest = hmac.new(secret.encode(), f"{ts}\n{secret}".encode(), hashlib.sha256).digest()
|
||
sign = quote_plus(base64.b64encode(digest).decode())
|
||
target = f"{url}{'&' if '?' in url else '?'}timestamp={ts}&sign={sign}"
|
||
return target, headers, _json({"msgtype": "markdown", "markdown": {"title": title, "text": text}})
|
||
payload = _json(_generic(notice, now))
|
||
if secret:
|
||
# 通用格式：接收端按 HMAC-SHA256(密钥, 「时间戳.请求体」) 验签，时间戳防重放
|
||
ts = str(int(now.timestamp()))
|
||
mac = hmac.new(secret.encode(), f"{ts}.".encode() + payload, hashlib.sha256).hexdigest()
|
||
headers |= {"X-AgentLab-Timestamp": ts, "X-AgentLab-Signature": f"sha256={mac}"}
|
||
return url, headers, payload
|
||
|
||
|
||
def _json(value: dict[str, Any]) -> bytes:
|
||
return json.dumps(value, ensure_ascii=False, separators=(",", ":")).encode()
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 推送
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
@dataclass
|
||
class Outcome:
|
||
"""一次推送（含重试）的结果。error 是中文说明，只有主机名，不含地址和密钥。"""
|
||
|
||
ok: bool
|
||
http_status: int | None
|
||
attempts: int
|
||
error: str | None = None
|
||
#: 这一次失败值不值得再试（只在单次尝试的结果上有意义）
|
||
retryable: bool = False
|
||
|
||
|
||
def _client() -> httpx.AsyncClient:
|
||
return httpx.AsyncClient(timeout=TIMEOUT_S, transport=_transport, follow_redirects=False)
|
||
|
||
|
||
async def send(fmt: str, url: str, secret: str | None, notice: Notice, *, retry: bool = True) -> Outcome:
|
||
"""推送一条通知。retry 时失败再试 3 次（退避），4xx（429 除外）不重试。失败只记主机名。"""
|
||
host = host_of(url)
|
||
delays = RETRY_DELAYS if retry else ()
|
||
last = Outcome(False, None, 0, "没有发出请求")
|
||
async with _client() as client:
|
||
for attempt in range(len(delays) + 1):
|
||
if attempt:
|
||
await asyncio.sleep(delays[attempt - 1])
|
||
target, headers, body = build_request(fmt, url, secret, notice)
|
||
last = await _post_once(client, fmt, target, headers, body, url, secret)
|
||
last.attempts = attempt + 1
|
||
if last.ok:
|
||
return last
|
||
logger.warning("定时任务「%s」的 Webhook（%s）第 %d 次推送失败：%s",
|
||
notice.schedule_name, host, attempt + 1, last.error)
|
||
if not last.retryable:
|
||
break
|
||
return last
|
||
|
||
|
||
async def _post_once(client: httpx.AsyncClient, fmt: str, target: str, headers: dict[str, str], body: bytes,
|
||
url: str, secret: str | None) -> Outcome:
|
||
host = host_of(url)
|
||
token = _SENDING.set((url, secret))
|
||
try:
|
||
response = await client.post(target, content=body, headers=headers)
|
||
except httpx.TimeoutException:
|
||
return Outcome(False, None, 1, f"请求 {host} 超时（{TIMEOUT_S:g} 秒）", retryable=True)
|
||
except httpx.ConnectError:
|
||
return Outcome(False, None, 1, f"无法连接 {host}", retryable=True)
|
||
except httpx.HTTPError as e:
|
||
return Outcome(False, None, 1, f"请求 {host} 失败（{type(e).__name__}）", retryable=True)
|
||
except httpx.InvalidURL:
|
||
# 保存时的地址校验加严之前存进库的地址：httpx 抛的不是 HTTPError。重试也一样，请人改地址
|
||
return Outcome(False, None, 1, f"发往 {host} 的 Webhook 地址无效，请重新填写地址")
|
||
except Exception as e: # noqa: BLE001 - 端口越界之类在传输层才抛（不是 HTTPError），不能让测试接口回 500
|
||
return Outcome(False, None, 1, f"无法向 {host} 发送请求（{type(e).__name__}），请检查 Webhook 地址")
|
||
finally:
|
||
_SENDING.reset(token)
|
||
status = response.status_code
|
||
reply = _reply_json(response)
|
||
if not 200 <= status < 300:
|
||
detail = _platform_message(reply)
|
||
text = f"{host} 返回 HTTP {status}" + (f"：{detail}" if detail else "")
|
||
return Outcome(False, status, 1, _scrub(text, url, secret)[:300], retryable=status == 429 or status >= 500)
|
||
if problem := _platform_error(fmt, url, reply):
|
||
return Outcome(False, status, 1, _scrub(f"{host} {problem}", url, secret)[:300], retryable=True)
|
||
return Outcome(True, status, 1)
|
||
|
||
|
||
def _reply_json(response: httpx.Response) -> dict[str, Any] | None:
|
||
try:
|
||
data = response.json()
|
||
except ValueError:
|
||
return None
|
||
return data if isinstance(data, dict) else None
|
||
|
||
|
||
def _platform_message(reply: dict[str, Any] | None) -> str:
|
||
if not reply:
|
||
return ""
|
||
text = reply.get("errmsg") or reply.get("msg") or reply.get("StatusMessage") or ""
|
||
return str(text)[:200]
|
||
|
||
|
||
def _platform_error(fmt: str, url: str, reply: dict[str, Any] | None) -> str | None:
|
||
"""HTTP 200 里的平台错误码：飞书 code（老接口 StatusCode），企业微信、钉钉 errcode。0 是成功。
|
||
|
||
按地址的主机认平台，不看选的格式：以前只看所选格式自己的字段（通用格式完全不看），飞书的地址选了通用格式、
|
||
平台回 code 19002，照样记「已送达」。主机不是这几个平台的（自建接收端、代理转发）才按所选格式读，通用格式
|
||
不解读——自建接收端的 code 字段是什么意思由它自己定。说明里写主机名（调用方拼在前面），不写平台名。"""
|
||
try:
|
||
host = (urlsplit(url).hostname or "").lower()
|
||
except ValueError:
|
||
host = ""
|
||
platform = _PLATFORM_HOSTS.get(host) or (fmt if fmt != "generic" else None)
|
||
if not reply or platform is None:
|
||
return None
|
||
code = reply.get("code", reply.get("StatusCode")) if platform == "feishu" else reply.get("errcode")
|
||
if code in (None, 0, "0"):
|
||
return None
|
||
detail = _platform_message(reply)
|
||
return f"返回错误码 {code}" + (f"：{detail}" if detail else "")
|
||
|
||
|
||
async def _record(schedule_id: str, fire_id: str | None, run_id: str | None, event: str, outcome: Outcome) -> None:
|
||
async with SessionLocal() as session:
|
||
session.add(WebhookDelivery(schedule_id=schedule_id, fire_id=fire_id, run_id=run_id, event=event,
|
||
status="ok" if outcome.ok else "failed", http_status=outcome.http_status,
|
||
attempts=outcome.attempts, error=outcome.error))
|
||
try:
|
||
await session.commit()
|
||
except Exception: # noqa: BLE001 - 任务刚被删掉（外键）之类：推送记录记不下不影响别的
|
||
logger.warning("定时任务 %s 的推送记录没能保存", schedule_id)
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 每一拍的通知扫描
|
||
# --------------------------------------------------------------------------
|
||
|
||
#: 运行到了这些状态才可能要推送
|
||
_WATCHED = ("succeeded", "failed", "cancelled", "interrupted")
|
||
#: 结束的运行 → 事件。cancelled 是有人手动停的，不推。interrupted 分停在审批和挂起两种，见 _state_of
|
||
_EVENT_OF = {"succeeded": "finished", "failed": "failed", "cancelled": None}
|
||
|
||
|
||
@dataclass
|
||
class _Job:
|
||
fmt: str
|
||
url: str
|
||
secret: str | None
|
||
notice: Notice
|
||
|
||
|
||
@dataclass
|
||
class _Due:
|
||
"""挑出来的一次状态变化接下来要做的事：运行成功结束时和上一次成功的运行比一次（compare_run），按条件 UPDATE
|
||
认领（把 notified_status 从 old 改成 state），认领到了、订阅了这个事件时推一条（job）。都在锁外做：对比要读快照、
|
||
重算，推送要等接收端，都不能挡着发起。pushed：认领到了、发出了推送（含失败的）。"""
|
||
|
||
fire_id: str
|
||
old: str | None
|
||
state: str
|
||
compare_run: str | None
|
||
job: _Job | None
|
||
pushed: bool = False
|
||
|
||
|
||
async def notify_due(now: datetime | None = None) -> int:
|
||
"""查已发起的触发，运行状态有变化的按订阅推送一次。返回这一拍推送了几条（含失败的）。
|
||
|
||
处理过的状态记在 notified_status 上，用条件 UPDATE 认领（WHERE notified_status = 旧值）：两拍撞在一起也只
|
||
推一次。没订阅这个事件、任务没配 Webhook 的也记下已处理——之后再配上 Webhook，不会把旧的补推一遍。
|
||
停在断点上的运行每一拍都要重新看（见 _state_of）：它的已处理状态带着第几段执行，和运行状态本身对不上。
|
||
|
||
运行成功结束、封存完的那一次，认领之前先和同一定时任务上一次成功的运行比一次（auto_compare），结束的通知带上计数
|
||
摘要和对比页链接。没配 Webhook 的也比：触发历史里要用。对比在锁外、认领之前做，最多等 auto_compare.WAIT_S 秒，
|
||
过了就不带对比推送：一次慢的对比最多让这一拍晚这么久，不会把之后所有的通知一直压着，也不拖着发起用的锁。比到一半
|
||
扫描被取消（停机只等通知扫描一个宽限），这次触发还没认领，下次照常比、照常推——以前先认领、后比，取消之后这条
|
||
通知再也不会推。比不了、出错不影响推送。
|
||
"""
|
||
now = now or utcnow()
|
||
dues: list[_Due] = []
|
||
base_url: str | None = None
|
||
cutoff = now - timedelta(days=SCAN_DAYS)
|
||
# 扫描窗口按触发记录的创建时间算，两种例外：运行记录窗口内有过变化（运行状态一变 updated_at 就跟着变），以及
|
||
# 停在审批上、有待审批的——以前只看触发记录，创建 30 天以后才停到审批上的运行收不到「等待审批」
|
||
waiting = select(Approval.id).where(Approval.run_id == Run.id, Approval.status == "pending").exists()
|
||
# 和发起共用一把锁：发起时先记一条「未能发起」再发起运行、成功了才改成 started，扫描不能撞在这中间。锁里只挑出
|
||
# 状态有变化的、拼好要推的消息；认领在锁外按条件 UPDATE 做，挑出来的是锁里看到的状态，和别的扫描撞上也只有一个
|
||
# 认领得到
|
||
async with service.launch_lock():
|
||
async with SessionLocal() as session:
|
||
rows = (await session.execute(
|
||
select(ScheduleFire, Run)
|
||
.outerjoin(Run, Run.id == ScheduleFire.run_id)
|
||
.where(or_(ScheduleFire.created_at >= cutoff, Run.updated_at >= cutoff,
|
||
and_(Run.status == "interrupted", waiting)), or_(
|
||
and_(ScheduleFire.status == "failed_to_start", ScheduleFire.notified_status.is_(None)),
|
||
and_(ScheduleFire.status == "started", Run.status.in_(_WATCHED),
|
||
or_(ScheduleFire.notified_status.is_(None), ScheduleFire.notified_status != Run.status)),
|
||
))
|
||
.order_by(ScheduleFire.created_at)
|
||
)).all()
|
||
if rows:
|
||
from app.api.settings import notify_settings
|
||
|
||
base_url = (await notify_settings(session))["public_base_url"]
|
||
for fire, run in rows:
|
||
seen = await _state_of(session, fire, run, now)
|
||
if seen.state is None or seen.state == fire.notified_status:
|
||
continue
|
||
schedule = await session.get(Schedule, fire.schedule_id)
|
||
url = decrypt(schedule.webhook_url) if schedule and schedule.webhook_url else None
|
||
job = None
|
||
if seen.event and url and schedule.webhook_format in FORMATS \
|
||
and seen.event in (schedule.webhook_events or []):
|
||
notice = await _notice(session, schedule, fire, run, seen, base_url)
|
||
job = _Job(schedule.webhook_format, url, decrypt(schedule.webhook_secret), notice)
|
||
# 只比成功结束、封存完的（见 auto_compare）：失败、取消的指标不全，等不来新封存的比了也是旧的
|
||
compare_run = run.id if run is not None and run.status == "succeeded" and seen.sealed else None
|
||
dues.append(_Due(fire.id, fire.notified_status, seen.state, compare_run, job))
|
||
if dues:
|
||
await asyncio.gather(*(_follow_up(due, base_url) for due in dues))
|
||
return sum(1 for due in dues if due.pushed)
|
||
|
||
|
||
async def _follow_up(due: _Due, base_url: str | None) -> None:
|
||
"""先比（需要的话），再认领，认领到了再推：对比摘要要写进这条通知；认领放在对比之后，比到一半被取消时这次触发
|
||
还没认领。对比自己兜住了超时和异常，出什么事都不挡推送。"""
|
||
brief = await auto_compare.compare_fire(due.fire_id, due.compare_run) if due.compare_run else None
|
||
if not await _mark(due.fire_id, due.old, due.state) or due.job is None:
|
||
return
|
||
if brief is not None:
|
||
due.job.notice.compare = brief
|
||
if base_url:
|
||
due.job.notice.compare_url = auto_compare.page_url(base_url, due.compare_run, brief["base_run_id"])
|
||
due.pushed = True
|
||
await _deliver(due.job)
|
||
|
||
|
||
@dataclass
|
||
class _Seen:
|
||
"""_state_of 的结论。state：要记下的已处理状态，还不能下结论时为 None；event：要推送的事件；pending：待审批数；
|
||
sealed：终态的运行封存完了（封存盖住了这一次的结束事件，见 auto_compare.sealed_now），只有这时才比、通知里才写
|
||
清单哈希。"""
|
||
|
||
state: str | None
|
||
event: str | None = None
|
||
pending: int = 0
|
||
sealed: bool = False
|
||
|
||
|
||
async def _state_of(session: Any, fire: ScheduleFire, run: Run | None, now: datetime) -> _Seen:
|
||
"""这次触发现在该记成什么状态、推什么。还不能下结论（没封存、审批刚批完还没接着执行）时状态为 None。
|
||
|
||
停在断点上（interrupted）的分两种，各推一次：
|
||
- 有待审批：停在审批，推 awaiting_approval。批完接着执行、又停在下一个审批点时要再推，所以已处理状态里带着这是
|
||
第几段执行（每段执行从一条 run.started 开始），同一个审批点不重复推。
|
||
- 没有待审批：通常是服务重启挂起的。它不会自己接着执行，这个任务之后的触发都会按「上一次还没结束」跳过，
|
||
不说一声任务就此静默停摆——按 failed 推一次，写明要去继续运行或放弃。
|
||
审批批完到运行改回 running 之间，审批是 answered、还没有 resolved：那不是挂起，等下一拍。
|
||
"""
|
||
if fire.status == "failed_to_start":
|
||
return _Seen("failed_to_start", "failed")
|
||
if run is None:
|
||
return _Seen(None)
|
||
if run.status in ("succeeded", "failed", "cancelled"):
|
||
# runner 先写终态和结束时间、再另起会话封存，两步之间被这一拍撞上就等。失败后接着跑完的运行这时还带着失败时
|
||
# 的清单哈希，不能只看有没有哈希，要看封存盖没盖住这一次的结束事件。没有结束时间的（服务重启时还在排队、被改
|
||
# 成 failed 的）永远不会封存：按它最后一次更新的时间等，以前按「现在」等，每一拍都等、永远不推。等过时限仍没
|
||
# 封存完的照常推，只是不比、不写清单哈希
|
||
sealed = await auto_compare.sealed_now(session, run)
|
||
since = run.finished_at or run.updated_at or fire.created_at
|
||
if not sealed and (now - service.as_utc(since)).total_seconds() < SEAL_WAIT_S:
|
||
return _Seen(None)
|
||
return _Seen(run.status, _EVENT_OF[run.status], sealed=sealed)
|
||
if run.status != "interrupted":
|
||
return _Seen(None)
|
||
counts = dict((await session.execute(
|
||
select(Approval.status, func.count()).where(
|
||
Approval.run_id == run.id, Approval.status.in_(("pending", "answered")))
|
||
.group_by(Approval.status))).all())
|
||
segment = await session.scalar(select(func.count()).select_from(RunEvent).where(
|
||
RunEvent.run_id == run.id, RunEvent.type == EventType.RUN_STARTED.value)) or 0
|
||
if counts.get("pending"):
|
||
return _Seen(f"interrupted:approval:{segment}", "awaiting_approval", counts["pending"])
|
||
if counts.get("answered"):
|
||
return _Seen(None)
|
||
return _Seen(f"interrupted:held:{segment}", "failed")
|
||
|
||
|
||
async def _mark(fire_id: str, old: str | None, state: str) -> bool:
|
||
cond = ScheduleFire.notified_status.is_(None) if old is None else ScheduleFire.notified_status == old
|
||
async with SessionLocal() as session:
|
||
result = await session.execute(update(ScheduleFire).where(ScheduleFire.id == fire_id, cond)
|
||
.values(notified_status=state))
|
||
await session.commit()
|
||
return result.rowcount == 1
|
||
|
||
|
||
async def _notice(session: Any, schedule: Schedule, fire: ScheduleFire, run: Run | None, seen: _Seen,
|
||
base_url: str | None) -> Notice:
|
||
tier, gaps = await _issuance_of(session, run) if run is not None else (None, None)
|
||
pending = seen.pending
|
||
return Notice(
|
||
event=seen.event or "", schedule_id=schedule.id, schedule_name=schedule.name, timezone=schedule.timezone,
|
||
fire_id=fire.id, fire_at=fire.fire_at, fire_kind=fire.kind,
|
||
run_id=run.id if run else None,
|
||
status=run.status if run else fire.status,
|
||
tier=tier, gaps=gaps,
|
||
run_url=f"{base_url}/runs/{run.id}" if base_url and run else None,
|
||
# 只写这一次结束时封的：停在审批上、封存还没补上的运行，身上可能还留着上一次失败时的清单哈希
|
||
manifest_hash=run.manifest_hash if run is not None and seen.sealed else None,
|
||
started_at=run.started_at if run else None,
|
||
finished_at=run.finished_at if run else None,
|
||
reason=fire.reason if fire.status == "failed_to_start" else None,
|
||
pending_approvals=pending or None,
|
||
suspended=run is not None and run.status == "interrupted" and not pending,
|
||
error_node=_failed_node(run) if run is not None and run.status == "failed" else None,
|
||
next_fire=bool(schedule.enabled and schedule.next_run_at is not None),
|
||
)
|
||
|
||
|
||
def _failed_node(run: Run) -> str | None:
|
||
"""失败节点在图上的名字（没有名字的给节点 id）。节点名是搭图时起的，不带查询到的数据。"""
|
||
if not run.error_node_id:
|
||
return None
|
||
for node in (run.graph or {}).get("nodes") or []:
|
||
if isinstance(node, dict) and node.get("id") == run.error_node_id:
|
||
label = (node.get("data") or {}).get("label") if isinstance(node.get("data"), dict) else None
|
||
return str(label or run.error_node_id)[:80]
|
||
return None
|
||
|
||
|
||
async def _issuance_of(session: Any, run: Run) -> tuple[str | None, int | None]:
|
||
"""出具档位和缺口条数：成果上的 _issuance，取不到时退回最后一条 issuance 事件（同 api/evidence.py 的读法）。"""
|
||
issuance = (run.output or {}).get("_issuance") if isinstance(run.output, dict) else None
|
||
if not isinstance(issuance, dict):
|
||
issuance = await session.scalar(select(RunEvent.data).where(
|
||
RunEvent.run_id == run.id, RunEvent.type == "issuance").order_by(RunEvent.seq.desc()).limit(1))
|
||
if not isinstance(issuance, dict) or not issuance.get("tier"):
|
||
return None, None
|
||
gaps = issuance.get("gaps")
|
||
return str(issuance["tier"]), len(gaps) if isinstance(gaps, list) else None
|
||
|
||
|
||
async def _deliver(job: _Job) -> None:
|
||
try:
|
||
outcome = await send(job.fmt, job.url, job.secret, job.notice)
|
||
except Exception: # noqa: BLE001 - 拼消息、签名出错也不能让这一拍的其他推送跟着失败
|
||
logger.exception("定时任务「%s」的 Webhook（%s）推送出错", job.notice.schedule_name, host_of(job.url))
|
||
outcome = Outcome(False, None, 0, "推送时出错，详细信息见服务日志")
|
||
await _record(job.notice.schedule_id, job.notice.fire_id, job.notice.run_id, job.notice.event, outcome)
|
||
|
||
|
||
async def send_test(schedule_id: str) -> dict[str, Any]:
|
||
"""按任务存的 Webhook 发一条测试消息（不重试），记一条 event=test 的推送记录。任务不存在抛 KeyError，
|
||
没配 Webhook、地址解不开抛 ValueError。
|
||
|
||
返回 {ok, http_status, message}。message 是 ok、http_status 之外的补充说明：失败时是原因（只有主机名），送达时
|
||
提醒去接收端看一眼。界面在「测试消息已送达（HTTP 200）」「测试消息发送失败」后面接它，这里不再重复那半句。"""
|
||
async with SessionLocal() as session:
|
||
schedule = await session.get(Schedule, schedule_id)
|
||
if schedule is None:
|
||
raise KeyError(schedule_id)
|
||
if not schedule.webhook_url or schedule.webhook_format not in FORMATS:
|
||
raise ValueError("这个定时任务没有配置 Webhook，请先填写地址并保存")
|
||
url = decrypt(schedule.webhook_url)
|
||
if not url:
|
||
raise ValueError("Webhook 地址无法解密（加密密钥可能已更换），请重新填写地址并保存")
|
||
fmt, secret = schedule.webhook_format, decrypt(schedule.webhook_secret)
|
||
notice = Notice(event="test", schedule_id=schedule.id, schedule_name=schedule.name,
|
||
timezone=schedule.timezone, fmt=fmt, signed=bool(secret) and fmt != "wecom")
|
||
outcome = await send(fmt, url, secret, notice, retry=False)
|
||
await _record(schedule_id, None, None, "test", outcome)
|
||
if outcome.ok:
|
||
return {"ok": True, "http_status": outcome.http_status, "message": "请到接收端确认已收到"}
|
||
return {"ok": False, "http_status": outcome.http_status, "message": outcome.error or "发送失败"}
|