mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-16 05:31:59 +00:00
350 lines
18 KiB
Python
350 lines
18 KiB
Python
"""登记的端点与平台执行器(A6 第 2、3、7 条,§3.9)。
|
||
|
||
端点 = 管理员登记的外部系统:按环境(test / prod)存 base_url、认证头(值加密)、allow_intranet、verify_tls。
|
||
绑定端点的 HTTP 能力(自定义工具 kind=http,config 写 {endpoint, method, path, query, body})不由模型给地址:
|
||
- 地址只能是登记的 base_url 加上路径模板,路径里的参数逐段 URL 编码后代入(/ ? # 都编码掉),模型拿不到任意 URL;
|
||
- 只有登记的端点能开内网访问,开了也照样拦元数据地址和高危端口(tools/net.py);
|
||
- 不跟随跳转:跳到 base_url 之外就不再是登记的那个系统了,3xx 原样交回。
|
||
|
||
环境:正式运行走 prod;探索运行走 test,端点没有 test 时演练(不发送,environment=simulated,Q4)。读取在探索运行里
|
||
没有 test 时走 prod:读不改外部系统,挡掉它只会让问数据用不上这个端点(pick_environment)。
|
||
|
||
声明了 tabular 的读取把响应落成和查询快照同一个形状的工件(kind=endpoint_snapshot):{source, columns, rows, truncated,
|
||
request, ...},交给模型的结果再带上 artifact。台账照 query 处理,报告的单元格标记、口径卡取值、运行对比都不用改。
|
||
外部响应的原文嵌在快照的 response 里,顶层字段全是平台自己拼的,外部系统伪造不了。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import logging
|
||
import re
|
||
from typing import Any
|
||
from urllib.parse import quote
|
||
|
||
import httpx
|
||
from sqlalchemy import select
|
||
from sqlalchemy.ext.asyncio import AsyncSession
|
||
|
||
from app.capabilities import secrets as vault
|
||
from app.capabilities.registry import EndpointResult, effective_effect
|
||
from app.core.config import settings
|
||
from app.db.models import Capability, CustomTool, Endpoint
|
||
from app.engine.expressions import render_template
|
||
from app.tools.net import UnsafeUrlError, assert_safe_url
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
ENVIRONMENTS = ("test", "prod")
|
||
SNAPSHOT_KIND = "endpoint_snapshot"
|
||
#: 落快照最多收多少行,和数据源查询的缺省行数上限一致
|
||
MAX_ROWS = 1000
|
||
_WHOLE = re.compile(r"^\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}$")
|
||
|
||
|
||
class EndpointError(ValueError):
|
||
"""端点或绑定它的能力配置不对:给人看的一句话。"""
|
||
|
||
|
||
# ---------------------------------------------------------------- 端点本身
|
||
|
||
|
||
def env_out(env: dict[str, Any] | None) -> dict[str, Any] | None:
|
||
if not env:
|
||
return None
|
||
return {"base_url": env.get("base_url") or "", "headers": vault.masked(env.get("headers")),
|
||
"allow_intranet": bool(env.get("allow_intranet")), "verify_tls": env.get("verify_tls", True) is not False}
|
||
|
||
|
||
def endpoint_out(row: Endpoint) -> dict[str, Any]:
|
||
envs = row.environments or {}
|
||
return {"id": row.id, "name": row.name, "description": row.description or "",
|
||
"environments": {k: env_out(envs.get(k)) for k in ENVIRONMENTS if envs.get(k)},
|
||
"idempotency_header": row.idempotency_header, "timeout_s": row.timeout_s, "version": row.version,
|
||
"updated_by": row.updated_by}
|
||
|
||
|
||
def seal_environments(given: dict[str, Any] | None, previous: dict[str, Any] | None) -> dict[str, Any]:
|
||
"""提交上来的 environments → 存库的样子。只认 test、prod;base_url 必须是 http(s);头的值加密,提交掩码值保持原值。
|
||
只配了测试环境的也收(接入阶段先在测试环境上试):正式运行要用的写入能力由发布检查 action.prod_missing 拦。"""
|
||
before = previous or {}
|
||
out: dict[str, Any] = {}
|
||
for key, env in (given or {}).items():
|
||
if key not in ENVIRONMENTS:
|
||
raise EndpointError(f"环境只能是测试(test)或生产(prod),当前为「{key}」")
|
||
if not env:
|
||
continue
|
||
base = str(env.get("base_url") or "").strip().rstrip("/")
|
||
if not re.match(r"^https?://[^/?#]+", base):
|
||
raise EndpointError(f"{_ENV_LABEL[key]}的地址需要以 http:// 或 https:// 开头")
|
||
if "?" in base or "#" in base:
|
||
raise EndpointError(f"{_ENV_LABEL[key]}的地址不能带查询参数,参数请写在能力的配置里")
|
||
out[key] = {"base_url": base,
|
||
"headers": vault.seal(env.get("headers"), (before.get(key) or {}).get("headers")),
|
||
"allow_intranet": bool(env.get("allow_intranet")),
|
||
"verify_tls": env.get("verify_tls", True) is not False}
|
||
if not out:
|
||
raise EndpointError("端点至少需要配置一个环境")
|
||
return out
|
||
|
||
|
||
_ENV_LABEL = {"test": "测试环境", "prod": "生产环境"}
|
||
|
||
|
||
def pick_environment(endpoint: Endpoint, *, formal: bool, effect: str) -> str:
|
||
"""这一次走哪个环境。正式运行 prod;探索运行 test,没有 test 时写入演练、读取走 prod。"""
|
||
if formal:
|
||
return "prod"
|
||
if (endpoint.environments or {}).get("test"):
|
||
return "test"
|
||
return "prod" if effect == "read" else "simulated"
|
||
|
||
|
||
async def find_endpoint(session: AsyncSession, ref: str | None) -> Endpoint | None:
|
||
"""按 id 找,找不到再按名字找:自定义工具的配置里两种写法都认。"""
|
||
if not ref:
|
||
return None
|
||
row = await session.get(Endpoint, ref)
|
||
if row is not None:
|
||
return row
|
||
return (await session.execute(select(Endpoint).where(Endpoint.name == ref))).scalar_one_or_none()
|
||
|
||
|
||
def bound_endpoint_ref(tool: CustomTool) -> str | None:
|
||
"""自定义工具绑定的端点(config.endpoint),没绑定为 None。"""
|
||
if tool.kind != "http":
|
||
return None
|
||
ref = (tool.config or {}).get("endpoint")
|
||
return str(ref) if ref else None
|
||
|
||
|
||
# ---------------------------------------------------------------- 拼请求
|
||
|
||
|
||
def _fill(value: Any, args: dict[str, Any]) -> Any:
|
||
"""请求体、查询参数里的模板。整个值就是一个 {{ 参数 }} 的按参数原样代入(金额还是数),别的按文字渲染。"""
|
||
if isinstance(value, str):
|
||
whole = _WHOLE.match(value.strip())
|
||
if whole:
|
||
return args.get(whole.group(1))
|
||
return render_template(value, args)
|
||
if isinstance(value, dict):
|
||
return {k: _fill(v, args) for k, v in value.items()}
|
||
if isinstance(value, list):
|
||
return [_fill(v, args) for v in value]
|
||
return value
|
||
|
||
|
||
def render_path(template: str, args: dict[str, Any]) -> str:
|
||
"""路径模板逐段代入:每个参数值整段 URL 编码(/ ? # % 都编码掉),模型给的值走不出这一段。. 和 .. 不收:
|
||
它们编码前后一样,拼进去就成了往上走一层。"""
|
||
encoded: dict[str, Any] = {}
|
||
for key, value in args.items():
|
||
text = "" if value is None else str(value)
|
||
if text in (".", ".."):
|
||
raise EndpointError(f"路径参数 {key} 的值不能是「{text}」")
|
||
encoded[key] = quote(text, safe="")
|
||
path = render_template(template or "", encoded)
|
||
return path if not path or path.startswith("/") else f"/{path}"
|
||
|
||
|
||
def build_request(endpoint: Endpoint, environment: str, config: dict[str, Any], args: dict[str, Any]
|
||
) -> dict[str, Any]:
|
||
"""{method, url, path, query, body}。没写 body 的:GET / HEAD / DELETE 不带请求体,其余把参数整个作为 JSON 请求体。"""
|
||
env = (endpoint.environments or {}).get(environment if environment != "simulated" else "test") \
|
||
or (endpoint.environments or {}).get("prod") or {}
|
||
method = str(config.get("method") or "GET").upper()
|
||
path = render_path(str(config.get("path") or ""), args)
|
||
query = {str(k): v for k, v in (_fill(config.get("query") or {}, args) or {}).items()
|
||
if v is not None and v != ""}
|
||
if "body" in config and config.get("body") is not None:
|
||
body = _fill(config["body"], args)
|
||
else:
|
||
body = None if method in ("GET", "HEAD", "DELETE") else dict(args)
|
||
base = str(env.get("base_url") or "").rstrip("/")
|
||
return {"method": method, "url": f"{base}{path}", "path": path, "query": query, "body": body}
|
||
|
||
|
||
# ---------------------------------------------------------------- 表格化
|
||
|
||
|
||
def pointer(doc: Any, ptr: str) -> Any:
|
||
"""JSON 指针(RFC 6901)。取不到为 None。"""
|
||
if ptr in ("", "/"):
|
||
return doc
|
||
cur = doc
|
||
for raw in ptr.lstrip("/").split("/"):
|
||
part = raw.replace("~1", "/").replace("~0", "~")
|
||
if isinstance(cur, dict):
|
||
cur = cur.get(part)
|
||
elif isinstance(cur, list) and part.isdigit() and int(part) < len(cur):
|
||
cur = cur[int(part)]
|
||
else:
|
||
return None
|
||
return cur
|
||
|
||
|
||
def tabulate(body: Any, tabular: dict[str, Any]) -> tuple[list[str], list[list[Any]], bool]:
|
||
"""按 tabular = {rows: 行所在的指针, columns: {列名: 行内指针}} 把响应排成表。"""
|
||
items = pointer(body, str(tabular.get("rows") or ""))
|
||
if not isinstance(items, list):
|
||
raise EndpointError("响应里按「行」的位置找不到列表,请检查能力登记里表格的行位置")
|
||
columns = {str(k): str(v) for k, v in (tabular.get("columns") or {}).items()}
|
||
names = list(columns)
|
||
rows = [[pointer(item, columns[n]) for n in names] for item in items[:MAX_ROWS]]
|
||
return names, rows, len(items) > MAX_ROWS
|
||
|
||
|
||
def masked_columns_of(tabular: dict[str, Any], masked: list[Any]) -> list[str]:
|
||
"""遮罩字段(JSON 指针)落在表格的哪几列:快照里记成 mask_columns,证据面板按它遮,口径同查询快照。"""
|
||
hidden = {str(m) for m in masked or []}
|
||
return [str(name) for name, ptr in (tabular.get("columns") or {}).items() if str(ptr) in hidden]
|
||
|
||
|
||
# ---------------------------------------------------------------- 平台执行器
|
||
|
||
|
||
def _client(**kwargs: Any) -> httpx.AsyncClient:
|
||
"""发请求用的客户端。测试里换成 MockTransport。"""
|
||
return httpx.AsyncClient(**kwargs)
|
||
|
||
|
||
def _parse(resp: httpx.Response) -> Any:
|
||
content = resp.content[: settings.http_tool_max_bytes]
|
||
text = content.decode(resp.encoding or "utf-8", errors="replace")
|
||
try:
|
||
return json.loads(text)
|
||
except ValueError:
|
||
return text
|
||
|
||
|
||
async def _resolve(session: AsyncSession, capability: Any) -> tuple[Capability | None, CustomTool, Endpoint]:
|
||
name = capability if isinstance(capability, str) else capability.name
|
||
cap = capability if isinstance(capability, Capability) else await session.get(Capability, name)
|
||
tool = (await session.execute(select(CustomTool).where(CustomTool.name == name))).scalar_one_or_none()
|
||
if tool is None or bound_endpoint_ref(tool) is None:
|
||
raise EndpointError(f"能力「{name}」没有绑定登记的端点,不能由平台执行")
|
||
endpoint = await find_endpoint(session, bound_endpoint_ref(tool))
|
||
if endpoint is None:
|
||
raise EndpointError(f"能力「{name}」绑定的端点不存在,可能已被删除。请到「工具 → 端点」检查")
|
||
return cap, tool, endpoint
|
||
|
||
|
||
async def preview_request(session: AsyncSession, capability: Any, args: dict[str, Any], *,
|
||
environment: str) -> dict[str, Any]:
|
||
"""这次会发出的请求(不含认证头):动作确认卡、演练回执用。"""
|
||
_, tool, endpoint = await _resolve(session, capability)
|
||
req = build_request(endpoint, environment, tool.config or {}, args)
|
||
return {**req, "endpoint": endpoint.name, "environment": environment}
|
||
|
||
|
||
async def endpoint_request(session: AsyncSession, capability: Any, args: dict[str, Any], *, environment: str,
|
||
idem_key: str | None = None, on_behalf: str | None = None,
|
||
run_id: str | None = None, node_id: str | None = None) -> EndpointResult:
|
||
"""平台执行器(registry.endpoint_request 的实现,签名同附录 C)。environment 由调用方按运行类别定(pick_environment);
|
||
传 test 而端点没有测试环境时演练。超时、连接中断交 unknown=True:对方可能已经处理了,写入要转核实,不能重发。"""
|
||
try:
|
||
cap, tool, endpoint = await _resolve(session, capability)
|
||
envs = endpoint.environments or {}
|
||
if environment == "test" and not envs.get("test"):
|
||
environment = "simulated"
|
||
req = build_request(endpoint, environment, tool.config or {}, args)
|
||
except EndpointError as e:
|
||
return EndpointResult(status=None, body=None, error=str(e), environment=environment)
|
||
shown = {k: req[k] for k in ("method", "path", "query", "body")}
|
||
if environment == "simulated":
|
||
return EndpointResult(status=None, body={"simulated": True, "endpoint": endpoint.name, "request": shown},
|
||
environment="simulated")
|
||
env = envs.get(environment)
|
||
if not env:
|
||
return EndpointResult(status=None, body=None, environment=environment,
|
||
error=f"端点「{endpoint.name}」没有配置{_ENV_LABEL.get(environment, environment)}")
|
||
try:
|
||
await assert_safe_url(req["url"], allow_intranet=bool(env.get("allow_intranet")))
|
||
except UnsafeUrlError as e:
|
||
return EndpointResult(status=None, body=None, error=str(e), environment=environment)
|
||
|
||
headers = vault.reveal(env.get("headers"))
|
||
if idem_key:
|
||
headers[endpoint.idempotency_header or "Idempotency-Key"] = idem_key
|
||
if cap is not None and cap.credential_mode == "service_on_behalf":
|
||
from app.auth.signing import ASSERTION_HEADER, assertion
|
||
|
||
from app.auth import current_mode
|
||
|
||
# 开了登录,发起人就是认证过的用户名;关登录时是自报的署名,断言里照实写 verified=false
|
||
headers[ASSERTION_HEADER] = assertion(endpoint.name, on_behalf,
|
||
verified=bool(on_behalf) and current_mode() != "off")
|
||
try:
|
||
async with _client(timeout=endpoint.timeout_s or settings.http_tool_timeout, follow_redirects=False,
|
||
trust_env=False, verify=env.get("verify_tls", True) is not False) as client:
|
||
resp = await client.request(req["method"], req["url"], params=req["query"] or None, headers=headers,
|
||
json=req["body"] if req["body"] is not None else None)
|
||
except (httpx.ConnectError, httpx.ConnectTimeout) as e:
|
||
# 连接没建立:请求没发出去,是确定的失败
|
||
return EndpointResult(status=None, body=None, error=f"无法连接端点「{endpoint.name}」:{e}",
|
||
environment=environment)
|
||
except httpx.HTTPError as e:
|
||
# 发出去之后超时、连接中断:对方可能已经处理了,结果未知
|
||
return EndpointResult(status=None, body=None, unknown=True, environment=environment,
|
||
error=f"端点「{endpoint.name}」没有返回确定的结果:{type(e).__name__}")
|
||
|
||
body = _parse(resp)
|
||
snapshot: str | None = None
|
||
if (cap is not None and cap.tabular and effective_effect(cap) == "read" and 200 <= resp.status_code < 300):
|
||
try:
|
||
snapshot = await _store_snapshot(cap, endpoint, environment, shown, resp.status_code, body,
|
||
run_id=run_id, node_id=node_id)
|
||
except EndpointError as e:
|
||
return EndpointResult(status=resp.status_code, body=body, error=str(e), environment=environment)
|
||
return EndpointResult(status=resp.status_code, body=body, snapshot_artifact=snapshot, environment=environment)
|
||
|
||
|
||
async def _store_snapshot(cap: Capability, endpoint: Endpoint, environment: str, request: dict[str, Any],
|
||
status: int, body: Any, *, run_id: str | None, node_id: str | None) -> str:
|
||
from app.core.artifact_store import put_json
|
||
|
||
columns, rows, truncated = tabulate(body, cap.tabular or {})
|
||
content: dict[str, Any] = {"source": endpoint.name, "capability": cap.name, "columns": columns, "rows": rows,
|
||
"row_count": len(rows), "truncated": truncated,
|
||
"request": {**request, "environment": environment}, "status": status,
|
||
"response": body}
|
||
if hidden := masked_columns_of(cap.tabular or {}, cap.masked or []):
|
||
content["mask_columns"] = hidden
|
||
return await put_json(content, kind=SNAPSHOT_KIND, run_id=run_id or "", node_id=node_id or "",
|
||
meta={"capability": cap.name, "endpoint": endpoint.name})
|
||
|
||
|
||
def tool_result(result: EndpointResult) -> dict[str, Any]:
|
||
"""交给模型的结果。落了快照的照查询快照的样子交(带 artifact),别的给状态码和响应体。"""
|
||
if result.error and not result.unknown:
|
||
return {"error": result.error}
|
||
if result.unknown:
|
||
return {"error": result.error, "unknown": True}
|
||
if result.environment == "simulated":
|
||
return {"simulated": True, "message": "演练:探索运行里端点没有测试环境,未发送请求", **(result.body or {})}
|
||
if result.snapshot_artifact:
|
||
from app.core.artifact_store import load
|
||
|
||
snapshot = load(result.snapshot_artifact) or {}
|
||
return {"artifact": result.snapshot_artifact,
|
||
**{k: snapshot.get(k) for k in ("source", "columns", "rows", "truncated", "request")}}
|
||
return {"status": result.status, "data": result.body, "environment": result.environment}
|
||
|
||
|
||
async def probe(endpoint: Endpoint, environment: str) -> dict[str, Any]:
|
||
"""测一下这个环境连不连得上:对 base_url 发一个 GET,只看有没有回应,不看状态码。"""
|
||
env = (endpoint.environments or {}).get(environment)
|
||
if not env:
|
||
return {"ok": False, "status": None,
|
||
"message": f"端点「{endpoint.name}」没有配置{_ENV_LABEL.get(environment, environment)}"}
|
||
try:
|
||
await assert_safe_url(env["base_url"], allow_intranet=bool(env.get("allow_intranet")))
|
||
async with _client(timeout=endpoint.timeout_s or settings.http_tool_timeout, follow_redirects=False,
|
||
trust_env=False, verify=env.get("verify_tls", True) is not False) as client:
|
||
resp = await client.get(env["base_url"], headers=vault.reveal(env.get("headers")))
|
||
except UnsafeUrlError as e:
|
||
return {"ok": False, "status": None, "message": str(e)}
|
||
except httpx.HTTPError as e:
|
||
return {"ok": False, "status": None, "message": f"无法连接:{type(e).__name__}"}
|
||
return {"ok": True, "status": resp.status_code, "message": f"已连接,返回状态码 {resp.status_code}"}
|