diff --git a/backend/app/api/copilot_gen.py b/backend/app/api/copilot_gen.py index 2acbeab4..bd49edee 100644 --- a/backend/app/api/copilot_gen.py +++ b/backend/app/api/copilot_gen.py @@ -41,6 +41,11 @@ _ASK_OPS = ("ask", "bp_ask") _STOP_OPS = ("done", "look", *_ASK_OPS) #: 不占次数的查看回话(need 没写、次数用完)连着来几次就不再接着写:模型在原地打转,再要下去只是花钱 _IDLE_LOOKS = 2 +#: 读到停下的那一行(done、look、ask)之后,最多再收多少字。照约定那一行就是这一次输出的最后一行,后面只剩流末尾那一段 +#: 用量(Anthropic 的输出 token、OpenAI 兼容系开了 stream_usage 的用量都在最后一段才报):读完它,这一次的花费才报得全, +#: 问数据页这一轮的花费、评测按角色的成本才不是下限。超过这么多字说明模型还在往下写闲话:不为它付钱,当场关流,这一次的 +#: 用量记成没报全(providers/usage.py 照实记估不出,不当成报全了) +DRAIN_CHARS = 400 # ========================================================================== @@ -72,9 +77,24 @@ class CopilotIncomplete(Exception): self.detail = detail +async def _drain(stream: Any, already: int) -> None: + """停下的那一行之后把模型的流读完(内容一概不要),只为收到流末尾才报的用量;读到超过 DRAIN_CHARS 字(模型还在写闲话) + 就不读了,交回调用方关流。already 是缓冲里那一行后面已经收到的字数。""" + from app.engine.state import message_text as _text + from app.engine.state import thinking_text + + seen = already + async for chunk in stream: + seen += len(_text(chunk)) + len(thinking_text(chunk) or "") + if seen > DRAIN_CHARS: + return + + async def _iter_ops(model: Any, messages: list[Any], *, lines: list[str] | None = None): """流式读模型输出,按行切出操作。done、look、ask 之后即停——后面就算有闲话也不要了;look 之后停下也就是 - 「读到这一行就停止本次输出」,模型那条流随之关掉。 + 「读到这一行就停止本次输出」。停下之后不马上关流:交出那一行、调用方接着要下一条时先把流读完(_drain,内容丢掉,最多 + DRAIN_CHARS 字),流末尾那一段用量才收得到——以前读到这一行就关流,输出 token 一个都拿不到,问数据这一轮的花费多半是下限 + (第 1 波遗留)。 除了操作,还把模型的思考原样转出去(thinking_text:Claude 的 thinking 块、OpenAI 兼容系的 reasoning_content), 不把非 JSON 行当成思考。lines 给了的话,认出来的操作行原样记进去:接着写时作为 assistant 消息交回模型。 @@ -101,6 +121,7 @@ async def _iter_ops(model: Any, messages: list[Any], *, lines: list[str] | None lines.append(line.strip()) yield op if op.get("op") in _STOP_OPS: + await _drain(stream, len(buffer)) return cut = _cut_off(chunk) or cut op = _parse_op_line(buffer) diff --git a/backend/app/api/evals.py b/backend/app/api/evals.py index dc5e9419..3c722aaa 100644 --- a/backend/app/api/evals.py +++ b/backend/app/api/evals.py @@ -24,6 +24,18 @@ GET /evaluations/{id}/compare?base=(和上一次或点名的那一次比:指纹差异、逐题转移和归因);GET /pairs/{pair_id}(A/B 配对表和 精确 McNemar);POST /suites/{id}/cleanup(按保留规则清理更早的明细)。「口径命中与重复度」的 E1–E4 直接用口径账本的 GET /api/ledger/stats(总方案 X2:只有一份实现)。 + +第 4 波评测页(细案 01 第 18 项)缺的几样,补在这里(统计、能不能做都由这里给,界面只显示): +- GET /suites 每个题集另带 latest(最近一次评测的状态和准确率)、trend(同一变体最近几次评完的准确率,旧的在前)、schedules + (目标是这个题集的定时任务)——列表页不再为每个题集再取一遍历次评测; +- GET /evaluations/{id} 另带 actions:停止、只重试没做完的、重新评分、钉住 / 取消基线现在能不能做(和那几个接口拒收的条件 + 同一份),以及 open(没做完的项数); +- POST /suites/{id}/cases/preview:写 SQL 定期望时的第一步——按此刻渲染、过只读守卫、跑一次和期望同一条路的 SQL 探针,交回 + 结果的列和前几行(界面上点选格或列);用到遮罩列的照保存时那一道拒收、照实写原因(不回 422); +- GET /signals:「口径命中与重复度」页签的判据(细案 copilot-look-and-caliber-ledger §3.8 的参考线:E1 ≥30% / <10%, + E3 ≥20%)、当前窗口和往前几个窗口各自落在哪一档、成员不少于 3 的问题类(可以加入题集或提取模板)。数都出自 + ledger/stats.compute_stats 和 chat/questions(全站各一份),这里只按窗口调它们、按判据分档。每个窗口都要聚一次问题,页面 + 另调 /api/ledger/stats:问题的向量由 chat/questions 在进程里留一份,同一句话只取一次(配了远程向量模型时不按窗口重发)。 """ from __future__ import annotations @@ -37,7 +49,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.api.coded import CodedHTTPException from app.api.deps import actor_of -from app.db.base import get_session, new_id, utcnow +from app.db.base import SessionLocal, get_session, new_id, utcnow from app.db.models_eval import EvalCase, EvalCatalogSnapshot, EvalRun, EvalSuite from app.eval import cases as case_rules from app.eval import gold as gold_rules @@ -223,8 +235,70 @@ async def _insert_case(session: AsyncSession, suite: EvalSuite, fields: dict[str # ========================================================================== +#: 列表里每个题集最多看最近几次评测(最近一次、趋势都从这几次里取);趋势最多几个点 +OVERVIEW_RUNS = 30 +TREND_POINTS = 8 + + +def _rate_of(run: EvalRun) -> dict[str, Any] | None: + """一次评测总表里的准确率(stats.rate 的形状:passed、n、rate、ci);还没有总表的为 None。""" + acc = (run.summary or {}).get("accuracy") + return dict(acc) if isinstance(acc, dict) else None + + +async def _overview(session: AsyncSession, suite_ids: list[str]) -> dict[str, dict[str, Any]]: + """每个题集:最近一次评测(不论结束没有)、同一变体最近几次评完的准确率(趋势,旧的在前)、目标是它的定时任务。 + + 趋势只连同一变体(fingerprint.variant_key,和上一次比挑基准同一个认法)的几次:换了模型、换了目录的那几次不连成一条线, + 免得把「换了配置」读成「变好了 / 变差了」。A/B 的几臂各是一次评测,同一变体的那一臂才进趋势。""" + from app.db.models import Schedule + from app.eval import runner + from app.scheduler import cron + + out: dict[str, dict[str, Any]] = {sid: {"latest": None, "trend": [], "schedules": []} for sid in suite_ids} + if not suite_ids: + return out + ranked = select(EvalRun, func.row_number().over(partition_by=EvalRun.suite_id, + order_by=EvalRun.created_at.desc()).label("n")) \ + .where(EvalRun.suite_id.in_(suite_ids)).subquery() + from sqlalchemy.orm import aliased + + run_alias = aliased(EvalRun, ranked) + rows = list((await session.execute(select(run_alias).where(ranked.c.n <= OVERVIEW_RUNS) + .order_by(run_alias.created_at.desc()))).scalars()) + await runner.reconcile_stale(session, rows) + by_suite: dict[str, list[EvalRun]] = {} + for row in rows: + by_suite.setdefault(row.suite_id, []).append(row) + for sid, runs in by_suite.items(): + latest = runs[0] + out[sid]["latest"] = {"id": latest.id, "status": latest.status, "running": runner.running(latest), + "arm": latest.arm, "pair_id": latest.pair_id, "trigger": latest.trigger, + "created_at": _iso(latest.created_at), "finished_at": _iso(latest.finished_at), + "counts": dict(latest.counts or {}), "accuracy": _rate_of(latest), + "traced_accuracy": dict((latest.summary or {}).get("traced_accuracy") or {}) or None} + finished = [r for r in runs if r.status in ("succeeded", "partial") and _rate_of(r) is not None] + if finished: + key = (finished[0].fingerprint or {}).get("variant_key") + same = [r for r in finished if (r.fingerprint or {}).get("variant_key") == key][:TREND_POINTS] + out[sid]["trend"] = [{"id": r.id, "created_at": _iso(r.created_at), "arm": r.arm, **(_rate_of(r) or {})} + for r in reversed(same)] + schedules = (await session.execute(select(Schedule).where( + Schedule.target_kind == "eval_suite", Schedule.target_id.in_(suite_ids)) + .order_by(Schedule.created_at))).scalars() + for row in schedules: + try: + text = cron.describe(row.cron, row.timezone) + except cron.CronError: + text = f"按 cron 表达式 {row.cron}({row.timezone})" + out[row.target_id]["schedules"].append({"id": row.id, "name": row.name, "enabled": row.enabled, + "description_text": text, "next_run_at": _iso(row.next_run_at)}) + return out + + @router.get("/suites") async def list_suites(archived: bool = False, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """题集(归档的不列,archived=true 时一起列)。每个题集另带 latest、trend、schedules(_overview)。""" query = select(EvalSuite).order_by(EvalSuite.updated_at.desc()) if not archived: query = query.where(EvalSuite.archived.is_(False)) @@ -239,7 +313,8 @@ async def list_suites(archived: bool = False, session: AsyncSession = Depends(ge counts[suite_id]["cases"] += n if enabled: counts[suite_id]["enabled"] += n - return {"suites": [_suite_out(r, counts=counts[r.id], site=site) for r in rows]} + overview = await _overview(session, [r.id for r in rows]) + return {"suites": [{**_suite_out(r, counts=counts[r.id], site=site), **overview[r.id]} for r in rows]} @router.post("/suites", status_code=201) @@ -266,6 +341,7 @@ async def get_suite(suite_id: str, session: AsyncSession = Depends(get_session)) active = [c for c in out if not c["archived"]] return {**_suite_out(suite, counts={"cases": len(active), "enabled": sum(1 for c in active if c["enabled"])}, site=await _site(session)), + **(await _overview(session, [suite_id]))[suite_id], "cases": out, "pending_upgrades": sum(1 for c in active if (c["upgrade"] or {}).get("pending"))} @@ -342,6 +418,62 @@ async def trial_unsaved(suite_id: str, request: Request, session: AsyncSession = return {"trial": (await _trial(session, suite, fields, actor_of(x_actor))).to_dict()} +#: 预览最多交回几行(界面上点选格或列用;多行期望的取值仍由试算按整份结果算) +PREVIEW_ROWS = 50 + + +@router.post("/suites/{suite_id}/cases/preview") +async def preview_sql(suite_id: str, request: Request, session: AsyncSession = Depends(get_session), + x_actor: str | None = Header(default=None)) -> dict[str, Any]: + """写 SQL 定期望的第一步:{datasource_id, sql}(SQL 可以带 {{ schedule.* }})。按此刻渲染、过只读守卫、跑一次和期望同一条路 + 的 SQL 探针(裸图探索运行,来源 eval),交回结果的列和前 PREVIEW_ROWS 行,界面上据此点选格或列、再去试算。 + + 结果里有遮罩列、取不到结果、运行失败、超时的,照保存时那一道拒收、照实写原因(status 不是 ok,reason、code 同试算)——不回 + 422:这是在写题的途中,界面把原因落在 SQL 那一格下面。{status, reason, code, columns, rows, row_count, truncated, run_id, + rendered}。""" + from app.core import artifact_store + from app.db.models import Artifact + + suite = await _suite(session, suite_id) + body = await _body(request) + probe = {"question": "(预览)", "gold": {"kind": "sql", "datasource_id": body.get("datasource_id"), + "sql": body.get("sql")}, + # 只为跑一次探针:「没有数据」这一种期望不取任何格,取不到值时下面照样交回结果 + "expectations": [{"id": "preview", "kind": "none"}]} + try: + fields = normalize_case(probe) + await resolve_case(session, fields, tz=suite.timezone) + except CaseInvalid as e: + # 没选数据源、SQL 空着、变量写错、没过只读守卫:照实写原因、指明是哪一格 + return {"status": "invalid", "reason": e.message, "code": e.code, "field": e.field, "columns": [], "rows": [], + "row_count": None, "truncated": False, "run_id": None, "rendered": {}} + outcome = await _trial(session, suite, fields, actor_of(x_actor)) + out: dict[str, Any] = {"status": outcome.status, "reason": outcome.reason, "code": outcome.code, "columns": [], + "rows": [], "row_count": None, "truncated": False, + "run_id": outcome.run_ids[0] if outcome.run_ids else None, + "rendered": dict(outcome.rendered or {})} + if outcome.status not in ("ok", "no_value") or not outcome.run_ids: + return out + async with SessionLocal() as read: + refs = (await read.execute(select(Artifact.id).where( + Artifact.run_id == outcome.run_ids[0], Artifact.kind == "query_snapshot", + Artifact.node_id == gold_rules.QUERY_NODE).order_by(Artifact.created_at.desc()))).scalars().all() + snapshot = None + for ref in refs: + try: + snapshot = artifact_store.load(ref) + except ValueError: + snapshot = None + if isinstance(snapshot, dict): + break + if not isinstance(snapshot, dict): + return {**out, "status": "failed", "reason": "预览的运行没有留下查询结果", "code": gold_rules.GOLD_FAILED} + rows = [list(r) if isinstance(r, (list, tuple)) else [r] for r in snapshot.get("rows") or []] + return {**out, "status": "ok", "reason": None, "code": None, "columns": [str(c) for c in snapshot.get("columns") or []], + "rows": rows[:PREVIEW_ROWS], "row_count": snapshot.get("row_count", len(rows)), + "truncated": bool(snapshot.get("truncated")) or len(rows) > PREVIEW_ROWS, "shown": min(len(rows), PREVIEW_ROWS)} + + @router.patch("/suites/{suite_id}/cases/{case_id}") async def update_case(suite_id: str, case_id: str, request: Request, session: AsyncSession = Depends(get_session), x_actor: str | None = Header(default=None)) -> dict[str, Any]: @@ -859,7 +991,29 @@ async def get_evaluation(evaluation_id: str, session: AsyncSession = Depends(get .where(EvalResult.eval_run_id == row.id, EvalResult.status == "running") .order_by(EvalResult.seq))).all() return {**_evaluation_out(row), - "current": [{"case_seq": seq, "repeat": rep, "arm": arm, "doing": doing} for seq, rep, arm, doing in running]} + "current": [{"case_seq": seq, "repeat": rep, "arm": arm, "doing": doing} for seq, rep, arm, doing in running], + "actions": await _actions(session, row)} + + +async def _actions(session: AsyncSession, row: EvalRun) -> dict[str, Any]: + """这次评测现在能做哪几件事(界面照它摆按钮,不自己判):和 stop / retry / rescore / pin 几个接口拒收的条件同一份。 + open 是这次评测(A/B 的话几臂合起来)还没做完的项数:停止、中断、出错、到了上限没开评的,重试就是把它们接着做。""" + from app.core import job_runtime as runtime + from app.db.models_eval import EvalResult + from app.eval import runner + + active = row.status in runtime.ACTIVE + leader = runner.leader_of(row) + arms = list((await session.execute(select(EvalRun.id).where( + (EvalRun.id == leader) | (EvalRun.pair_id == leader)))).scalars()) + open_items = int(await session.scalar(select(func.count()).select_from(EvalResult).where( + EvalResult.eval_run_id.in_(arms), EvalResult.status != "ok")) or 0) + suite = await session.get(EvalSuite, row.suite_id) + cleaned = row.cleaned_at is not None + finished = row.status in ("succeeded", "partial") + return {"stop": active, "retry": not active and not cleaned and open_items > 0 and suite is not None + and not suite.archived, "rescore": not active and not cleaned, + "pin": finished and not row.pinned, "unpin": bool(row.pinned), "open": open_items, "cleaned": cleaned} @router.get("/evaluations/{evaluation_id}/results") @@ -876,9 +1030,18 @@ async def evaluation_results(evaluation_id: str, session: AsyncSession = Depends EvalCase.id.in_({r["case_id"] for r in results if r["case_id"]})))).all()) if results else {} for r in results: r["question"] = questions.get(r["case_id"]) + r["cost"] = _result_cost(r.get("usage") or {}) return {"evaluation_id": row.id, "cleaned": row.cleaned_at is not None, "results": results} +def _result_cost(usage: dict[str, Any]) -> dict[str, Any]: + """一题的花费:助手(runner 记的 assistant 已含复核)+ 写作 + 结论裁判,和这次评测记在账上的同一份;估不出金额的调用另计 + 次数(界面写「≥」或「金额无法估算」)。加在服务端:界面不算数。""" + roles = [usage.get(k) or {} for k in ("assistant", "writer", "judge")] + return {"cost_usd": round(sum(float(r.get("cost_usd") or 0) for r in roles), 6), + "unpriced_calls": sum(int(r.get("unpriced_calls") or 0) for r in roles)} + + @router.get("/evaluations/{evaluation_id}/results/{result_id}") async def evaluation_result(evaluation_id: str, result_id: str, session: AsyncSession = Depends(get_session) ) -> dict[str, Any]: @@ -997,6 +1160,97 @@ async def get_pair(pair_id: str, session: AsyncSession = Depends(get_session)) - return {"pair_id": pair_id, "evaluations": [_evaluation_out(r) for r in rows], "table": table, "cleaned": cleaned} +# ========================================================================== +# 口径命中与重复度(E1、E3):判据、分档、窗口趋势、成员不少于 3 的问题类 +# ========================================================================== + +#: 判据(docs/design/copilot-look-and-caliber-ledger.md §3.8,已修订为持续统计的参考线):E1 等价命中率 ≥30% 值得按口径走、 +#: <10% 先只做对账、介于两者之间先加强规范化;E3 问题重复度 ≥20% 沉淀有价值。只作参考,不卡门 +E1_ROUTE, E1_LOW, E3_WORTH = 0.30, 0.10, 0.20 +SIGNAL_CRITERIA = { + "e1": [{"band": "route", "at_least": E1_ROUTE, "text": "等价命中率 ≥ 30%:值得按已治理的口径走"}, + {"band": "normalize", "at_least": E1_LOW, "text": "10%–30%:先加强规范化,让写法不同的等价 SQL 认得出"}, + {"band": "reconcile", "at_least": 0.0, "text": "< 10%:先只做口径对账"}], + "e3": [{"band": "worth", "at_least": E3_WORTH, "text": "问题重复度 ≥ 20%:把反复问的问题沉淀成口径有价值"}, + {"band": "not_yet", "at_least": 0.0, "text": "< 20%:反复问的问题还不多"}], + "note": "参考线出自口径账本的设计文档,只作参考,不卡门", +} +SIGNAL_DAYS_MAX = 365 +SIGNAL_POINTS_MAX = 12 +#: 每类最多列几个问题 +CLUSTER_MEMBERS = 10 + + +def _e1_band(rate: float | None) -> str | None: + if rate is None: + return None + return "route" if rate >= E1_ROUTE else "normalize" if rate >= E1_LOW else "reconcile" + + +def _e3_band(share: float | None) -> str | None: + if share is None: + return None + return "worth" if share >= E3_WORTH else "not_yet" + + +def _point(stats: dict[str, Any], since: Any, until: Any) -> dict[str, Any]: + """一个窗口的 E1、E3(取自 ledger/stats.compute_stats 的结果,不另算)和各自落在哪一档。""" + e1, e3 = stats.get("e1") or {}, stats.get("e3") or {} + rate = (e1.get("rates") or {}).get("proven") + share = e3.get("cluster_share") + return {"since": _iso(since), "until": _iso(until), + "e1": {"rate": rate, "n": int(e1.get("with_candidates") or 0), "queries": int(e1.get("queries") or 0), + "band": _e1_band(rate)}, + "e3": {"share": share, "n": int(e3.get("questions") or 0), "lower_bound": e3.get("repeated_share"), + "band": _e3_band(share)}} + + +@router.get("/signals") +async def signals(days: int = 30, points: int = 6, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """「口径命中与重复度」页签:判据、最近 days 天和往前 points - 1 个同样长的窗口各自的 E1、E3(趋势,旧的在前),以及最近 + days 天里成员不少于 3 的问题类。 + + E1、E3 的数只出自口径账本的 compute_stats(总方案 X2,全站一份;评测运行不计入,见 engine/origin.counted_runs),问题类只出自 + chat/questions(和 E3、认证队列同一份:只取问题原文、不含 history、排除系统派生的运行)。界面上的详细分布另调 + GET /api/ledger/stats。每一类给代表问题、最近几个成员和最近一个挂着运行的成员(「提取模板」就从那次运行提取)。""" + from datetime import timedelta + + from app.api.deps import current_principal + from app.chat.questions import cluster_questions, embedder, recent_questions + from app.ledger.stats import QUESTION_LIMIT, REPEAT_RUNS, compute_stats + + if not 1 <= days <= SIGNAL_DAYS_MAX: + raise CodedHTTPException(422, f"窗口需要是 1 到 {SIGNAL_DAYS_MAX} 天", case_rules.CASE_INVALID, field="days") + if not 1 <= points <= SIGNAL_POINTS_MAX: + raise CodedHTTPException(422, f"趋势需要是 1 到 {SIGNAL_POINTS_MAX} 个窗口", case_rules.CASE_INVALID, + field="points") + principal = current_principal() + now = utcnow() + span = timedelta(days=days) + trend = [] + for i in reversed(range(points)): + until = now - span * i + since = until - span + trend.append(_point(await compute_stats(session, since=since, until=until, principal=principal), since, until)) + since = now - span + questions = [q for q in await recent_questions(session, principal, limit=QUESTION_LIMIT) + if q.at is None or q.at >= since] + clusters = [] + for c in await cluster_questions(questions) if questions else []: + if c.count < REPEAT_RUNS: + continue + with_run = next((m for m in c.members if m.run_id), None) + clusters.append({"count": c.count, "representative": c.members[0].text[:500], + "first_at": _iso(c.members[-1].at), "last_at": _iso(c.members[0].at), + "members": [{"text": m.text[:500], "at": _iso(m.at), "run_id": m.run_id, "turn_id": m.turn_id} + for m in c.members[:CLUSTER_MEMBERS]], + "run_id": with_run.run_id if with_run else None}) + clusters.sort(key=lambda c: c["count"], reverse=True) + return {"window": {"since": _iso(since), "until": _iso(now), "days": days}, "criteria": SIGNAL_CRITERIA, + "current": trend[-1], "trend": trend, "clusters": clusters, "questions": len(questions), + "threshold": REPEAT_RUNS, "embedder": embedder()} + + @router.post("/suites/{suite_id}/cleanup") async def cleanup_suite(suite_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: """按保留规则清理这个题集更早的评测明细(最近 keep_runs 次、钉住的基线、A/B 的另几臂留着;总表永久保留)。""" diff --git a/backend/app/api/schedules.py b/backend/app/api/schedules.py index 5d641070..a7d08b7f 100644 --- a/backend/app/api/schedules.py +++ b/backend/app/api/schedules.py @@ -349,7 +349,22 @@ async def _fire_outs(session: AsyncSession, fires: list[ScheduleFire]) -> list[d 对比和比的时候两边的封存绑定(auto_compare):哪一边现在不在终态(被继续执行)、被删掉、清单哈希和比的时候不一样 (重新封存过)都给 null——对比页按现在的封存现算,记下的计数和点进去看到的会对不上,或者干脆比不了。工件读不出来、 被改过的同样给 null。基准已经不是现在挑出来的那一次(auto_compare.outdated:中间隔着的运行事后接着运行成功了)也 - 给 null:运行详情里同名的入口比的是另一次。""" + 给 null:运行详情里同名的入口比的是另一次。 + + inputs 只在目标是数据源维护时给:{steps, jobs: [{step, kind, id, tables, scope?}], skipped: [{step, reason}]}——这一次 + 建了哪几个探查、剖析任务,哪几步没执行和原因(app/data/source_maintenance 记在触发上的那一份)。维护没有运行,触发历史、 + 「上次触发」和立即运行的答复靠它写出建了什么,不写「未发起运行」。别的目标给 null:工作流触发的 inputs 是按模板渲染的 + 工作流输入,触发历史不列。""" + maintenance = set((await session.execute(select(Schedule.id).where( + Schedule.id.in_({f.schedule_id for f in fires}), Schedule.target_kind == "source_maintenance"))).scalars()) \ + if fires else set() + + def inputs_of(f: ScheduleFire) -> dict[str, Any] | None: + if f.schedule_id not in maintenance: + return None + raw = f.inputs if isinstance(f.inputs, dict) else {} + return {key: raw[key] for key in ("steps", "jobs", "skipped") if key in raw} + compared = {f.id: f.compare_artifact for f in fires if f.compare_artifact and f.run_id} compares = await auto_compare.briefs(list(compared.values())) stale = await auto_compare.outdated(session, compares) @@ -389,6 +404,7 @@ async def _fire_outs(session: AsyncSession, fires: list[ScheduleFire]) -> list[d "run_status": current.get(f.run_id, (None, None))[0] if f.run_id else None, # 目标是评测题集时这一次触发建的那次评测(没有运行 id) "eval_run_id": f.eval_run_id, + "inputs": inputs_of(f), "reason": None if f.id in launching else f.reason, "notified": notified.get(f.id, []), "compare": compare_of(f), diff --git a/backend/app/chat/questions.py b/backend/app/chat/questions.py index 6e7e9201..0c99e042 100644 --- a/backend/app/chat/questions.py +++ b/backend/app/chat/questions.py @@ -17,9 +17,19 @@ 聚类用 memory/embeddings 的向量贪心聚:按时间从新到旧,和某一类的种子足够像就并进去,否则自立一类。本地向量是词频 哈希、语义能力弱,结果是下限——调用方据 embedder 照实写明用的是哪个向量模型。 + +**向量只取一次。** 同一批问题会被聚很多遍:评测页「口径命中与重复度」一次打开,/api/evals/signals 按窗口调 compute_stats +(缺省 6 个窗口,每个都聚一次),页面另调 /api/ledger/stats;认证队列、治理页的探索问题聚类也各聚一遍。配了远程向量模型时 +每聚一遍都把几百句问题重新发一次,页面慢、还花钱。所以问题原文 → 向量在进程里留一份(_vectors:最多 VECTOR_CACHE_MAX 句, +最近用到的留着),同一句话只取一次;一次里重复的问法(聚类量的正是这个)也只发一句。缓存认的是「哪一个 embedder、哪一个取向量 +的函数」算的:设置页换了向量模型(memory/embeddings.configure 换了对象)就整份丢掉,不把两个向量空间的坐标混着比。取向量按事件 +循环串行:页面并行取的两个接口不会把同一批问题各发一遍。 """ from __future__ import annotations +import asyncio +import weakref +from collections import OrderedDict from collections.abc import Sequence from dataclasses import dataclass, field from datetime import datetime @@ -30,6 +40,8 @@ from sqlalchemy.ext.asyncio import AsyncSession #: 缺省的相似度阈值(同治理接口一直以来的 0.62) DEFAULT_THRESHOLD = 0.62 +#: 进程里最多留几句问题的向量:取问题的地方一次最多 500 句(ledger/stats、治理接口的上限),留两批的量,窗口来回切不用重取 +VECTOR_CACHE_MAX = 1000 @dataclass(frozen=True) @@ -86,14 +98,65 @@ async def recent_questions(session: AsyncSession, principal: Any, *, limit: int) return out[:limit] +#: 问题原文 → 向量(最近用到的在后)和它们是谁算的(见模块说明「向量只取一次」) +_VECTORS: "OrderedDict[str, Any]" = OrderedDict() +_VECTORS_OF: tuple[Any, Any] | None = None +_VECTOR_LOCKS: "weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, asyncio.Lock]" = weakref.WeakKeyDictionary() + + +def _vector_lock() -> asyncio.Lock: + """取向量的锁,按事件循环各一把(asyncio.Lock 不能跨循环用)。""" + loop = asyncio.get_running_loop() + lock = _VECTOR_LOCKS.get(loop) + if lock is None: + lock = _VECTOR_LOCKS[loop] = asyncio.Lock() + return lock + + +def _owner() -> tuple[Any, Any]: + from app.memory import embeddings + + return embeddings.get_embedder(), embeddings.embed_texts + + +def _same_owner(a: tuple[Any, Any] | None, b: tuple[Any, Any]) -> bool: + return a is not None and a[0] is b[0] and a[1] is b[1] + + +async def _vectors(texts: Sequence[str]) -> list[Any]: + """这些问题的向量(顺序同 texts):进程里有的直接用,没有的去重后一次取齐、留下来。""" + global _VECTORS_OF + + async with _vector_lock(): + owner = _owner() + if not _same_owner(_VECTORS_OF, owner): + _VECTORS.clear() + _VECTORS_OF = owner + got = {t: _VECTORS[t] for t in texts if t in _VECTORS} + missing = [t for t in dict.fromkeys(texts) if t not in got] + if missing: + fresh = await owner[1](missing) + got.update(zip(missing, fresh)) + # 取的这一会儿设置页换了向量模型:这一批照用(同一次聚类里坐标是一致的),不留给下一次 + if not _same_owner(_VECTORS_OF, _owner()): + return [got[t] for t in texts] + for t in missing: + _VECTORS[t] = got[t] + for t in dict.fromkeys(texts): + _VECTORS.move_to_end(t) + while len(_VECTORS) > VECTOR_CACHE_MAX: + _VECTORS.popitem(last=False) + return [got[t] for t in texts] + + async def cluster_questions(questions: Sequence[Question], *, threshold: float = DEFAULT_THRESHOLD, ) -> list[QuestionCluster]: - """向量贪心聚类(顺序同 questions:从新到旧)。""" + """向量贪心聚类(顺序同 questions:从新到旧)。向量取自 _vectors:同一句话在进程里只取一次。""" from app.memory import embeddings if not questions: return [] - vectors = await embeddings.embed_texts([q.text for q in questions]) + vectors = await _vectors([q.text for q in questions]) clusters: list[tuple[int, QuestionCluster]] = [] for i, question in enumerate(questions): for seed, cluster in clusters: @@ -112,4 +175,5 @@ def embedder() -> str: return embeddings.embedder_id() -__all__ = ["DEFAULT_THRESHOLD", "Question", "QuestionCluster", "cluster_questions", "embedder", "recent_questions"] +__all__ = ["DEFAULT_THRESHOLD", "VECTOR_CACHE_MAX", "Question", "QuestionCluster", "cluster_questions", "embedder", + "recent_questions"] diff --git a/backend/app/notify/service.py b/backend/app/notify/service.py index 5490dacd..98dd098e 100644 --- a/backend/app/notify/service.py +++ b/backend/app/notify/service.py @@ -538,7 +538,8 @@ async def notify_source_changed(source_id: str) -> int: async def notify_eval(eval_run_id: str, event: str, *, passed: int, graded: int, regressed: int) -> int: """定时评测的通知(eval.regressed 有一道题从通过变成不通过、eval.failed 没评完:失败、到了上限、止损)。只推订阅了它的 全局渠道。消息里只有题集名、状态、通过几题、退步几题、评测编号和链接(细案 01 做法 15、总方案 X6):不放期望值、答案里的数、 - 题目原文。链接去发起它的那条定时任务的触发历史(评测页在第 4 波)。""" + 题目原文。链接去这次评测的详情页(/evals/runs/{评测 id}:进度、逐题结果、和上一次比;第 4 波评测页上线之前是定时任务的 + 触发历史),退步的那几题在详情页上点得开。""" from app.db.models_eval import EvalRun, EvalSuite async with SessionLocal() as session: @@ -560,7 +561,7 @@ async def notify_eval(eval_run_id: str, event: str, *, passed: int, graded: int, if run.arm: lines.append(("对照", run.arm[:40])) lines.append(("评测", f"#{eval_run_id[:6]}")) - links = [("查看定时任务", f"{base}/schedules/{run.schedule_id}?tab=history")] if base and run.schedule_id else [] + links = [("查看评测", f"{base}/evals/runs/{eval_run_id}")] if base else [] message = {"event": event, "status": run.status, "ok": False, "title": f"定时评测「{name}」{'有退步' if regressed_event else '没有评完'}", "lines": lines, "links": links, "subject": {"eval_run_id": eval_run_id, "suite_id": run.suite_id, "schedule_id": run.schedule_id}} diff --git a/backend/app/providers/usage.py b/backend/app/providers/usage.py index 37563e6b..9404039a 100644 --- a/backend/app/providers/usage.py +++ b/backend/app/providers/usage.py @@ -13,9 +13,10 @@ async generator:with 块跨着 yield 的话,变量在两次 yield 之间留 **金额。** 按 providers/catalog.estimate_cost 算,和运行的 llm.end 一个口径。估不出的调用记 unpriced_calls:模型不在价格 目录里,或者用量没报全而目录价又不是 0。估不出的不当成 0 元,调用方据此写「≥」或「金额无法估算」。用量没报全有两种: -回复没带用量(有的网关流式不给);读到一半就不读了——生成过程读到 done、look、ask 那一行就关掉模型的流(后面的闲话不要、 -也不再为它付钱),用量在流的末尾才报全(Anthropic 的输出 token 在最后一条,OpenAI 兼容系开了 stream_usage 也在最后), -关掉的那一次只拿得到已经报了的部分。这一次照样计次数、计已报的 token,记成估不出,不悄悄少算。 +回复没带用量(有的网关流式不给);读到一半就不读了——用量在流的末尾才报全(Anthropic 的输出 token 在最后一条,OpenAI +兼容系开了 stream_usage 也在最后)。生成过程读到 done、look、ask 那一行之后照样把流读完再关(api/copilot_gen._drain), +流末尾那一段用量收得到;只有模型在那一行之后还往下写超过 DRAIN_CHARS 字的闲话时才当场关流(不为闲话付钱),连接断了也一样, +这一次只拿得到已经报了的部分:照样计次数、计已报的 token,记成估不出,不悄悄少算。 模型名按发起调用时的参数取(invocation_params 的 model / model_name,就是接入里配的模型 id,价格目录按它查),取不到再 看回复里的 model_name。 @@ -85,7 +86,7 @@ class Meter(BaseCallbackHandler): row["unreported"] += 1 def on_llm_error(self, error: BaseException, *, run_id: UUID, **kwargs: Any) -> None: - """流读到一半停了(生成过程读到 done / look / ask 就关流,或者连接断了):LangChain 照样交来已经收到的那几段。 + """流读到一半停了(生成过程在最后一行之后读到太多闲话就关流,或者连接断了):LangChain 照样交来已经收到的那几段。 有收到东西的照计一次、记已报的 token,算用量没报全;一段都没收到的(请求刚发出去就被拒了)不计。""" from langchain_core.outputs import ChatGenerationChunk diff --git a/backend/tests/test_eval_schedule.py b/backend/tests/test_eval_schedule.py index 441d4644..1bf3c7b3 100644 --- a/backend/tests/test_eval_schedule.py +++ b/backend/tests/test_eval_schedule.py @@ -73,6 +73,34 @@ def received(monkeypatch): return got +async def _public_url(url: str): + """临时设对外访问地址(通知里的链接按它拼),交回复原的函数:别的测试不受影响。""" + from app.core import public_url + from app.db.models import Setting + + async with SessionLocal() as session: + row = await session.get(Setting, "notify") + before = dict(row.value or {}) if row is not None else None + if row is None: + session.add(Setting(key="notify", value={"public_base_url": url})) + else: + row.value = {**(row.value or {}), "public_base_url": url} + await session.commit() + await public_url.refresh(session) + + async def restore() -> None: + async with SessionLocal() as session: + row = await session.get(Setting, "notify") + if before is None and row is not None: + await session.delete(row) + elif row is not None: + row.value = before + await session.commit() + await public_url.refresh(session) + + return restore + + def stub(monkeypatch, *, delay: float = 0.0, path: str = "asked"): async def drive(self): await asyncio.sleep(delay) @@ -212,8 +240,12 @@ async def test_an_unfinished_scheduled_evaluation_pushes_failed(client, monkeypa suite_id = await es.make_suite("none", cases=ASK * 4, name="虚构失败题集", budget={"max_parallel": 1}) body = await _schedule(client, suite_id) await _channel(["eval.failed"]) - fire = await service.fire_once(body["id"], utcnow().replace(microsecond=0), "manual") - await es.wait_eval(fire.eval_run_id) + restore = await _public_url("https://agentlab.example.com") + try: + fire = await service.fire_once(body["id"], utcnow().replace(microsecond=0), "manual") + await es.wait_eval(fire.eval_run_id) + finally: + await restore() async with SessionLocal() as session: run = await session.get(EvalRun, fire.eval_run_id) skipped = (await session.execute(select(EvalResult).where( @@ -223,8 +255,9 @@ async def test_an_unfinished_scheduled_evaluation_pushes_failed(client, monkeypa message = json.loads(request.content) assert message["event"] == "eval.failed" and message["title"] == "定时评测「虚构失败题集」没有评完" assert "通过:0 / 3 题" in message["text"] - # 链接去那条定时任务的触发历史(设了对外访问地址才有链接) - assert all("/schedules/" in link["url"] for link in message["links"]) + # 链接去这次评测的详情页(第 4 波评测页上线之前去定时任务的触发历史);设了对外访问地址才有链接 + assert [link["url"] for link in message["links"]] == [f"https://agentlab.example.com/evals/runs/{fire.eval_run_id}"], \ + message["links"] def test_the_eval_events_say_they_are_for_scheduled_evaluations(): diff --git a/backend/tests/test_evals_ui_api.py b/backend/tests/test_evals_ui_api.py new file mode 100644 index 00000000..2307c84d --- /dev/null +++ b/backend/tests/test_evals_ui_api.py @@ -0,0 +1,255 @@ +"""评测页(第 4 波,细案 01 第 18 项)在 app/api/evals.py 里补的几样:统计、能不能做都由服务端给,界面只显示。 + +- 题集列表每个题集带最近一次评测、同一变体最近几次评完的准确率(旧的在前,换了变体的不连成一条线)、目标是它的定时任务; +- 评测详情带 actions:只重试没做完的、重新评分、钉住 / 取消基线现在能不能做,和接口拒收的条件同一份; +- 写 SQL 定期望的预览:交回结果的列和前几行;用到遮罩列的照保存时那一道拒收,没过只读守卫的指明是哪一格(都不回 422); +- 「口径命中与重复度」:判据、窗口趋势(出自 ledger/stats.compute_stats)、成员不少于 3 的问题类(出自 chat/questions), + 评测运行不进问题类。 + +库、表、题集名、问法都是虚构的。 +""" +from __future__ import annotations + +import uuid +from datetime import timedelta + +import pytest +from httpx import ASGITransport, AsyncClient +from sqlalchemy import delete + +from app.db.base import SessionLocal, utcnow +from app.db.models import Conversation, ConversationTurn, Run, Schedule +from app.db.models_eval import EvalResult, EvalRun +from app.engine.runner import run_manager +from app.main import app +from tests import eval_samples as es +from tests.fixtures import dual_check_shop as shop +from tests.fixtures.sources import drop_source + + +@pytest.fixture(autouse=True) +async def engine_up(): + await run_manager.setup() + yield + await run_manager.shutdown() + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +@pytest.fixture +async def source(tmp_path): + name, sid = await shop.add_source(shop.build(tmp_path / "shop.db")) + yield {"name": name, "id": sid} + await drop_source(sid) + + +def _summary(passed: int, n: int) -> dict: + return {"accuracy": {"passed": passed, "n": n, "rate": round(passed / n, 4) if n else None, "ci": None}, + "traced_accuracy": {"passed": passed, "n": n, "rate": round(passed / n, 4) if n else None, "ci": None}} + + +async def _evaluation(suite_id: str, *, status: str, key: str, passed: int, n: int, minutes_ago: int, + arm: str | None = None, pinned: bool = False, cleaned: bool = False) -> str: + async with SessionLocal() as session: + row = EvalRun(suite_id=suite_id, status=status, trigger="manual", variant={}, resolved_models={}, + fingerprint={"variant_key": key}, summary=_summary(passed, n) if status != "queued" else {}, + counts={"total": n, "done": n}, arm=arm, pinned=pinned, + cleaned_at=utcnow() if cleaned else None) + session.add(row) + await session.flush() + row.created_at = utcnow() - timedelta(minutes=minutes_ago) + await session.commit() + return row.id + + +async def test_the_suite_list_carries_the_latest_run_a_trend_of_the_same_variant_and_its_schedules(client, source): + suite_id = await es.make_suite(source["id"], name=f"虚构题集_{uuid.uuid4().hex[:6]}") + old = await _evaluation(suite_id, status="succeeded", key="A", passed=1, n=4, minutes_ago=50) + other = await _evaluation(suite_id, status="succeeded", key="B", passed=4, n=4, minutes_ago=40) + failed = await _evaluation(suite_id, status="failed", key="A", passed=0, n=4, minutes_ago=30) + mid = await _evaluation(suite_id, status="partial", key="A", passed=2, n=4, minutes_ago=20) + new = await _evaluation(suite_id, status="succeeded", key="A", passed=3, n=4, minutes_ago=10) + async with SessionLocal() as session: + sched = Schedule(name="虚构每周回归", workflow_id="", workflow_name="虚构题集", target_kind="eval_suite", + target_id=suite_id, cron="0 9 * * 1", timezone="Asia/Shanghai", enabled=True) + session.add(sched) + await session.commit() + sched_id = sched.id + try: + listed = {s["id"]: s for s in (await client.get("/api/evals/suites")).json()["suites"]} + got = listed[suite_id] + assert got["latest"]["id"] == new and got["latest"]["accuracy"]["rate"] == 0.75 + # 趋势只连同一变体、评完了的:B 那一次、失败的那一次都不进;旧的在前 + assert [p["id"] for p in got["trend"]] == [old, mid, new], got["trend"] + assert [p["rate"] for p in got["trend"]] == [0.25, 0.5, 0.75] + assert other not in [p["id"] for p in got["trend"]] and failed not in [p["id"] for p in got["trend"]] + [schedule] = got["schedules"] + assert schedule["id"] == sched_id and schedule["enabled"] is True and "09:00" in schedule["description_text"] + detail = (await client.get(f"/api/evals/suites/{suite_id}")).json() + assert detail["latest"]["id"] == new and [s["id"] for s in detail["schedules"]] == [sched_id] + finally: + async with SessionLocal() as session: + await session.execute(delete(Schedule).where(Schedule.id == sched_id)) + await session.commit() + await es.drop_suites() + + +async def test_the_evaluation_says_what_can_be_done_now(client, source): + suite_id = await es.make_suite(source["id"], name=f"虚构题集_{uuid.uuid4().hex[:6]}") + try: + run_id = await _evaluation(suite_id, status="partial", key="A", passed=1, n=2, minutes_ago=5) + async with SessionLocal() as session: + session.add(EvalResult(eval_run_id=run_id, seq=1, case_id=None, case_seq=1, case_hash="h", repeat=0, + status="ok", verdict="pass")) + session.add(EvalResult(eval_run_id=run_id, seq=2, case_id=None, case_seq=2, case_hash="h", repeat=0, + status="cancelled", error="评测已停止")) + await session.commit() + actions = (await client.get(f"/api/evals/evaluations/{run_id}")).json()["actions"] + assert actions == {"stop": False, "retry": True, "rescore": True, "pin": True, "unpin": False, "open": 1, + "cleaned": False} + pinned = await client.post(f"/api/evals/evaluations/{run_id}/pin", json={"pinned": True}) + assert pinned.status_code == 200 + actions = (await client.get(f"/api/evals/evaluations/{run_id}")).json()["actions"] + assert actions["pin"] is False and actions["unpin"] is True + + done = await _evaluation(suite_id, status="succeeded", key="A", passed=2, n=2, minutes_ago=4, cleaned=True) + actions = (await client.get(f"/api/evals/evaluations/{done}")).json()["actions"] + # 明细清理了的:不能重试、不能重评(和接口拒收的条件一致) + assert actions["retry"] is False and actions["rescore"] is False and actions["cleaned"] is True + assert (await client.post(f"/api/evals/evaluations/{done}/rescore")).status_code == 409 + finally: + await es.drop_suites() + + +async def test_previewing_an_expected_sql_gives_columns_and_rows(client, source): + made = await client.post("/api/evals/suites", json={"name": f"虚构题集_{uuid.uuid4().hex[:6]}"}) + suite_id = made.json()["id"] + try: + body = {"datasource_id": source["id"], + "sql": "SELECT status AS 状态, SUM(amount) AS gmv FROM orders GROUP BY status ORDER BY status"} + got = (await client.post(f"/api/evals/suites/{suite_id}/cases/preview", json=body)).json() + assert got["status"] == "ok" and got["columns"] == ["状态", "gmv"], got + assert got["rows"] and all(len(r) == 2 for r in got["rows"]) and got["run_id"] + assert got["rendered"]["sql"].startswith("SELECT status") + # 没过只读守卫:指明是 SQL 那一格,不回 422 + res = await client.post(f"/api/evals/suites/{suite_id}/cases/preview", + json={"datasource_id": source["id"], "sql": "DELETE FROM orders"}) + assert res.status_code == 200 and res.json()["status"] == "invalid" and res.json()["field"] == "gold.sql" + res = await client.post(f"/api/evals/suites/{suite_id}/cases/preview", json={"sql": "SELECT 1"}) + assert res.json()["status"] == "invalid" and res.json()["field"] == "gold.datasource_id" + finally: + await es.drop_suites() + + +async def test_a_preview_touching_a_masked_column_is_refused(client, tmp_path): + name, sid = await shop.add_source(shop.build(tmp_path / "masked.db"), options={"mask_columns": ["region"]}) + made = await client.post("/api/evals/suites", json={"name": f"虚构题集_{uuid.uuid4().hex[:6]}"}) + try: + got = (await client.post(f"/api/evals/suites/{made.json()['id']}/cases/preview", json={ + "datasource_id": sid, "sql": "SELECT region AS 地区, SUM(amount) AS gmv FROM orders GROUP BY region"})).json() + assert got["status"] == "refused" and got["code"] == "eval_gold_masked" and "遮罩" in got["reason"], got + assert got["rows"] == [] and got["columns"] == [], "遮罩列的值一格都不能交出去" + finally: + await es.drop_suites() + await drop_source(sid) + + +async def _asked(question: str, n: int, *, origin: str = "chat") -> list[str]: + async with SessionLocal() as session: + conv = Conversation(kind="chat", title="虚构对话") + session.add(conv) + await session.flush() + ids = [] + for i in range(n): + run = Run(workflow_name="问数据", status="succeeded", graph={}, input={"question": question}, + origin_kind=origin, run_class="exploratory") + session.add(run) + await session.flush() + session.add(ConversationTurn(conversation_id=conv.id, seq=i, question=question, run_id=run.id, + status="done")) + ids.append(run.id) + await session.commit() + return ids + + +async def test_signals_give_criteria_trend_and_repeated_question_clusters(client): + question = f"虚构门店 {uuid.uuid4().hex[:6]} 上周的销售额是多少?" + runs = await _asked(question, 3) + eval_question = f"虚构评测题 {uuid.uuid4().hex[:6]} 本月退款率?" + await _asked(eval_question, 3, origin="eval") + got = (await client.get("/api/evals/signals", params={"days": 30, "points": 3})).json() + assert [c["band"] for c in got["criteria"]["e1"]] == ["route", "normalize", "reconcile"] + assert got["criteria"]["e3"][0]["at_least"] == 0.2 + assert len(got["trend"]) == 3 and got["current"] == got["trend"][-1] + assert got["trend"][0]["until"] <= got["trend"][1]["since"] or got["trend"][0]["until"] == got["trend"][1]["since"] + assert got["current"]["e3"]["n"] >= 3 and got["current"]["e3"]["band"] in ("worth", "not_yet") + mine = [c for c in got["clusters"] if c["representative"] == question] + assert mine and mine[0]["count"] == 3 and mine[0]["run_id"] in runs, got["clusters"] + assert not [c for c in got["clusters"] if c["representative"] == eval_question], "评测运行不进问题类" + assert got["threshold"] == 3 and got["embedder"] + assert (await client.get("/api/evals/signals", params={"days": 0})).status_code == 422 + + +async def test_one_visit_to_the_signals_tab_embeds_each_question_once(client, monkeypatch): + """「口径命中与重复度」页签一次打开:/signals 按窗口调 compute_stats(每个窗口都聚一次问题),页面另调 /ledger/stats—— + 同一句问题原文不管在几个窗口、几个接口里出现,都只取一次向量;再打开一遍一次都不取(配了远程向量模型时每取一次都花钱)。 + 两个接口同时到(页面是并行取的)也一样。""" + import asyncio + from collections import Counter + + from app.memory import embeddings + + real = embeddings.embed_texts + seen: Counter[str] = Counter() + + async def counting(texts): + seen.update(texts) + return await real(texts) + + monkeypatch.setattr(embeddings, "embed_texts", counting) + mark = uuid.uuid4().hex[:6] + asked = [f"虚构门店 {mark} 第 {i} 问的销售额是多少?" for i in range(3)] + for question in asked: + await _asked(question, 3) + a, b = await asyncio.gather(client.get("/api/evals/signals", params={"days": 30, "points": 6}), + client.get("/api/ledger/stats")) + assert a.status_code == 200 and b.status_code == 200 + mine = {t: n for t, n in seen.items() if mark in t} + assert set(mine) == set(asked) and set(mine.values()) == {1}, mine + assert sum(c["count"] for c in a.json()["clusters"] if mark in c["representative"]) >= 9, "聚类照旧:九次提问都在" + seen.clear() + await client.get("/api/evals/signals", params={"days": 30, "points": 6}) + await client.get("/api/ledger/stats") + assert not [t for t in seen if mark in t], "再打开一遍:向量都在进程里,一次都不取" + + +def test_bands_follow_the_reference_lines(): + from app.api import evals + + assert [evals._e1_band(x) for x in (None, 0.05, 0.1, 0.29, 0.3)] == [None, "reconcile", "normalize", "normalize", + "route"] + assert [evals._e3_band(x) for x in (None, 0.19, 0.2)] == [None, "not_yet", "worth"] + + +async def test_each_result_carries_its_total_cost(client, source): + """逐题结果带这一题的花费:助手(已含复核)+ 写作 + 结论裁判,估不出的另计次数——界面直接显示,不自己加。""" + suite_id = await es.make_suite(source["id"], name=f"虚构题集_{uuid.uuid4().hex[:6]}") + try: + run_id = await _evaluation(suite_id, status="succeeded", key="A", passed=1, n=1, minutes_ago=1) + async with SessionLocal() as session: + session.add(EvalResult(eval_run_id=run_id, seq=1, case_id=None, case_seq=1, case_hash="h", repeat=0, + status="ok", verdict="pass", usage={ + "assistant": {"cost_usd": 0.012, "unpriced_calls": 0}, + "review": {"cost_usd": 0.002, "unpriced_calls": 0}, + "writer": {"cost_usd": 0.03, "unpriced_calls": 1}, + "judge": {"cost_usd": 0.001, "calls": 1}})) + await session.commit() + [row] = (await client.get(f"/api/evals/evaluations/{run_id}/results")).json()["results"] + # 复核已经算在助手里(runner 记的 assistant = 建图 + 复核),不重复加 + assert row["cost"] == {"cost_usd": 0.043, "unpriced_calls": 1}, row["cost"] + finally: + await es.drop_suites() diff --git a/backend/tests/test_frontend_eval_unions.py b/backend/tests/test_frontend_eval_unions.py new file mode 100644 index 00000000..39967637 --- /dev/null +++ b/backend/tests/test_frontend_eval_unions.py @@ -0,0 +1,95 @@ +"""评测页的前端约定(frontend/src/typesEval.ts)和文案(lib/termsEval.ts 的 EVAL_TEXT)里的取值,和服务端的常量逐个对上。 + +评测页上的结论、作答路径、转移、归因、目录视图、状态都是服务端给的取值,前端照着写叫法(界面层不放逻辑,叫法在 termsEval)。 +服务端加了一种(比如作答路径「被驳回」),前端的联合类型和叫法表没跟上,界面就写出英文原值;前端多写了一种,就在等一个 +永远不会来的值。这里直接读两份 .ts 文件比(同 test_frontend_type_unions.py 的写法),改了一边没改另一边就过不去。 +""" +from __future__ import annotations + +import re +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +TYPES_EVAL = ROOT / "frontend" / "src" / "typesEval.ts" +TERMS_EVAL = ROOT / "frontend" / "src" / "lib" / "termsEval.ts" + +_COMMENT = re.compile(r"/\*.*?\*/|//[^\n]*", re.S) + + +def _source(path: Path) -> str: + return _COMMENT.sub("", path.read_text(encoding="utf-8")) + + +def _union(name: str) -> list[str]: + text = _source(TYPES_EVAL) + found = re.search(rf"export type {name} =(.*?)(?=\nexport |\n\n|\Z)", text, re.S) + assert found is not None, f"typesEval.ts 里没有 export type {name}" + return re.findall(r"'([^']+)'", found.group(1)) + + +def _labels(path: str) -> list[str]: + """EVAL_TEXT 里某一张叫法表(a.b 这样的路径)的键,按写的顺序。只认一行写完、或几行写完的对象字面量。""" + text = _source(TERMS_EVAL) + *parents, leaf = path.split(".") + start = 0 + for part in parents: + hit = re.search(rf"\n\s*{part}: \{{", text[start:]) + assert hit is not None, f"EVAL_TEXT 里没有 {part}" + start += hit.end() + hit = re.search(rf"\n\s*{leaf}: \{{(.*?)\}}( as Record<[^>]+>)?,", text[start:], re.S) + assert hit is not None, f"EVAL_TEXT 里没有 {path}" + return re.findall(r"(\w+): '", hit.group(1)) + + +def test_statuses_triggers_and_kinds(): + from app.core.job_runtime import ACTIVE, FINISHED + from app.db.models_eval import CASE_KINDS, RESULT_PATHS, RUN_TRIGGERS, VERDICTS + + assert _union("EvalStatus") == [*ACTIVE, *FINISHED] + assert _union("EvalTrigger") == list(RUN_TRIGGERS) + assert _union("EvalCaseKind") == list(CASE_KINDS) + assert _union("EvalVerdict") == list(VERDICTS) + assert _union("EvalPath") == list(RESULT_PATHS) + assert _labels("status") == [*ACTIVE, *FINISHED] + assert _labels("trigger") == list(RUN_TRIGGERS) + assert _labels("verdict") == list(VERDICTS) + assert _labels("path") == list(RESULT_PATHS) + + +def test_expectations_gold_and_trial(): + from app.eval.cases import EXPECTATION_KINDS, GOLD_KINDS + from app.eval.gold import STATUSES, UPGRADE_LABEL + + assert _union("EvalExpectationKind") == list(EXPECTATION_KINDS) + assert _labels("expectation.kind") == list(EXPECTATION_KINDS) + assert _union("EvalGoldKind") == list(GOLD_KINDS) + assert _labels("gold.kind") == list(GOLD_KINDS) + assert _union("EvalUpgradePolicy") == list(UPGRADE_LABEL) + # 试算的结论:服务端的几种,加上作答前后算出来不一样时的 drifted(gold.settle) + assert set(STATUSES) | {"drifted"} == set(_union("EvalTrialStatus")) + assert set(_union("EvalTrialStatus")) <= set(_labels("trialStatus")) + + +def test_scores_compare_and_catalog_modes(): + from app.eval.compare import CAUSES, TRANSITIONS + from app.eval.score import OUTCOMES, TEXT_OUTCOMES + from app.eval.variants import CATALOG_MODES + + assert _union("EvalOutcome") == list(OUTCOMES) + assert _labels("outcome") == list(OUTCOMES) + assert _union("EvalTextOutcome") == list(TEXT_OUTCOMES) + assert _labels("textOutcome") == list(TEXT_OUTCOMES) + assert _union("EvalTransition") == list(TRANSITIONS) + assert _labels("compare.transition") == list(TRANSITIONS) + assert _union("EvalCause") == list(CAUSES) + assert _labels("compare.cause") == list(CAUSES) + assert _union("EvalCatalogMode") == list(CATALOG_MODES) + assert _labels("start.catalogMode") == list(CATALOG_MODES) + + +def test_signal_bands_follow_the_server(): + from app.api.evals import SIGNAL_CRITERIA + + assert _union("EvalE1Band") == [c["band"] for c in SIGNAL_CRITERIA["e1"]] + assert _union("EvalE3Band") == [c["band"] for c in SIGNAL_CRITERIA["e3"]] + assert set(_labels("signals.band")) == {c["band"] for c in [*SIGNAL_CRITERIA["e1"], *SIGNAL_CRITERIA["e3"]]} diff --git a/backend/tests/test_maintenance_fire_history.py b/backend/tests/test_maintenance_fire_history.py new file mode 100644 index 00000000..1d49b235 --- /dev/null +++ b/backend/tests/test_maintenance_fire_history.py @@ -0,0 +1,92 @@ +"""定时任务页的触发历史认得数据源维护(复核 W4R-05)。 + +数据源维护的一次触发建的是探查、剖析任务,没有运行:触发记录的 inputs({steps, jobs, skipped},app/data/source_maintenance) +要交到触发历史、页头的「上次触发」和「立即运行」的答复里,界面才写得出建了哪几个任务、哪几步没执行,不再写「未发起运行」。 +工作流目标的触发不给 inputs(那是按模板渲染的工作流输入,触发历史不列)。 + +库、表、名字都是虚构的。 +""" +from __future__ import annotations + +import pytest +from httpx import ASGITransport, AsyncClient +from sqlalchemy import delete + +from app.db.base import SessionLocal, utcnow +from app.db.models import Schedule, ScheduleFire +from app.main import app +from app.scheduler import service +from tests.fixtures.sources import drop_source +from tests.test_source_maintenance import _schedule, _source, _wait_jobs + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +@pytest.fixture +async def made(): + ids: list[str] = [] + yield ids + async with SessionLocal() as session: + await session.execute(delete(Schedule).where(Schedule.id.in_(ids))) + await session.commit() + + +async def test_a_maintenance_fire_carries_the_jobs_it_started_and_the_steps_not_run(client, tmp_path, made): + sid = await _source(client, tmp_path / "shop.db", profile=False) + try: + schedule_id = await _schedule(sid) + made.append(schedule_id) + ran = (await client.post(f"/api/schedules/{schedule_id}/run")).json() + fire = ran["fire"] + assert ran["run_id"] is None and fire["status"] == "started" and fire["run_id"] is None + jobs = fire["inputs"]["jobs"] + assert [j["kind"] for j in jobs] == ["introspect"] and jobs[0]["id"] + assert {s["step"] for s in fire["inputs"]["skipped"]} == {"profile_needed", "observe"} + async with SessionLocal() as session: + await _wait_jobs(await session.get(ScheduleFire, fire["id"])) + + listed = (await client.get(f"/api/schedules/{schedule_id}/fires")).json()["items"] + assert [f["id"] for f in listed] == [fire["id"]] + assert listed[0]["inputs"] == fire["inputs"] + assert set(listed[0]["inputs"]) == {"steps", "jobs", "skipped"} + head = (await client.get(f"/api/schedules/{schedule_id}")).json() + assert head["last_fire"]["inputs"]["jobs"] == jobs + rows = (await client.get("/api/schedules")).json()["items"] + assert [s["last_fire"]["inputs"]["jobs"] for s in rows if s["id"] == schedule_id] == [jobs] + finally: + await drop_source(sid) + + +async def test_a_workflow_fire_does_not_list_its_rendered_inputs(client, made): + async with SessionLocal() as session: + row = Schedule(name="虚构周报 · 每周", cron="0 9 * * 1", timezone="Asia/Shanghai", + **service.target_columns("workflow", "wf-fictional"), input_template={"date": "{{ schedule.yesterday }}"}) + session.add(row) + await session.flush() + session.add(ScheduleFire(schedule_id=row.id, fire_at=utcnow().replace(microsecond=0), kind="manual", + status="failed_to_start", reason="虚构的原因", inputs={"date": "2026-10-15"})) + await session.commit() + made.append(row.id) + [got] = (await client.get(f"/api/schedules/{row.id}/fires")).json()["items"] + assert got["inputs"] is None + + +def test_the_steps_and_their_names_on_the_schedules_page_match_the_server(): + """定时任务页照服务端的步骤和叫法写(types.ts 的 ScheduleMaintenanceStep、SCHEDULE_TEXT.maintenanceStep):服务端加了一步, + 界面没跟上就写出英文原值。直接读两份 .ts 比(同 test_frontend_type_unions 的写法)。""" + import re + from pathlib import Path + + from app.data.source_maintenance import STEP_LABEL, STEPS + + src = Path(__file__).resolve().parents[2] / "frontend" / "src" + types = (src / "types.ts").read_text(encoding="utf-8") + union = re.search(r"export type ScheduleMaintenanceStep =([^\n]*)", types) + assert union is not None and re.findall(r"'([^']+)'", union.group(1)) == list(STEPS) + terms = (src / "lib" / "terms.ts").read_text(encoding="utf-8") + labels = re.search(r"\n\s*maintenanceStep: \{([^}]*)\}", terms) + assert labels is not None and dict(re.findall(r"(\w+): '([^']+)'", labels.group(1))) == STEP_LABEL diff --git a/backend/tests/test_question_vectors.py b/backend/tests/test_question_vectors.py new file mode 100644 index 00000000..f17cee7b --- /dev/null +++ b/backend/tests/test_question_vectors.py @@ -0,0 +1,130 @@ +"""问题聚类的向量只取一次(app/chat/questions.py 模块说明「向量只取一次」)。 + +评测页「口径命中与重复度」一次打开要按窗口聚好几遍问题,页面另调账本统计;配了远程向量模型时每遍都把几百句问题重发一次。 +这里守进程里那一份问题原文 → 向量: + +- 同一句话只取一次,一次里重复的问法也只发一句;顺序、对应关系照旧; +- 换了向量模型(或取向量的函数)就整份丢掉,不把两个向量空间的坐标混着比;取的这一会儿换了的,这一批照用、不留; +- 最多留 VECTOR_CACHE_MAX 句,最近用到的留着;一次超过上限也照样每句拿到自己的向量。 + +问题都是虚构的。 +""" +from __future__ import annotations + +import pytest + +from app.chat import questions as qmod +from app.chat.questions import Question, cluster_questions +from app.memory import embeddings + + +@pytest.fixture(autouse=True) +def fresh_cache(): + qmod._VECTORS.clear() + qmod._VECTORS_OF = None + yield + qmod._VECTORS.clear() + qmod._VECTORS_OF = None + + +DIM = 32 + + +def _fake(calls: list[list[str]], *, scale: float = 1.0, made: dict[str, list[float]] | None = None): + """每句话一个确定的向量:「同类」开头的几句彼此相同(聚得到一起),别的各占一个方向(互不相像)。made 记下给过的向量。""" + seen: dict[str, int] = {} + + async def embed(texts): + calls.append(list(texts)) + out = [] + for t in texts: + vec = [0.0] * DIM + vec[0 if t.startswith("同类") else 1 + seen.setdefault(t, len(seen))] = scale + out.append(vec) + if made is not None: + made[t] = vec + return out + + return embed + + +def _qs(*texts: str) -> list[Question]: + return [Question(text=t, at=None) for t in texts] + + +async def test_each_text_is_embedded_once_and_duplicates_in_one_call_are_sent_once(monkeypatch): + calls: list[list[str]] = [] + monkeypatch.setattr(embeddings, "embed_texts", _fake(calls)) + got = await cluster_questions(_qs("同类:上周销售额", "同类:上周销售额", "别的:库存", "同类:上周销售额")) + assert calls == [["同类:上周销售额", "别的:库存"]], "一次里重复的问法只发一句" + assert [c.count for c in got] == [3, 1] + again = await cluster_questions(_qs("别的:库存", "同类:上周销售额", "新的:退款率")) + assert calls[1:] == [["新的:退款率"]], "已经取过的不再取" + assert [c.count for c in again] == [1, 1, 1] + + +async def test_a_new_embedder_drops_every_vector(monkeypatch): + first: list[list[str]] = [] + second: list[list[str]] = [] + monkeypatch.setattr(embeddings, "embed_texts", _fake(first)) + await cluster_questions(_qs("同类:本月退款率", "别的:毛利")) + monkeypatch.setattr(embeddings, "embed_texts", _fake(second, scale=2.0)) + await cluster_questions(_qs("同类:本月退款率", "别的:毛利")) + assert second == [["同类:本月退款率", "别的:毛利"]], "换了取向量的函数:整份重取" + assert qmod._VECTORS["同类:本月退款率"][0] == 2.0 + + # 设置页换了向量模型(configure 换了 embedder 对象)同理 + other = embeddings.LocalEmbedder() + monkeypatch.setattr(embeddings, "get_embedder", lambda: other) + await cluster_questions(_qs("同类:本月退款率")) + assert second[1:] == [["同类:本月退款率"]] + + +async def test_switching_embedder_mid_call_keeps_the_batch_but_not_the_cache(monkeypatch): + calls: list[list[str]] = [] + later: list[list[str]] = [] + + async def switching(texts): + calls.append(list(texts)) + # 取的这一会儿设置页换了向量模型 + monkeypatch.setattr(embeddings, "embed_texts", _fake(later)) + return [[1.0] + [0.0] * (DIM - 1) for _ in texts] + + monkeypatch.setattr(embeddings, "embed_texts", switching) + got = await cluster_questions(_qs("同类:华东门店", "同类:华北门店")) + assert [c.count for c in got] == [2], "这一批照用旧向量算完" + assert not qmod._VECTORS, "旧向量不留给下一次" + await cluster_questions(_qs("同类:华东门店")) + assert later == [["同类:华东门店"]] + + +async def test_the_cache_is_bounded_and_keeps_the_recently_used(monkeypatch): + calls: list[list[str]] = [] + made: dict[str, list[float]] = {} + monkeypatch.setattr(embeddings, "embed_texts", _fake(calls, made=made)) + monkeypatch.setattr(qmod, "VECTOR_CACHE_MAX", 3) + texts = [f"别的:第{i}问" for i in range(5)] + got = await qmod._vectors(texts) + assert got == [made[t] for t in texts] and len({tuple(v) for v in got}) == 5, "一次超过上限也每句拿到自己的向量" + assert list(qmod._VECTORS) == texts[2:], "只留最近的 3 句" + await qmod._vectors([texts[2]]) + await qmod._vectors(["别的:又一问"]) + assert list(qmod._VECTORS) == [texts[4], texts[2], "别的:又一问"], "最近用到的留着,最久没用的先丢" + + +async def test_two_callers_at_once_send_the_batch_once(monkeypatch): + """页面并行取 /api/evals/signals 和 /api/ledger/stats:同一批问题不各发一遍。""" + import asyncio + + calls: list[list[str]] = [] + inner = _fake(calls) + + async def slow(texts): + await asyncio.sleep(0.02) + return await inner(texts) + + monkeypatch.setattr(embeddings, "embed_texts", slow) + batch = _qs("同类:上周客单价", "别的:会员数") + a, b = await asyncio.gather(cluster_questions(batch), cluster_questions(batch)) + assert calls == [["同类:上周客单价", "别的:会员数"]] + assert [c.count for c in a] == [c.count for c in b] == [1, 1] diff --git a/backend/tests/test_schedules_api.py b/backend/tests/test_schedules_api.py index 10b623f1..e3d5f753 100644 --- a/backend/tests/test_schedules_api.py +++ b/backend/tests/test_schedules_api.py @@ -43,8 +43,8 @@ SCHEDULE_KEYS = {"id", "name", "workflow_id", "workflow_name", "cron", "timezone # 定时的目标(next-round X4):种类、id、叫法、现在的名字、还在不在 "target_kind", "target_id", "target_label", "target_name", "target_exists"} FIRE_KEYS = {"id", "fire_at", "kind", "status", "run_id", "run_status", "reason", "notified", "compare", "created_at", - # 目标是评测题集时这一次触发建的那次评测 - "eval_run_id"} + # 目标是评测题集时这一次触发建的那次评测;目标是数据源维护时这一次建了哪些任务、哪几步没执行(别的目标为 null) + "eval_run_id", "inputs"} @pytest.fixture(autouse=True) diff --git a/backend/tests/test_usage_meter.py b/backend/tests/test_usage_meter.py index c69259f6..ed0c35b5 100644 --- a/backend/tests/test_usage_meter.py +++ b/backend/tests/test_usage_meter.py @@ -5,7 +5,10 @@ 不当成 0 元; - metered 只计生成过程自己的调用(含它在一步里起的任务);调用方在两次 yield 之间做的事(发起运行)不计——运行的用量 另有 llm.end,包进来就重复了;嵌套时内层并进外层; -- answer_ops 末尾的 usage 记下这一轮建图的每一次调用(起草、自查修正各一次);复核的结果带 usage。 +- answer_ops 末尾的 usage 记下这一轮建图的每一次调用(起草、自查修正各一次);复核的结果带 usage; +- 读到 done / look / ask 那一行之后照样把模型的流读完再关(读完只为拿到流末尾才报的用量):输出 token 报全了、金额估得出, + 不再是下限;模型在那一行之后还往下写一大段闲话的,读到上限就当场关流、照实记成没报全; +- 评测的作答(app/eval/answer.Driver)记的助手用量就是问数据页收到的那条 usage:同一个口径。 模型都是剧本,不调真接口;价格用目录里登记过的模型 id,名字之外都是虚构的。 """ @@ -190,9 +193,19 @@ async def client(): yield c +async def _sse(client, body: dict) -> list[dict]: + events = [] + async with client.stream("POST", "/api/copilot/generate-stream", json=body) as r: + async for line in r.aiter_lines(): + if line.startswith("data: "): + events.append(json.loads(line[6:])) + return events + + async def test_answer_ops_ends_with_the_usage_of_every_build_call(client, monkeypatch): - """起草、自查修正各一次。生成过程读到 done 就关流(后面的闲话不要),流末尾才报的输出 token 拿不到:这两次照样计次数、 - 计已报的输入 token,记成估不出(unpriced_calls),金额只是估得出的那部分——不悄悄少算。""" + """起草、自查修正各一次。读到 done 那一行之后照样把模型的流读完(模型照约定 done 就是最后一行,读完只多收流末尾那一段 + 用量):输出 token 报全了、金额估得出,这一轮的花费不再是下限(第 1 波的遗留:以前读到 done 就关流,输出 token 一个都 + 拿不到,两次都记成估不出)。""" import app.api.copilot as copilot model = Scripted(model=PRICED, replies=[GRAPH, FIX], tokens=(3000, 500)) @@ -201,19 +214,96 @@ async def test_answer_ops_ends_with_the_usage_of_every_build_call(client, monkey return model, PRICED monkeypatch.setattr(copilot, "get_chat_model", _model) - events = [] - async with client.stream("POST", "/api/copilot/generate-stream", json={"instruction": "数一数", "intent": "answer"}) as r: - async for line in r.aiter_lines(): - if line.startswith("data: "): - events.append(json.loads(line[6:])) + events = await _sse(client, {"instruction": "数一数", "intent": "answer"}) assert [e["op"] for e in events if e["op"] == "check"] and events[-2]["op"] == "final" spent = events[-1] assert spent["op"] == "usage", spent assert model.calls == 2, spent # 起草一次、自查修正一次 - assert spent["by_model"][PRICED]["calls"] == 2, spent - assert spent["by_model"][PRICED]["input_tokens"] == 6000 and spent["by_model"][PRICED]["output_tokens"] == 0 - assert spent["cost_usd"] == pytest.approx(estimate_cost(PRICED, 6000, 0), abs=1e-6) - assert spent["unpriced_calls"] == 2, "关流前没报全的两次记成估不出" + row = spent["by_model"][PRICED] + assert row["calls"] == 2, spent + assert (row["input_tokens"], row["output_tokens"]) == (6000, 1000), "流末尾才报的输出 token 也收到了" + assert spent["unpriced_calls"] == 0, "读完了的流用量报全了,不记估不出" + assert spent["cost_usd"] == pytest.approx(estimate_cost(PRICED, 6000, 1000), abs=1e-6) + + +async def test_a_look_line_also_waits_for_the_usage_at_the_end_of_the_stream(client, monkeypatch): + """读到 look 停下查看、ask 等用户回答:同样把这一次的流读完再关,查看前那一次的输出 token 不丢。""" + import app.api.copilot as copilot + + looked = _ops({"op": "look", "kind": "table", "id": "不存在的表", "need": "看看有哪些列"}) + model = Scripted(model=PRICED, replies=[looked, _ops({"op": "ask", "question": "要按哪一天算?"})], tokens=(1200, 80)) + + async def _model(*_a, **_k): + return model, PRICED + + monkeypatch.setattr(copilot, "get_chat_model", _model) + events = await _sse(client, {"instruction": "看一下再问", "intent": "answer"}) + spent = events[-1] + assert spent["op"] == "usage" and model.calls == 2, [e["op"] for e in events] + row = spent["by_model"][PRICED] + assert (row["input_tokens"], row["output_tokens"], row["unpriced_calls"]) == (2400, 160, 0), spent + + +class Chatty(Scripted): + """done 那一行之后还往下写一大段闲话(不守约定),用量照样在最后一段才报。""" + + chatter: int = 40 + + async def _astream(self, messages, stop=None, run_manager=None, **kwargs): + lines = self._next() + inp, out = self.tokens + for i, line in enumerate(lines): + meta = {"input_tokens": inp, "output_tokens": 0, "total_tokens": inp} if i == 0 else None + yield ChatGenerationChunk(message=AIMessageChunk(content=line + "\n", **({"usage_metadata": meta} if meta + else {}))) + for _ in range(self.chatter): + yield ChatGenerationChunk(message=AIMessageChunk(content="这是一段写在最后一行之后、没人要的闲话。\n")) + yield ChatGenerationChunk(message=AIMessageChunk(content="", usage_metadata={ + "input_tokens": 0, "output_tokens": out, "total_tokens": out})) + + +async def test_chatter_after_the_last_line_is_cut_off_and_recorded_as_not_fully_reported(client, monkeypatch): + """读完流只为拿用量,不为闲话付钱:最后一行之后又收到超过上限的字就当场关流。这一次的用量没报全,照实记成估不出,不当成 + 报全了,也不悄悄当成 0 元。""" + import app.api.copilot as copilot + from app.api import copilot_gen + + model = Chatty(model=PRICED, replies=[_ops({"op": "ask", "question": "要按哪一天算?"})], tokens=(500, 300)) + + async def _model(*_a, **_k): + return model, PRICED + + monkeypatch.setattr(copilot, "get_chat_model", _model) + assert model.chatter * len("这是一段写在最后一行之后、没人要的闲话。\n") > copilot_gen.DRAIN_CHARS + events = await _sse(client, {"instruction": "按天看销售额", "intent": "answer"}) + assert events[-1]["op"] == "usage" and model.calls == 1, [e["op"] for e in events] + row = events[-1]["by_model"][PRICED] + assert (row["calls"], row["input_tokens"], row["output_tokens"], row["unpriced_calls"]) == (1, 500, 0, 1), row + + +async def test_the_eval_driver_records_the_same_usage_the_chat_page_receives(client, monkeypatch): + """评测的作答(app/eval/answer.Driver._generate → _take)收下的助手用量,就是问数据页经 SSE 收到的那条 usage:同一个 + 生成过程(answer_ops)、同一份计量,读完流之后两边都是报全了的数。""" + import app.api.copilot as copilot + from app.eval import answer + + replies = [_ops({"op": "ask", "question": "要按哪一天算?"})] + model = Scripted(model=PRICED, replies=replies, tokens=(700, 90)) + + async def _model(*_a, **_k): + return model, PRICED + + monkeypatch.setattr(copilot, "get_chat_model", _model) + chat = (await _sse(client, {"instruction": "按天看销售额", "intent": "answer"}))[-1] + assert chat["op"] == "usage" + driver = answer.Driver("按天看销售额", datasource_ids=[], ctx=answer.TurnContext( + suite_id="s", suite_name="虚构题集", eval_run_id="e", case_id=None, case_seq=1, actor=None)) + ops, refused = await driver._generate("按天看销售额", None) + assert refused is None and any(o.get("op") == "ask" for o in ops) + driver._take(ops) + got = driver.turn.usage["assistant"] + assert got["by_model"] == chat["by_model"] and got["cost_usd"] == chat["cost_usd"], (got, chat) + assert got["by_model"][PRICED]["output_tokens"] == 90 and got["unpriced_calls"] == 0 async def test_a_build_on_an_unpriced_model_says_it_cannot_be_priced(client, monkeypatch): diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 81b7835f..59bcda87 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -19,6 +19,8 @@ import { ToolsPage } from './pages/ToolsPage' import { KnowledgePage } from './pages/KnowledgePage' import { SettingsPage } from './pages/SettingsPage' import { SchedulesPage } from './pages/schedules/SchedulesPage' +// 评测(下一轮第 4 波,评测-D8):顶级导航,搭建者以上可见 +import { EvalsPage } from './pages/evals/EvalsPage' import { NotFound } from './pages/NotFound' // 应用平台:使用面(FE-5)、搭建面的应用编辑(FE-6) import { AppsHome } from './pages/apps/AppsHome' @@ -188,6 +190,10 @@ function Shell() { } /> } /> } /> + {/* 评测:/evals 题集列表,/evals/signals、/evals/snapshots 两个页签,/evals/suites/:id 题集,/evals/runs/:id 一次评测 */} + } /> + } /> + } /> {/* 应用:固定路径(inbox)写在 /apps/:appId 前面,读的人一眼看出它不是应用 id */} } /> {/* 要在 /apps/:appId 之前:create 不是应用 id */} @@ -347,10 +353,11 @@ function Nav({ stalled }: { stalled: CatalogCheck[] }) { const live = studio.startsWith('run') || chat === 'run' const pages = usePages() - // 整条导航约 640px 高(九个导航项),窗口再矮,最底下的遥测点和署名就被切掉、够不着。 - // 矮窗口(521–660px)按高度收紧到 500px 左右(导航项 36px),≤520px 再收一档到 460px 左右(导航项 32px、 - // 图标和字之间只留 2px),而不是让导航滚动:导航一旦 overflow,挂在右侧的悬停提示和遥测浮层也会被 - // 一起裁掉。加导航项时在 600px、520px、480px 高的窗口里量一遍(check-shell 的「窗口高 …px」) + // 整条导航约 690px 高(十个导航项),窗口再矮,最底下的遥测点和署名就被切掉、够不着。 + // 矮窗口(521–660px)按高度收紧到 540px 左右(导航项 36px),≤520px 再收一档到 470px 左右(导航项 30px、 + // 图标和字之间只留 2px:17px 图标 + 2px + 11px 字正好放下),而不是让导航滚动:导航一旦 overflow,挂在右侧的 + // 悬停提示和遥测浮层也会被一起裁掉。加导航项时在 600px、520px、480px 高的窗口里量一遍(check-shell 的「窗口高 …px」)。 + // 第十项「评测」加进来时量过:≤520px 那档还是 32px 的话,480px 高的窗口里遥测点超出视口 12px,所以收到 30px return (