mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 01:51:57 +00:00
重评时把评分规则版本记进指纹;去掉几处没用的导入
为什么:按新规则重评之后,和别的评测比时「评分规则」那一项要照实标出变了;总表上另记重评的时间。
This commit is contained in:
@@ -13,7 +13,7 @@ from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.base import SessionLocal, utcnow
|
||||
from app.db.models_eval import EvalCase, EvalResult, EvalRun
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -66,7 +66,10 @@ async def rescore(eval_run_id: str) -> int:
|
||||
setattr(row, key, value)
|
||||
await session.flush()
|
||||
run = await session.get(EvalRun, eval_run_id)
|
||||
await runner.refresh_summary(session, run)
|
||||
# 指纹里的评分规则版本跟着换:和别的评测比时照实标出「评分规则」一项变没变
|
||||
run.fingerprint = {**dict(run.fingerprint or {}), "scorer_version": score.SCORER_VERSION}
|
||||
summary = await runner.refresh_summary(session, run)
|
||||
run.summary = {**summary, "rescored_at": utcnow().isoformat()}
|
||||
await session.commit()
|
||||
return changed
|
||||
|
||||
|
||||
@@ -33,9 +33,8 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
import copy
|
||||
import logging
|
||||
import time
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
@@ -236,8 +235,6 @@ class _Arm:
|
||||
view: dict[str, Any] | None = None
|
||||
#: 这一臂的名字(A/B 时)
|
||||
name: str | None = None
|
||||
#: 这一臂各项:{(题目 id, 第几次): 结果 id}
|
||||
items: dict[tuple[str, int], str] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -13,7 +13,7 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
import json
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from datetime import timedelta, timezone
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
@@ -22,7 +22,7 @@ from sqlalchemy import delete, select
|
||||
|
||||
from app.core.crypto import encrypt
|
||||
from app.db.base import SessionLocal, utcnow
|
||||
from app.db.models import NotifyChannel, Schedule, ScheduleFire
|
||||
from app.db.models import NotifyChannel, Schedule
|
||||
from app.db.models_eval import EvalResult, EvalRun
|
||||
from app.engine.runner import run_manager
|
||||
from app.eval import answer as answer_mod
|
||||
|
||||
@@ -198,7 +198,7 @@ async def test_reading_the_sealed_answer_and_the_per_case_sql_match(engine_on, m
|
||||
async def test_rescoring_reads_the_seal_and_never_calls_a_model(engine_on, monkeypatch, tmp_path):
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models_eval import EvalResult, EvalRun
|
||||
from app.eval import rescore
|
||||
from app.eval import rescore, score
|
||||
from app.providers import mock_model
|
||||
from tests import eval_samples as es
|
||||
from tests import ledger_samples as ls
|
||||
@@ -229,5 +229,7 @@ async def test_rescoring_reads_the_seal_and_never_calls_a_model(engine_on, monke
|
||||
async with SessionLocal() as session:
|
||||
row = (await session.execute(select(EvalResult).where(EvalResult.eval_run_id == er_id))).scalar_one()
|
||||
assert row.verdict == "pass" and row.scores[0]["outcome"] == "traced"
|
||||
assert (await session.get(EvalRun, er_id)).summary["accuracy"]["passed"] == 1
|
||||
rescored = await session.get(EvalRun, er_id)
|
||||
assert rescored.summary["accuracy"]["passed"] == 1 and rescored.summary["rescored_at"]
|
||||
assert rescored.fingerprint["scorer_version"] == score.SCORER_VERSION
|
||||
await es.drop_suites()
|
||||
|
||||
@@ -7,8 +7,6 @@
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import timedelta
|
||||
|
||||
import pytest
|
||||
|
||||
import app.db.models # noqa: F401 - conftest 按已导入的模型建表
|
||||
|
||||
Reference in New Issue
Block a user