mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 00:01:58 +00:00
真实模型验收查出的五条修正：①比率拆解里比率本身是有量纲的比率（客单价，元 / 单）时不套百分数，以前 160.76 显示成 16,076.0%，各段分子都不超过分母（退款率这类占比）照旧按百分数；②写作规则补两条——判定语一句只引用一份结论、不把几份判定语用逗号拼成一句，「核对不了」是系统的核对说明不是结论；③说法核对的来源判断看整个分句（主来源写在判断词后面的「主要由订单量放大带来」也能认），点名的段按去掉结尾量词的词根兜底（订单量 认 订单数）；④数据目录还没维护时候选维度照实说目录没维护、提示怎么补，不说成认不出事实表；⑤评测估不出每题花费时金额照 0 记但记成未计价，账本上看得出；金额上限照旧按累计金额判
This commit is contained in:
ThearchyHelios
@@ -233,7 +233,8 @@ _RATE_COLUMNS = ("pct", "share")
|
||||
|
||||
def _rate_formats(columns: Sequence[str], rate: Sequence[str] = _RATE_COLUMNS,
|
||||
points: Sequence[str] = ()) -> dict[str, str]:
|
||||
"""结果表各列记的换算：rate 里的列（占比、变化率、权重、比率本身）记 pct，points 里的列（比率的变化）记 pp。"""
|
||||
"""结果表各列记的换算：rate 里的列（占比、变化率、权重）记 pct，points 里的列（比率的变化）记 pp。
|
||||
比率本身（如客单价）不是占比，不进 rate——套上百分数会显示成 16,062.5%。"""
|
||||
return {**{c: CONV_PCT for c in rate if c in columns},
|
||||
**{c: CONV_POINTS for c in points if c in columns}}
|
||||
|
||||
@@ -695,11 +696,13 @@ def _ratio_result(env: Env, params: dict[str, Any], used: list[str], segs: list[
|
||||
return build(env, params, used, table=table, values=values, verdict=verdict, criterion=criterion,
|
||||
phrase=ph.with_caveats(main, caveats, skip=("comparability",) if outcome is None else ()),
|
||||
caveats=caveats,
|
||||
# 权重、各段的比率按百分数显示；结构效应、自身效应、合计影响是比率的变化，按百分点
|
||||
formats=_rate_formats(columns, rate=("w_base", "w_target", "r_base", "r_target"),
|
||||
# 权重（占比）按百分数；结构效应、自身效应、合计影响是比率的变化，按百分点。各段的比率和整体比率本身：
|
||||
# 各段分子都不超过分母时它是个占比（退款率），按百分数；否则它是有量纲的比率（客单价，元 / 单，可以大于 1），
|
||||
# 照原值显示——一律套百分数会把客单价 160.76 显示成 16,076.0%
|
||||
formats=_rate_formats(columns, rate=("w_base", "w_target") + (("r_base", "r_target") if proportion else ()),
|
||||
points=("mix", "rate", "total")),
|
||||
named={k: CONV_PCT for k in ("r_base_total", "r_target_total")}
|
||||
| {k: CONV_POINTS for k in ("delta", "mix_total", "rate_total")}
|
||||
named={k: CONV_POINTS for k in ("delta", "mix_total", "rate_total")}
|
||||
| ({k: CONV_PCT for k in ("r_base_total", "r_target_total")} if proportion else {})
|
||||
| {"mix_share": CONV_PCT, "rate_share": CONV_PCT})
|
||||
|
||||
|
||||
|
||||
@@ -538,18 +538,47 @@ def _subjects(result: Mapping[str, Any]) -> list[str]:
|
||||
return out
|
||||
|
||||
|
||||
#: 「订单数」写在句子里常写成「订单量」「订单数变多」：比到这几类量词结尾的同一个词根时按同一个段算
|
||||
_SUFFIXES = ("数量", "笔数", "单数", "数", "量", "额")
|
||||
|
||||
|
||||
def _stem(text: str) -> str:
|
||||
"""段的词根：「订单数」「订单量」都是「订单」。去掉结尾的量词，词根至少两个字（「单数」不能把「订单数」拆成「订」，
|
||||
所以短的量词先试）。"""
|
||||
for suffix in _SUFFIXES:
|
||||
if text.endswith(suffix) and len(text) - len(suffix) >= 2:
|
||||
return text[: -len(suffix)]
|
||||
return text
|
||||
|
||||
|
||||
def _named(result: Mapping[str, Any], names: str) -> list[str]:
|
||||
"""分句里点名的段（原样，按出现的先后）。ASCII 的名字要整词出现（「A」不算点名了「AB」）。"""
|
||||
"""分句里点名的段（原样，按出现的先后）。ASCII 的名字要整词出现（「A」不算点名了「AB」）。
|
||||
除了原样的名字，去掉结尾量词的词根也算点名（「订单量」点名了「订单数」）：写作者换个量词的说法很常见。
|
||||
词根只在原样一个都没认出来时才用，免得把「线上」认成「线上渠道」。"""
|
||||
flat = _same_length_norm(names)
|
||||
hits: list[tuple[int, str]] = []
|
||||
for key, original in _segment_names(result).items():
|
||||
|
||||
def find(key: str) -> float:
|
||||
at = flat.find(key)
|
||||
while at >= 0:
|
||||
before, after = flat[at - 1:at], flat[at + len(key):at + len(key) + 1]
|
||||
if not (key.isascii() and any(ch.isascii() and ch.isalnum() for ch in (before, after) if ch)):
|
||||
hits.append((at, original))
|
||||
break
|
||||
return at
|
||||
at = flat.find(key, at + 1)
|
||||
return -1.0
|
||||
|
||||
hits: list[tuple[float, str]] = []
|
||||
for key, original in _segment_names(result).items():
|
||||
at = find(key)
|
||||
if at >= 0:
|
||||
hits.append((at, original))
|
||||
if not hits:
|
||||
for key, original in _segment_names(result).items():
|
||||
stem = _stem(key)
|
||||
if stem == key:
|
||||
continue
|
||||
at = find(stem)
|
||||
if at >= 0:
|
||||
hits.append((at, original))
|
||||
return [name for _, name in sorted(hits)]
|
||||
|
||||
|
||||
@@ -831,7 +860,10 @@ def _object(sentence: Sentence, hit: Hit, left: int, right: int) -> tuple[str, .
|
||||
return (names[hit.start:hit.end],)
|
||||
if hit.side != "before":
|
||||
return (names[left:right],)
|
||||
spans = [names[left:hit.end]]
|
||||
# 「主导」「贡献最大」这类，主语可以在判断词前面（「线上主导」），也可以在它后面（「主要由订单量放大带来」；
|
||||
# 「主要是华东」）：先看整个分句（近的在前），再看前面一个个分句。只看判断词前面会把
|
||||
# 「这波涨幅主要由订单量放大带来」认成没点名，或者错认到前一个分句的名上
|
||||
spans = [names[left:right]]
|
||||
cut = left - 1
|
||||
while cut > 0:
|
||||
prev, _ = _clause(sentence.text, cut, cut)
|
||||
@@ -1095,7 +1127,10 @@ ANALYSIS_RULES_PROMPT = (
|
||||
"没做成的原因，不下这类判断。"
|
||||
"没有对应的分析结论，就只陈述数字和变化。\n"
|
||||
"- 拆解只说明变化来自哪里，不说明为什么。写原因（因为、由于、导致、受…影响）要有知识库原话或运行输入作依据，"
|
||||
"否则写成待核实的推测，比如「可能与促销结束有关，待核实」。"
|
||||
"否则写成待核实的推测，比如「可能与促销结束有关，待核实」。\n"
|
||||
"- 判定语整句引用（[[a:A1]]）时一句只引用一份结论，不要把几份不同维度、不同期间的判定语用逗号拼成一句；"
|
||||
"要说两件事就分成两句，各自挂各自的依据。\n"
|
||||
"- 「核对不了」「未能核对」是系统对核对情况的说明，不是结论：不要把它写进正文当结论，也不要据此下判断。"
|
||||
)
|
||||
#: 写作目录里没有分析结论时，拼在写作规则后面
|
||||
ANALYSIS_NONE_PROMPT = (
|
||||
|
||||
@@ -47,6 +47,8 @@ NOTE_NO_NOTES = "拆解所用的数据源还没有数据目录，候选维度核
|
||||
NOTE_NO_DECOMPOSE = "这次没有按维度拆解，候选维度没有核对"
|
||||
NOTE_NO_QUERY = "拆解所用的查询取不回来，候选维度核对不了"
|
||||
NOTE_NO_FACT = "拆解所用查询的表在数据目录里认不出哪张是事实表，候选维度核对不了"
|
||||
NOTE_NOT_MAINTAINED = ("数据目录还没标表类型和字段的度量类型（目录还没维护），列不出候选维度；"
|
||||
"在数据目录里把事实表、属性列这些标上，这一处就能列出没拆过的维度")
|
||||
|
||||
|
||||
def _alias_order(alias: str) -> tuple[int, str]:
|
||||
@@ -172,6 +174,19 @@ def _related(notes: Mapping[str, Any], tables: Mapping[str, Any]) -> list[tuple[
|
||||
return out
|
||||
|
||||
|
||||
def _maintained(tables: Mapping[str, Any]) -> bool:
|
||||
"""目录维护过没有：任何一张表标了类型，或者任何一个字段标了度量类型。"""
|
||||
for notes in tables.values():
|
||||
if not isinstance(notes, Mapping):
|
||||
continue
|
||||
if _value(notes.get("kind")):
|
||||
return True
|
||||
for body in (notes.get("columns") or {}).values():
|
||||
if isinstance(body, Mapping) and _value(body.get("measure")):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _fact_tables(names: list[str], tables: Mapping[str, Any]) -> list[tuple[str, dict[str, Any]]] | None:
|
||||
"""查询里的事实表；认不出的为 None。"""
|
||||
found = [hit for name in names if (hit := _table_notes(tables, name))]
|
||||
@@ -227,7 +242,9 @@ def _undecomposed(decomposes: list[dict[str, Any]], catalog: Mapping[str, Any],
|
||||
masked = {name_key(str(c)) for c in [*_names(facts_of.get("masked")), *_names(snapshot.get("mask_columns"))]}
|
||||
facts = _fact_tables(sql_tables(str(snapshot.get("sql") or "")), tables)
|
||||
if facts is None:
|
||||
return None, NOTE_NO_FACT
|
||||
# 目录还没维护（表类型、字段的度量类型都没标）时说清楚是目录没维护，不说成「认不出事实表」——
|
||||
# 后者听着像数据的问题，实际上补一下目录就能列
|
||||
return None, NOTE_NO_FACT if _maintained(tables) else NOTE_NOT_MAINTAINED
|
||||
for table, notes in [*facts, *(r for t, n in facts for r in _related(n, tables))]:
|
||||
for column in (notes.get("columns") or {}):
|
||||
known.setdefault(name_key(str(column)), (source, table, name_key(str(column))))
|
||||
|
||||
@@ -469,12 +469,16 @@ class EvalExecution(runtime.JobRun):
|
||||
预留了多少))，评完按实际找补。"""
|
||||
from app.engine.spend import eval_spend
|
||||
|
||||
per_item = float(((self.budget.get("estimate") or {}).get("per_item_usd")) or 0.0) * max(1, len(self.arms))
|
||||
# 估不出金额的题集（没评过、模型又不在价格目录里）按 0 预留，但记成「金额估不出」：拿 0 去比上限等于说它不花钱，
|
||||
# 账上也得看得出这一次其实有几笔没计价
|
||||
estimate = (self.budget.get("estimate") or {}).get("per_item_usd")
|
||||
priced = estimate is not None
|
||||
per_item = float(estimate or 0.0) * max(1, len(self.arms))
|
||||
cap = self.budget.get("per_run_max_usd")
|
||||
if cap is not None and self.spent + self.reserved + per_item > float(cap) + 1e-9:
|
||||
return LIMIT_RUN.format(limit=float(cap)), None
|
||||
daily = self.budget.get("daily_max_usd")
|
||||
day = await eval_spend.charge(per_item, limit=float(daily) if daily is not None else None, priced=True)
|
||||
day = await eval_spend.charge(per_item, limit=float(daily) if daily is not None else None, priced=priced)
|
||||
if day is None:
|
||||
return LIMIT_DAILY.format(limit=float(daily)), None
|
||||
self.reserved += per_item
|
||||
|
||||
@@ -189,3 +189,19 @@ def test_a_contradicting_sentence_next_to_the_phrase_is_caught(driver_catalog):
|
||||
assert ac.results_of({"A1": stored}) == {}
|
||||
assert ac.results_of({"A1": stored}, lambda alias, entry: driver_catalog["A1"]["result"])["A1"]["verdict"]["code"] \
|
||||
== "driver"
|
||||
|
||||
|
||||
def test_a_factor_source_named_after_the_verb_is_recognised():
|
||||
"""V5（真实模型验收 6/2）：因子拆解正确的主来源被判不符。「这波涨幅主要由订单量放大带来」里主来源写在判断词后面
|
||||
（「订单量」），以前只看判断词前面，认成没点名、或者错认到前一个分句的「客单价」上。"""
|
||||
result = {"op": "decompose", "mode": "factor", "status": "ok",
|
||||
"verdict": {"code": "driver", "subjects": [["订单数"]], "direction": "up"},
|
||||
"table": {"columns": ["factor", "base", "target", "pct", "contrib", "share"],
|
||||
"labels": {"factor": "因子", "share": "占变化的比重"},
|
||||
"rows": [["订单数", "900", "1100", "0.2222", "200", "0.998"],
|
||||
["客单价", "160", "160", "0", "0", "0.002"]]},
|
||||
"values": {"top_share": "0.998"}, "phrase": "销售额上升 200，主要来自订单数的变化"}
|
||||
text = "这波涨幅主要由订单量放大带来，客单价基本没动。"
|
||||
sentence = ac.Sentence(text=text, shown=text, base=0, analyses=["A2"], names=[None] * (len(text) + 1),
|
||||
pieces=[], unit=0, phrases=[], inputs=False, quoted=False)
|
||||
assert [v["code"] for v in ac.check_sentence(sentence, {"A2": result})] == []
|
||||
|
||||
@@ -105,7 +105,7 @@ def test_the_ratio_decomposition_formats_weights_as_percent_and_effects_as_point
|
||||
result = run_sync(analyze("decompose", "ratio", snaps=[rates], query="Q3", by=["region"], period="week",
|
||||
base="2026-W41", target="2026-W42", numerator="refunds", denominator="orders",
|
||||
label="退款率"))
|
||||
# 权重、各段的比率是比率本身 → pct；结构效应、自身效应、合计影响是比率的变化 → pp
|
||||
# 权重是占比、各段分子都不超过分母，比率本身也是占比 → pct；结构效应、自身效应、合计影响是比率的变化 → pp
|
||||
assert result["table"]["formats"] == {"w_base": CONV_PCT, "w_target": CONV_PCT, "r_base": CONV_PCT,
|
||||
"r_target": CONV_PCT, "mix": CONV_POINTS, "rate": CONV_POINTS,
|
||||
"total": CONV_POINTS}
|
||||
@@ -355,4 +355,21 @@ def test_the_verifiable_report_renders_the_conclusion_format():
|
||||
assert "正文显示61.9%（与按显示规则渲染的结果一致）" in share
|
||||
# 出处面板写结果表记的列名（「占变化的比重」），不把列 id 当叫法
|
||||
cell = next(text for text in out["opened"].values() if "结果表 · 第 1 行 · 占变化的比重" in text)
|
||||
assert "正文显示61.9%" in cell
|
||||
assert "正文显示61.9%" in cell
|
||||
|
||||
def test_a_ratio_with_a_unit_is_not_shown_as_a_percentage():
|
||||
"""比率自身是有量纲的比率（客单价 = 销售额 / 订单数，元 / 单，可以大于 1）时不套百分数：以前整体比率 160.76 显示成
|
||||
16,076.0%，各段的 160.62 显示成 16,062.5%，判定语里却是「客单价下降 9.89（-6.2%）」。各段分子都不超过分母的（退款率这种
|
||||
占比）照旧按百分数。"""
|
||||
rows = [["华东", "2026-W41", 160_000, 1000], ["华东", "2026-W42", 141_000, 1000],
|
||||
["华南", "2026-W41", 180_000, 1100], ["华南", "2026-W42", 175_000, 1100]]
|
||||
snaps = snap("Q5", "SELECT region, week, gmv, orders FROM sales GROUP BY region, week",
|
||||
["region", "week", "gmv", "orders"], rows,
|
||||
types={"region": "text", "week": "text", "gmv": "number", "orders": "number"})
|
||||
result = run_sync(analyze("decompose", "ratio", snaps=[snaps], query="Q5", by=["region"], period="week",
|
||||
base="2026-W41", target="2026-W42", numerator="gmv", denominator="orders",
|
||||
label="客单价"))
|
||||
assert result["table"]["formats"] == {"w_base": CONV_PCT, "w_target": CONV_PCT, "mix": CONV_POINTS,
|
||||
"rate": CONV_POINTS, "total": CONV_POINTS}
|
||||
assert "r_base_total" not in result["value_formats"] and "r_target_total" not in result["value_formats"]
|
||||
assert result["value_formats"]["mix_share"] == CONV_PCT
|
||||
|
||||
@@ -352,3 +352,29 @@ def test_e1_follows_the_analysis_inputs_down_the_drill():
|
||||
assert _query_inputs(found["a2"], found, {"q1": {}, "q2": {}}) == ["q1", "q2"]
|
||||
assert _query_inputs({"inputs": ["gone"]}, found, {"q1": {}}) == [], "取不回的输入不算"
|
||||
|
||||
|
||||
|
||||
|
||||
async def test_an_estimate_that_cannot_be_computed_still_marks_the_calls_unpriced(monkeypatch):
|
||||
"""真实模型验收 V10：题集从没评过、模型又不在价格表里时估不出每题花费。以前按 0 预留却记成「有价」，
|
||||
评测账本上看不出这些都是未计价的调用；现在金额照 0 记，但记成估不出。"""
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models_eval import EvalSuite
|
||||
from app.engine.spend import eval_spend
|
||||
from app.eval import runner
|
||||
from app.eval.variants import normalize_variant
|
||||
from tests import eval_samples as es
|
||||
|
||||
async def no_estimate(*args, **kwargs):
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(runner, "estimate", no_estimate)
|
||||
before = await eval_spend.daily()
|
||||
suite_id = await es.make_suite("none", cases=[{"question": "问 1", "expectations": [{"kind": "ask"}]}],
|
||||
budget={"max_parallel": 1})
|
||||
async with SessionLocal() as session:
|
||||
suite = await session.get(EvalSuite, suite_id)
|
||||
[run] = await runner.start(session, suite, [normalize_variant({})], actor="评测测试")
|
||||
await es.wait_eval(run.id)
|
||||
after = await eval_spend.daily()
|
||||
assert after.get("unpriced_calls", 0) > before.get("unpriced_calls", 0), after
|
||||
|
||||
Reference in New Issue
Block a user