mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-22 22:51:59 +00:00
合入复核修复(说法核对):结论句判定回到原来的词表(扩充的判断词只给裁判排先后)、表格正文不核、补因果与来源的口子、说法表按样本调整(96 句样本表)、连续按末尾方向核、复核认三种日志且只写说明、分析结论算可引用的来源、自动修复对认不出的取值照 claims 做、候选维度写明只认码值表、scan 加只按词认
This commit is contained in:
@@ -43,7 +43,7 @@ def codes(found: list[dict]) -> list[str]:
|
||||
|
||||
@pytest.mark.parametrize("text, claim", [
|
||||
("下降主要来自华东。", "source"), ("增长主要由线上贡献。", "source"), ("下降集中在华东、华北。", "source"),
|
||||
("绝大部分来自线下。", "source"), ("华东拖累了整体。", "source"), ("线上带动了增长。", "source"),
|
||||
("增长绝大部分来自线下。", "source"), ("华东拖累了整体。", "source"), ("线上带动了增长。", "source"),
|
||||
("华东贡献最大。", "source"), ("下降主要是华东下滑造成的。", "source"),
|
||||
("变化分散在各地区。", "dispersed"), ("各地区都在下降。", "dispersed"), ("各渠道普遍下滑。", "dispersed"),
|
||||
("全面下降。", "dispersed"),
|
||||
@@ -53,7 +53,7 @@ def codes(found: list[dict]) -> list[str]:
|
||||
("销售额明显偏离往常。", "anomaly"), ("超出过去 12 周的常规波动范围。", "anomaly"),
|
||||
("销售额明显高于正常水平。", "anomaly"), ("波动剧烈。", "anomaly"),
|
||||
("本周创历史新高。", "record"), ("销售额创新高。", "record"), ("前所未有。", "record"),
|
||||
("在常规波动范围内。", "normal"), ("属于正常波动。", "normal"), ("并不异常。", "normal"),
|
||||
("在常规波动范围内。", "normal"), ("属于正常波动。", "normal"), ("这一降幅并不异常。", "normal"),
|
||||
("线上销售额持续上升。", "trend"), ("呈上升趋势。", "trend"), ("一路走低。", "trend"), ("稳步增长。", "trend"),
|
||||
("连续三周下降。", "trend"), ("连续下滑。", "trend"),
|
||||
("没有明显的上升或下降趋势。", "no_trend"), ("上升趋势不明显。", "no_trend"), ("基本平稳。", "no_trend"),
|
||||
@@ -138,13 +138,13 @@ def test_dafu_needs_a_baseline_outside_the_band_in_the_same_direction():
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text, claim", [
|
||||
("并未出现明显异常。", "normal"), # 否定隔着「出现明显」
|
||||
("本周没有出现异常。", "normal"),
|
||||
("订单数并未出现明显异常。", "normal"), # 否定隔着「出现明显」
|
||||
("本周销售额没有出现异常。", "normal"),
|
||||
("不在正常范围内。", "anomaly"),
|
||||
("不属于正常波动。", "anomaly"),
|
||||
("没有显著高于华东。", "no_diff"),
|
||||
("并非主要来自华东。", "source_not"),
|
||||
("并不分散。", "source"),
|
||||
("下降并非主要来自华东。", "source_not"),
|
||||
("下降并不分散。", "source"),
|
||||
("没有持续上升。", "no_trend"),
|
||||
])
|
||||
def test_a_negation_flips_to_the_opposite_class(text, claim):
|
||||
@@ -196,8 +196,10 @@ def test_ascii_names_need_word_boundaries_but_chinese_neighbours_are_fine():
|
||||
result["table"]["rows"][0][0] = "AB"
|
||||
result["table"]["rows"][1][0] = "A"
|
||||
assert said("下降主要来自AB店。", "A1", results={"A1": result}) == []
|
||||
assert said("下降主要来自XAB。", "A1", results={"A1": result}) == [], "XAB 不是点名 AB,也没点名别的段"
|
||||
assert codes(said("下降主要来自A。", "A1", results={"A1": result})) == [], "一个字的名字不认"
|
||||
# XAB 不是点名 AB、一个字的名字不认:都当没点名哪一段,说的来源不是拆解里的段
|
||||
[miss] = said("下降主要来自XAB。", "A1", results={"A1": result})
|
||||
assert miss["code"] == ac.MISMATCH and "不是这份拆解里的段" in miss["message"]
|
||||
assert codes(said("下降主要来自A。", "A1", results={"A1": result})) == [ac.MISMATCH]
|
||||
|
||||
|
||||
def test_named_segments_are_matched_across_full_and_half_width():
|
||||
@@ -254,6 +256,29 @@ def test_trends_check_the_direction_and_runs_check_the_length():
|
||||
assert codes(said("线上销售额持续上升。", "A1", results={"A1": flat})) == [ac.MISMATCH]
|
||||
|
||||
|
||||
def test_runs_follow_the_run_direction_and_the_number_of_periods_they_claim():
|
||||
"""「连续 N 期上升」按趋势结论记的末尾连续同向核(run_direction、run_length):方向一致、期数不少于 N(没写 N 的至少
|
||||
3 期),不看整体的趋势判定——判定语在方向相反时写「但最近已连续 N 期上升」。"""
|
||||
rising = fake_result("trend", None, "rising", direction="up", values={"run_length": "4", "run_direction": "up"})
|
||||
assert said("线上销售额已连续 4 周上升。", "A1", results={"A1": rising}) == []
|
||||
[miss] = said("线上销售额持续增长,已连续 5 周上升。", "A1", results={"A1": rising})
|
||||
assert miss["code"] == ac.MISMATCH and miss["claim"] == "trend" and "只有 4 期,不够说连续 5 期" in miss["message"]
|
||||
# 整体下降、最近连续上升:「连续上升」对得上,「持续上升」对不上
|
||||
turned = fake_result("trend", None, "falling", direction="down", values={"run_length": "4", "run_direction": "up"})
|
||||
assert said("订单数最近已连续四周上升。", "A1", results={"A1": turned}) == []
|
||||
assert codes(said("订单数已连续三周下降。", "A1", results={"A1": turned})) == [ac.MISMATCH]
|
||||
assert codes(said("订单数持续上升。", "A1", results={"A1": turned})) == [ac.MISMATCH]
|
||||
# 没有明显趋势、末尾连续下降:也能说连续
|
||||
flat = fake_result("trend", None, "no_trend", direction=None, values={"run_length": "3", "run_direction": "down"})
|
||||
assert said("订单数三连降。", "A1", results={"A1": flat}) == []
|
||||
assert codes(said("订单数四连降。", "A1", results={"A1": flat})) == [ac.MISMATCH]
|
||||
assert codes(said("订单数三连涨。", "A1", results={"A1": flat})) == [ac.MISMATCH]
|
||||
# 没记 run_direction 的老结论按整体的趋势方向
|
||||
old = fake_result("trend", None, "rising", direction="up", values={"run_length": "3"})
|
||||
assert said("订单数连续上升。", "A1", results={"A1": old}) == []
|
||||
assert codes(said("订单数连续下降。", "A1", results={"A1": old})) == [ac.MISMATCH]
|
||||
|
||||
|
||||
def test_differences_check_the_verdict_and_the_order_of_the_two_groups():
|
||||
differs = fake_result("differ", "rate", "differs", subjects=[["西南"], ["华东"]], direction="up")
|
||||
assert said("西南的退款率显著高于华东。", "A1", results={"A1": differs}) == []
|
||||
@@ -330,11 +355,65 @@ def test_a_cause_outside_the_decomposition_needs_its_own_evidence():
|
||||
{"text": "由于促销结束导致下降。", "inputs": True},
|
||||
{"text": "由于数据只到 10 月 16 日,本周不完整。"},
|
||||
{"text": "由于样本少,差异不能下结论。"},
|
||||
{"text": "以上是本周下降的主要原因。"}, # 原因只是指代:列出原因的那几句各自核
|
||||
{"text": "下降的原因主要是:"},
|
||||
])
|
||||
def test_guesses_quotes_inputs_and_data_caveats_are_not_unsupported_causes(sentence):
|
||||
assert ac.check_sentence(ac.Sentence(**sentence), {}) == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text, found", [
|
||||
("下降主要由于促销结束。", [(ac.CAUSAL, "causal")]), # 「主要由」不吃掉「由于」
|
||||
("华东下降主要由于促销结束。", [(ac.CAUSAL, "causal")]), # 原因在「由于」后面,前面的华东不算
|
||||
("促销结束导致华东下降。", [(ac.CAUSAL, "causal")]), # 原因在「导致」前面
|
||||
("华东受促销结束影响下降。", [(ac.CAUSAL, "causal")]), # 原因在「受…影响」中间
|
||||
("下降主要由促销结束造成。", [(ac.MISMATCH, "source"), (ac.CAUSAL, "causal")]),
|
||||
("下降主要来自线上渠道。", [(ac.MISMATCH, "source")]), # 说的来源不是拆解(按地区)里的段
|
||||
("下降主要来自新客流失。", [(ac.MISMATCH, "source")]),
|
||||
("华东下降主要来自线上渠道。", [(ac.MISMATCH, "source")]), # 「主要来自」看后面
|
||||
("下降主要由华东造成。", []),
|
||||
("华东是本周下降的主要原因。", []), # 原因点名的是段:按来源核
|
||||
("下降的主要原因是华东门店客流减少。", []),
|
||||
("华南是本周下降的主要原因。", [(ac.MISMATCH, "source")]),
|
||||
("下降主要源于华东。", []),
|
||||
("华东下滑,拖累了整体。", []), # 拖累看整个分句,没点名的按主要来源
|
||||
])
|
||||
def test_sources_and_causes_attached_to_a_decomposition(text, found):
|
||||
"""挂了按地区拆解、判「主要来自华东」的结论(复核 R4):原因按因果词的那一边认;说的来源得是拆解里的段。"""
|
||||
assert [(v["code"], v["claim"]) for v in said(text, "A1", results={"A1": fake_result()})] == found
|
||||
|
||||
|
||||
def test_a_source_needs_no_name_when_the_clause_quotes_the_phrase_or_says_structure():
|
||||
driver = fake_result()
|
||||
phrase = ac.Sentence(text="下降主要来自以下地区:\x00\x00\x00\x00。", analyses=["A1"], phrases=[(11, 15, "A1")])
|
||||
assert ac.check_sentence(phrase, {"A1": driver}) == []
|
||||
elsewhere = ac.Sentence(text="\x00\x00\x00\x00,下降主要来自这一地区。", analyses=["A1"], phrases=[(0, 4, "A1")])
|
||||
assert codes(ac.check_sentence(elsewhere, {"A1": driver})) == [ac.MISMATCH], "判定语在别的分句"
|
||||
mix = fake_result(mode="ratio", verdict="mix_driven", subjects=[["华东"]])
|
||||
assert said("退款率上升主要来自结构变化。", "A1", results={"A1": mix}) == []
|
||||
rate = fake_result(mode="ratio", verdict="rate_driven", subjects=[["华东"]])
|
||||
assert said("退款率上升主要来自各段自身的变化。", "A1", results={"A1": rate}) == []
|
||||
assert [v["claim"] for v in said("退款率上升主要来自结构变化。", "A1", results={"A1": rate})] == ["source", "mix"]
|
||||
|
||||
|
||||
def test_before_side_sources_look_back_to_the_nearest_named_clause():
|
||||
driver = fake_result()
|
||||
assert said("华东门店表现最弱,贡献最大。", "A1", results={"A1": driver}) == []
|
||||
assert said("华南基本持平,华东贡献最大。", "A1", results={"A1": driver}) == [], "最近的分句点了名就不再往前找"
|
||||
assert codes(said("新客贡献最大。", "A1", results={"A1": driver})) == [ac.MISMATCH]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text, results, found", [
|
||||
("下降主要由华东造成。", {}, [ac.UNCITED]), # 没有拆解可比:并进来源
|
||||
("下降主要由华东造成。", {"A2": fake_result()}, [ac.UNCITED]), # 华东是目录里拆解的段:并进来源
|
||||
("下降主要由促销结束造成。", {"A2": fake_result()}, [ac.UNCITED, ac.CAUSAL]),
|
||||
("下降主要由促销结束造成。", {}, [ac.UNCITED]),
|
||||
])
|
||||
def test_without_a_decomposition_attached_a_cause_merges_into_the_source_only_when_it_could_be_one(text, results,
|
||||
found):
|
||||
assert codes(said(text, results=results)) == found
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 文档:哪里不扫、怎么挂、版本
|
||||
# --------------------------------------------------------------------------
|
||||
@@ -413,6 +492,19 @@ def test_system_rendered_phrases_quotes_headings_and_code_are_not_scanned():
|
||||
assert codes(ac.check_doc(doc, catalog())) == [ac.UNCITED]
|
||||
|
||||
|
||||
def test_table_cells_written_by_the_writer_are_not_checked():
|
||||
"""表格的格不核(表头、正文都不核):一格是一个值不是一句话,状态列的「异常」、说明列的「主要来自线上」不是判断
|
||||
(judge.claim_priority 也不送表格的格)。表格外的段落、列表照核。"""
|
||||
raw = ("本周销售额 [[m:gmv]]。[[see:m:gmv]]\n\n| 门店 | 状态 | 说明 |\n|---|---|---|\n| 甲店 | 异常 | 主要来自线上 |\n"
|
||||
"| 乙店 | 正常 | 持续增长 |\n\n```\n异常订单激增\n```\n\n## 主要来自华东的下降\n\n- 华东拖累明显\n")
|
||||
doc = compose_doc(raw, catalog(), analysis_claims="check")
|
||||
tables = {u["id"] for b in doc["blocks"] if b["type"] == "table" for u in b["units"]}
|
||||
assert len(tables) == 9, "表格切成了一格一格"
|
||||
[v] = found(doc)
|
||||
assert v["code"] == ac.UNCITED and v["text"] == "拖累" and v["unit"] not in tables
|
||||
assert [v["text"] for v in found(verify_doc(doc, catalog()))] == ["拖累"]
|
||||
|
||||
|
||||
def test_rendered_values_name_segments_but_are_not_judgement_words():
|
||||
# [[v:…]] 渲染出来的「华东」能点名,渲染出来的「异常」不是写作者下的判断
|
||||
doc = {"markdown": "", "blocks": [{"type": "paragraph", "units": [{"id": "u0", "kind": "claim", "cites": ["A1"],
|
||||
@@ -438,6 +530,24 @@ def test_compose_and_verify_bind_the_sentence_to_its_see_marker():
|
||||
assert [v["code"] for v in verify_doc(tampered, cat)["violations"]] == [ac.UNCITED]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("raw, want", [
|
||||
("销售额 [[m:gmv]],下降主要来自线上渠道。[[see:A1]]", [(ac.MISMATCH, "主要来自")]),
|
||||
("销售额 [[m:gmv]],下降主要来自新客流失。[[see:A1]]", [(ac.MISMATCH, "主要来自")]),
|
||||
("销售额 [[m:gmv]],下降主要由于促销结束。[[see:A1]]", [(ac.CAUSAL, "由于")]),
|
||||
("销售额 [[m:gmv]],下降主要由促销结束造成。[[see:A1]]", [(ac.MISMATCH, "主要由"), (ac.CAUSAL, "造成")]),
|
||||
("销售额 [[m:gmv]],下降主要来自华东。[[see:A1]]", []),
|
||||
("销售额 [[m:gmv]],下降主要来自 [[v:A1.r0.region]]。[[see:A1]]", []),
|
||||
("销售额 [[m:gmv]],下降主要来自以下地区:[[a:A1]]。", []),
|
||||
("[[a:A1]],下降主要来自这一地区。", [(ac.MISMATCH, "主要来自")]),
|
||||
])
|
||||
def test_strict_documents_close_the_source_and_cause_loopholes(raw, want):
|
||||
"""复核 R4 的几个口子(strict 下挂一份按地区拆解、判「主要来自华东」的 A1):经组装和复核都核出来。"""
|
||||
cat = catalog(A1=fake_result())
|
||||
doc = compose_doc(raw, cat, analysis_claims="strict")
|
||||
assert [(v["code"], v["text"]) for v in found(doc)] == want
|
||||
assert [(v["code"], v["text"]) for v in found(verify_doc(doc, cat))] == want
|
||||
|
||||
|
||||
def test_inputs_and_quotes_in_the_same_sentence_support_a_cause():
|
||||
doc = compose_doc("由于本周有 [[i:promo]] 活动,销售额 [[m:gmv]]。[[see:m:gmv]]", catalog(), analysis_claims="check")
|
||||
assert found(doc) == []
|
||||
@@ -454,12 +564,18 @@ def test_the_prompt_rules_depend_on_whether_there_are_conclusions():
|
||||
assert "[[a:A1]]" in ac.ANALYSIS_RULES_PROMPT
|
||||
|
||||
|
||||
def test_judgement_words_make_a_sentence_a_claim_for_the_judge():
|
||||
def test_judgement_words_rank_claims_for_the_judge_but_do_not_make_sentences_claims():
|
||||
"""判断词只管裁判送判的先后(judge._JUDGEMENT):一句话算不算结论仍按原来的方向词、因果词(evidence._DIRECTIONAL),
|
||||
结论句策略要求挂依据、出具缺口照它数——扩了就改了已发布版本的档位。"""
|
||||
from app.engine.judge import claim_priority
|
||||
|
||||
doc = compose_doc("华东异常。\n\n下面主要看三个方面。\n\n建议持续关注。\n\n近 8 周的趋势图如下:", catalog())
|
||||
doc = compose_doc("华东异常。\n\n华东异常,退款 12 单。\n\n下面是主要渠道的表现。\n\n促销期间活动效果显著。\n\n"
|
||||
"整体运行平稳,未见异常。\n\n建议持续关注。\n\n近 8 周的趋势图如下:\n\n华东退款 12 单。", catalog())
|
||||
kinds = {doc["markdown"][slice(*u["span"])]: (u["kind"], claim_priority(u, b, doc["markdown"]))
|
||||
for b, u in evidence.iter_units(doc)}
|
||||
assert kinds["华东异常。"] == ("claim", 2)
|
||||
assert kinds["下面主要看三个方面。"][0] == "connective" and kinds["建议持续关注。"][0] == "connective"
|
||||
assert kinds["近 8 周的趋势图如下:"][0] == "connective", "引出图表的话不算判断"
|
||||
for text in ("华东异常。", "下面是主要渠道的表现。", "促销期间活动效果显著。", "整体运行平稳,未见异常。", "建议持续关注。",
|
||||
"近 8 周的趋势图如下:"):
|
||||
assert kinds[text] == ("connective", None), text
|
||||
assert doc["stats"]["uncited_claims"] == 2, "只有两句带数字的算结论"
|
||||
assert kinds["华东异常,退款 12 单。"] == ("claim", 2), "结论句里的判断词让裁判先判它"
|
||||
assert kinds["华东退款 12 单。"] == ("claim", 3)
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
"""说法表的样本表(engine/analysis_claims,细案 01 §9.1):常见的周报写法逐句过 scan,每句期望认出的类(按出现的先后,
|
||||
否定翻过之后的)或「不核」(空)。以后改说法表都过这张表:误报(正常的句子被打回)让报告多重写一次、strict 下降档,
|
||||
漏报只是少核一处,宁可漏报不误报。
|
||||
|
||||
前半是第 1 波复核时的 70 句(复核第 4 节),后半是这次调整顺带看清楚的写法。和复核标记不一样的只有一句:「退款主要集中在
|
||||
生鲜品类」复核标的是判断,可它说的是退款的构成、没说变化,按「来源、集中只在说了变化时才核」不核。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from app.engine import analysis_claims as ac
|
||||
|
||||
#: 不核
|
||||
NO: tuple[str, ...] = ()
|
||||
|
||||
SAMPLES: list[tuple[str, tuple[str, ...]]] = [
|
||||
# ---- 陈述数字和变化:不核 ----
|
||||
("本周销售额 12.3 万元,环比下降 5.2%。", NO),
|
||||
("整体平稳,各品类变化不大。", NO),
|
||||
("退款率略有下降。", NO),
|
||||
("订单数明显回升,客单价基本持平。", NO), # 「明显」只是程度,不收「明显 + 涨跌」
|
||||
("环比基本持平。", NO),
|
||||
("本周数据只到 10 月 16 日,周数据不完整。", NO),
|
||||
("客单价相当于上周的 1.1 倍。", NO),
|
||||
("订单量与上周相当。", NO), # 比上期:说的是持平,不是两组差异
|
||||
# ---- 判断:核 ----
|
||||
("销售额同比增长 8.1%,主要是因为国庆假期客流增加。", ("causal",)),
|
||||
("受节假日影响,线下门店客流明显回升。", ("causal",)),
|
||||
("华东销售额大幅增长 32%。", ("anomaly",)),
|
||||
("线上渠道拉动整体增长。", ("source",)),
|
||||
("华东下滑拖累了整体表现。", ("source",)),
|
||||
("各地区普遍上涨。", ("dispersed",)),
|
||||
("已连续三周下降。", ("trend",)),
|
||||
("西南地区退款率显著高于其他地区。", ("differ",)),
|
||||
("两个渠道的转化率相当。", ("no_diff",)),
|
||||
("促销效果显著,带动订单量增长。", ("source",)),
|
||||
("由于华东仓库搬迁,发货延迟导致退款增加。", ("causal", "causal")),
|
||||
("线上销售额持续增长,已连续 5 周上升。", ("trend", "trend")),
|
||||
("退款金额骤增,需重点关注。", ("anomaly",)), # 「需」在后一个分句,不管前面的判断
|
||||
("新客贡献最大。", ("source",)),
|
||||
("这主要是由于华东仓库搬迁。", ("causal",)),
|
||||
("整体来看,各渠道均有所下降。", ("dispersed",)),
|
||||
("销量下降主要由促销结束造成。", ("source", "causal")),
|
||||
("下降主要来自客流减少。", ("source",)),
|
||||
("退款主要集中在生鲜品类。", NO), # 复核标的是判断;说的是构成、没说变化,不核
|
||||
# ---- 业务上的「异常」、数据质量:不核 ----
|
||||
("本周异常订单 35 单,已全部处理。", NO),
|
||||
("部分门店数据异常,已提交核查。", NO),
|
||||
("部分订单异常,已退款。", NO),
|
||||
("物流异常的订单已全部退款。", NO),
|
||||
("支付异常的订单已全部退款。", NO),
|
||||
("门店营业异常 2 家。", NO),
|
||||
("整体运行平稳,未见异常。", NO), # 主语不是指标、波动
|
||||
("系统运行正常,无异常。", NO),
|
||||
# ---- 说的是构成,不是变化从哪来:不核 ----
|
||||
("本周订单主要来自线上渠道,占 65%。", NO),
|
||||
("销售额集中在周末两天。", NO),
|
||||
("华东门店集中在上海、杭州。", NO),
|
||||
("客户分散在各个城市。", NO),
|
||||
("线上渠道占主导。", NO),
|
||||
# ---- 建议、计划、预测:不核 ----
|
||||
("下一步将全面提升门店服务水平。", NO),
|
||||
("建议加大投放,拉动新客增长。", NO), # 「建议」管到后面的分句
|
||||
("通过会员活动带动复购。", NO), # 带动的不是变化
|
||||
("下周将持续提升转化率。", NO),
|
||||
("建议持续降低退款率。", NO),
|
||||
("计划逐步提高线上占比。", NO),
|
||||
("下一步将大幅提升配送效率。", NO),
|
||||
("下周重点主要是提升转化率。", NO),
|
||||
("避免缺货导致客诉。", NO),
|
||||
("预计下周将继续保持增长态势。", NO),
|
||||
# ---- 别的误报:不核 ----
|
||||
("订单来源于线上渠道。", NO), # 「来源于」不是「源于」
|
||||
("数据来源于业务系统,金额含税。", NO),
|
||||
("本周数据源表结构变化,部分字段暂缺。", NO), # 表结构不是结构变化
|
||||
# ---- 漏报,补上:核 ----
|
||||
("华东是本周下降的主要原因。", ("causal",)), # 原因点名的是拆解的段时按来源核
|
||||
("近七成的降幅来自华东。", ("source",)),
|
||||
("超过一半的下降来自华东。", ("source",)),
|
||||
("增长主要靠线上。", ("source",)),
|
||||
("降幅主要在华东。", ("source",)),
|
||||
("下降主要体现在华东。", ("source",)),
|
||||
("增长以线上为主。", ("source",)),
|
||||
("销售额创年内新高。", ("record",)),
|
||||
("销售额创近一年新高。", ("record",)),
|
||||
("整体趋势向好。", ("trend",)),
|
||||
("本周销售额波动较大。", ("anomaly",)),
|
||||
("订单量三连降。", ("trend",)),
|
||||
("退款率显著下降。", ("anomaly",)),
|
||||
("销售额远高于往常。", ("anomaly",)),
|
||||
("本周销售额出现异动。", ("anomaly",)),
|
||||
("华东和华南一涨一跌。", ("offset",)),
|
||||
# ---- 这次调整顺带看清楚的写法 ----
|
||||
("订单大幅减少。", ("anomaly",)), # 「订单」只在「订单异常」里是业务状态
|
||||
("不少门店大幅下滑。", ("anomaly",)),
|
||||
("订单异常下降。", ("anomaly",)), # 「异常」后面跟着涨跌:说的是指标
|
||||
("华东线上订单出现异常的下降。", ("anomaly",)),
|
||||
("异常的门店已全部整改。", NO),
|
||||
("两个渠道的转化率与上周相当。", NO),
|
||||
("华东和华南的退款率水平相当。", ("no_diff",)),
|
||||
("销售额并未出现明显异常。", ("normal",)),
|
||||
("本周没有出现异常。", NO),
|
||||
("退款率并未显著下降。", ("normal",)),
|
||||
("下降主要来自华东,建议加大华东投放。", ("source",)), # 「建议」在后一个分句
|
||||
("销售额\x00\x00,环比\x00\x00,主要来自华东。", ("source",)), # 引用渲染的数涂成 \x00,「环比」说的是变化
|
||||
("订单主要来自线上渠道,占比较上周提升 5 个百分点。", NO), # 变化在后一个分句:前面说的是构成
|
||||
("新增客户主要来自线上渠道。", NO),
|
||||
("增量主要来自线上。", ("source",)),
|
||||
("下降主要源于华东。", ("source",)),
|
||||
("订单以线上为主。", NO),
|
||||
("线上渠道表现突出,贡献最大。", ("source",)),
|
||||
("下降的主要原因是华东门店客流减少。", ("causal",)),
|
||||
("受促销结束影响,华东销售额下降。", ("causal",)),
|
||||
("近 8 周销售额持续下降,已连续三周下降。", ("trend", "trend")),
|
||||
("线上销售额十二连涨。", ("trend",)),
|
||||
("整体走势向下。", ("trend",)),
|
||||
("销售额创下了近一年来新高。", ("record",)),
|
||||
("创新能力最高的门店是华东店。", NO),
|
||||
("预计下周销售额将创新高。", NO),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text, expected", SAMPLES, ids=[t for t, _ in SAMPLES])
|
||||
def test_the_sample_table(text, expected):
|
||||
found = ac.scan(text)
|
||||
assert tuple(h.claim for h in found) == expected, [(h.claim, h.word) for h in found]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text, expected", SAMPLES, ids=[t for t, _ in SAMPLES])
|
||||
def test_a_sentence_is_reported_without_conclusions_exactly_when_it_carries_a_judgement(text, expected):
|
||||
"""没有分析结论可挂时:认出判断的句子至少报一处(没挂、原因没依据),不核的一处都不报。"""
|
||||
assert bool(ac.check_sentence(ac.Sentence(text=text), {})) == bool(expected)
|
||||
|
||||
|
||||
def test_the_sample_table_is_not_shorter_than_the_review():
|
||||
assert len(SAMPLES) >= 70 and len({t for t, _ in SAMPLES}) == len(SAMPLES)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text, count", [("已连续三周下降。", 3), ("线上销售额持续增长,已连续 5 周上升。", 5),
|
||||
("线上销售额十二连涨。", 12), ("连续两个月下滑。", 2), ("连续多周上升。", None),
|
||||
("订单量三连降。", 3), ("连续10周上升。", 10)])
|
||||
def test_runs_carry_the_number_of_periods_they_claim(text, count):
|
||||
[run] = [h for h in ac.scan(text) if h.run]
|
||||
assert run.count == count
|
||||
|
||||
|
||||
@pytest.mark.parametrize("label, claims", [("主要来自华东的销售额", ["source"]), ("集中在线上的订单", ["source"]),
|
||||
("建议持续降低的退款率", ["trend"]), ("部分订单异常", ["anomaly"]),
|
||||
("销售额", [])])
|
||||
def test_bare_scanning_finds_judgement_words_in_a_name(label, claims):
|
||||
"""认叫法里夹带的判断词(bare=True)只按词认:叫法是一个名字不是一句话,构成、建议计划、业务状态这些整句话的规矩不用。"""
|
||||
assert [h.claim for h in ac.scan(label, bare=True)] == claims
|
||||
assert ac.scan(label) == []
|
||||
@@ -95,6 +95,31 @@ def test_the_autofix_writes_it_in():
|
||||
assert write["data"]["config"]["analysis_claims"] == "strict"
|
||||
|
||||
|
||||
def test_an_unrecognized_value_is_left_to_its_own_fix_like_claims():
|
||||
"""analysis_claims 写了认不出的值:门禁 G3 跳过、交给校验报,自动修复的 G3 那一条也不碰它(以前顺手改成了 strict),
|
||||
由校验那条问题自己的修复管——照 claims 的做法:受管级别改成 strict,别的级别三档要人选。"""
|
||||
graph = traceable({**STRICT, "numbers": "off", "analysis_claims": "loose"})
|
||||
spec = GraphSpec.model_validate(graph)
|
||||
fixes = {f["id"]: f for f in plan_fixes(spec, publish_issues(spec, level="governed"), level="governed")}
|
||||
assert fixes["governed.report_policy:write"]["preview"] == {"field": "numbers", "before": "off", "after": "strict"}
|
||||
bad = fixes["report.analysis_claims_invalid:write"]
|
||||
assert bad["kind"] == "auto" and bad["preview"] == {"field": "analysis_claims", "before": "loose", "after": "strict"}
|
||||
out = apply_fixes(graph, list(fixes), level="governed")
|
||||
write = next(n for n in out["graph"]["nodes"] if n["id"] == "write")["data"]["config"]
|
||||
assert out["ok"] is True and write["analysis_claims"] == "strict" and write["numbers"] == "strict"
|
||||
|
||||
published = traceable({"analysis_claims": "loose"})
|
||||
spec = GraphSpec.model_validate(published)
|
||||
fid = "report.analysis_claims_invalid:write"
|
||||
[fix] = [f for f in plan_fixes(spec, publish_issues(spec, level="published"), level="published") if f["id"] == fid]
|
||||
assert fix["kind"] == "choice" and [o["value"] for o in fix["options"]] == ["off", "check", "strict"]
|
||||
assert [o["label"] for o in fix["options"]] == [option_label("analysis_claims", v) for v in ("off", "check", "strict")]
|
||||
assert "default" not in fix
|
||||
out = apply_fixes(published, [fid], {fid: "check"}, level="published")
|
||||
assert out["applied"] == [fid] and next(n for n in out["graph"]["nodes"] if n["id"] == "write")[
|
||||
"data"]["config"]["analysis_claims"] == "check"
|
||||
|
||||
|
||||
def test_a_governed_upgrade_writes_the_strict_setting():
|
||||
from app.engine.upgrade import _policy
|
||||
|
||||
|
||||
@@ -265,6 +265,61 @@ async def test_the_scope_is_written_only_when_there_are_conclusions(monkeypatch,
|
||||
assert "analysis_scope" not in doc
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 上游的可引用来源认分析结论(00-总方案 D10)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def card_entry(alias: str) -> dict:
|
||||
"""分析卡做的结论(台账条目带 item、不带 call_id)。"""
|
||||
return {**a_entry(alias, fake_result()), "item": "drop", "node_id": "card"}
|
||||
|
||||
|
||||
def agent_entry(alias: str) -> dict:
|
||||
"""Agent 的分析工具现场做的结论(带 call_id)。"""
|
||||
return {**a_entry(alias, fake_result()), "call_id": "call-1", "node_id": "ask"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cells", [True, False])
|
||||
def test_conclusions_from_an_analysis_card_are_citable_evidence(cells):
|
||||
"""「证据来自」只选了分析卡:分析卡的结论总能引用(判定语、具名的数),不报「上游没有可引用的来源」;受管级别的正式
|
||||
出具没开「单元格引用」时也不报「只有查询结果」。"""
|
||||
assert report_mod._no_evidence({"A1": card_entry("A1")}, cells) is None
|
||||
assert report_mod._no_evidence({"Q1": {"kind": "query"}, "A1": card_entry("A1")}, cells) is None
|
||||
|
||||
|
||||
def test_conclusions_from_an_agent_follow_the_cells_rule():
|
||||
catalog = {"Q1": {"kind": "query"}, "A1": agent_entry("A1")}
|
||||
assert report_mod._no_evidence(catalog, True) is None
|
||||
warning = report_mod._no_evidence(catalog, False)
|
||||
assert warning and "只有查询结果和 Agent 现场做的分析" in warning and "无法引用单元格和这些分析结论" in warning
|
||||
plain = report_mod._no_evidence({"Q1": {"kind": "query"}}, False)
|
||||
assert plain and "只有查询结果,没有口径卡指标" in plain and "Agent" not in plain
|
||||
|
||||
|
||||
def test_nothing_citable_names_analysis_conclusions_too():
|
||||
warning = report_mod._no_evidence({}, True)
|
||||
assert warning and all(kind in warning for kind in ("口径卡指标", "查询结果", "知识库检索", "运行输入", "分析结论"))
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 判断词不改一句话算不算结论:已发布版本的档位不变
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("extra", ["整体运行平稳,未见异常。", "下面是主要渠道的表现。", "促销期间活动效果显著。"])
|
||||
async def test_judgement_words_do_not_turn_connective_sentences_into_uncited_claims(monkeypatch, extra):
|
||||
"""已发布的受管版本(结论句要挂依据、出口带契约):没有数字、没有引用的话照旧是连接性的话,不因为带了「主要」「异常」
|
||||
「显著」就算没挂依据的结论、记缺口降档(判断词只给裁判排送判的先后,judge._JUDGEMENT)。"""
|
||||
graph = weekly(numbers="strict", on_violation="flag", claims="require_citation")
|
||||
graph["nodes"][-1]["data"]["config"]["contract"] = {
|
||||
"report_from": "write", "metrics_from": ["caliber"], "required": ["gmv"], "strict": True}
|
||||
script(monkeypatch, f"{PLAIN}\n\n{extra}")
|
||||
issuance = (await finish(graph)).output["_issuance"]
|
||||
assert issuance["tier"] == "formal" and issuance["gaps"] == []
|
||||
assert issuance["claims"] == {"policy": "require_citation", "uncited_claims": 0, "uncited": []}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 取值写错
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
@@ -476,3 +476,97 @@ def test_a_rewrite_counts_by_the_final_judgement():
|
||||
judge={"counts": {"supported": 2, "contradicted": 1, "insufficient": 0}})]
|
||||
assert rv.scan(good, OK_OUTPUT) == [] and rv.notes(good) == ["报告「写周报」有 1 句结论裁判认为证据不支持,改写后都有了依据"]
|
||||
assert [s.kind for s in rv.scan(bad, OK_OUTPUT)] == ["report_rewrite"] and rv.notes(bad) == []
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 分析能力一轮的信号:只写说明、不自动重来(00-总方案 §3.7)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
#: check 下判断词重写一次后仍没挂对分析结论:报告照常产出,标在报告里
|
||||
CLAIMS_LEFT = "报告有 1 处分析说法未通过核对(1 处判断没挂分析结论:「主要来自」),已要求模型重写 1 次仍未改掉:…这几处已标注"
|
||||
REFUSED = "「周报分析」的波动基线没有做成:历史只有 5 期,至少要 8 期"
|
||||
SKIPPED = "指标「销售额环比」用到的两次查询没做成可比性检查:探查超时。指标照常计算,两期能不能直接比请自行核对"
|
||||
|
||||
|
||||
def test_analysis_signals_are_registered_as_gaps_that_only_need_a_note():
|
||||
signals = rv.scan([warn(CLAIMS_LEFT, "report_analysis_claims"), warn(REFUSED, "analysis_refused"),
|
||||
warn(SKIPPED, "metric_comparability_skipped")], OK_OUTPUT)
|
||||
assert [(s.kind, s.severity) for s in signals] == [
|
||||
("report_analysis_claims", rv.DEGRADED), ("analysis_refused", rv.DEGRADED),
|
||||
("metric_comparability_skipped", rv.DEGRADED)]
|
||||
assert {s.kind for s in signals} == rv.NOTE_ONLY
|
||||
assert not rv.should_retry(signals) and not rv.NOTE_ONLY & (rv.RETRYABLE | rv.REBUILD)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_run_with_only_note_only_signals_is_annotated_without_calling_the_model():
|
||||
model = FakeModel(AssertionError("不该叫模型"))
|
||||
signals = rv.scan([warn(CLAIMS_LEFT, "report_analysis_claims"), warn(REFUSED, "analysis_refused")], OK_OUTPUT)
|
||||
result = await rv.review(model, question="q", answer="答案", signals=signals, locked=True)
|
||||
assert model.calls == 0
|
||||
assert result.verdict == "annotated" and result.retry is False and result.severity == rv.DEGRADED
|
||||
assert result.note == f"{CLAIMS_LEFT};{REFUSED}" and result.answer is None
|
||||
# 和别的信号一起出现时照常叫模型
|
||||
mixed = [*signals, rv.Signal("tool_error", "工具 x 调用失败", rv.BROKEN)]
|
||||
model = FakeModel({"note": "取数失败了一次。", "answer": "", "retry": True})
|
||||
assert (await rv.review(model, question="q", answer="答案", signals=mixed)).note == "取数失败了一次。"
|
||||
assert model.calls == 1
|
||||
|
||||
|
||||
def _left_after_rewrite(mode: str, violations: list[dict], count: int = 2) -> list[dict]:
|
||||
return [
|
||||
ev(EventType.NODE_STARTED, node_id="write", label="写周报"),
|
||||
ev(EventType.LOG, node_id="write", level="warn", code="report_repair", count=count, attempt=1,
|
||||
message=f"报告有 {count} 处未通过核对(「12」、「主要来自」),已要求模型重写(第 1 次)"),
|
||||
ev(EventType.REPORT_CHECKED, node_id="write", ok=not violations, violations=violations, repairs=1,
|
||||
analysis_claims=mode),
|
||||
]
|
||||
|
||||
|
||||
CLAIM = {"code": "analysis_claim_uncited", "text": "主要来自", "message": "「主要来自」是分析判断"}
|
||||
|
||||
|
||||
def test_check_mode_leftovers_do_not_make_the_rewrite_count_as_failed():
|
||||
"""check 下重写一次后仍有的说法违规是标注(同缺失断言,各有自己的信号):「改过一次」算改好了,不再另报 report_repair;
|
||||
复核被叫起来时照实说剩下的是哪几处。"""
|
||||
rows = [*_left_after_rewrite("check", [CLAIM]), warn(CLAIMS_LEFT, "report_analysis_claims")]
|
||||
assert [s.kind for s in rv.scan(rows, OK_OUTPUT)] == ["report_analysis_claims"]
|
||||
assert rv.notes(rows) == ["报告「写周报」首稿有 2 处没过核对,重写后只剩已标出的 1 处分析说法"]
|
||||
# 一处都没改掉:不说「改好了」,剩下的由它自己的信号说
|
||||
assert rv.notes(_left_after_rewrite("check", [CLAIM], count=1)) == []
|
||||
|
||||
|
||||
def test_absence_claim_leftovers_are_annotations_too():
|
||||
absent = {"code": "absence_claim", "text": "没有查到", "message": "缺失断言"}
|
||||
rows = _left_after_rewrite("check", [absent, CLAIM], count=3)
|
||||
assert rv.scan(rows, OK_OUTPUT) == []
|
||||
assert rv.notes(rows) == ["报告「写周报」首稿有 3 处没过核对,重写后只剩已标出的 1 处缺失断言、1 处分析说法"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode, violations", [
|
||||
("strict", [CLAIM]), # strict 下是出具缺口,不是标注
|
||||
("check", [CLAIM, {"code": "uncited_number", "text": "12"}]), # 还有别的违规
|
||||
("off", [CLAIM]),
|
||||
("check", [CLAIM] * 20), # 带满了 20 条:后面可能还有
|
||||
])
|
||||
def test_other_leftovers_still_count_as_a_failed_rewrite(mode, violations):
|
||||
rows = _left_after_rewrite(mode, violations)
|
||||
assert [s.kind for s in rv.scan(rows, OK_OUTPUT)] == ["report_repair"] and rv.notes(rows) == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_review_endpoint_does_not_call_the_model_for_a_check_mode_leftover(client, monkeypatch):
|
||||
model = FakeModel(AssertionError("不该叫模型"))
|
||||
|
||||
async def fake_model(*a, **k):
|
||||
return model, "fake"
|
||||
|
||||
monkeypatch.setattr("app.api.copilot.get_chat_model", fake_model)
|
||||
output = {"周报": "周报:销售额 45,678.5元,下降主要来自华东。",
|
||||
"_evidence": {"report_node": "write", "doc_artifact": "a" * 64, "fields": ["周报"]}}
|
||||
run_id = await _make_run(client, output, [*_left_after_rewrite("check", [CLAIM]),
|
||||
warn(CLAIMS_LEFT, "report_analysis_claims")])
|
||||
body = (await client.post("/api/copilot/review", json={"run_id": run_id, "question": "q"})).json()
|
||||
assert model.calls == 0
|
||||
assert body["verdict"] == "annotated" and body["note"] == CLAIMS_LEFT and body["retry_instruction"] is None
|
||||
assert [s["kind"] for s in body["signals"]] == ["report_analysis_claims"]
|
||||
|
||||
Reference in New Issue
Block a user