mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 00:11:59 +00:00
证据目录的省略警告照实说省略了什么:放不下不一定省略了查询。真实运行里查询全部列出、只是有的只列前几行,警告却写「0 次只列编号和摘要()。写作模型看不到这些查询的明细」。现在分三种写:有查询只列编号和摘要的照旧;查询都列出、有的只列前几行的写「N 次查询都列出了数据,其中 M 次只列前 3 行」并按来源列次数;只省略了表和字段说明的照实说。事件多带 compact、compact_rows,各来源带 compact;时间线按同样三种写标题。另外把测试夹具、注释和 check-decode 里沿用的那次运行的真实查询次数换成虚构的次数或概括说法
This commit is contained in:
@@ -3277,6 +3277,7 @@ class WriterCatalog:
|
||||
- trimmed:放不下、有省略;nodes 是各来源的查询数 [{node_id, title, total, full, compact, omitted}],
|
||||
full 列出前 20 行、compact 只列前几行、omitted 只列编号和摘要;omitted 是只列摘要的查询编号
|
||||
- empty_sources:结构化结论为空的来源(Agent 的名字)
|
||||
- compact_rows:只列前几行的查询列了几行
|
||||
清单有省略、或有来源的结构化结论为空时,报告节点加写作规则(ABSENCE_RULE);有省略时组装文档另查缺失断言
|
||||
"""
|
||||
|
||||
@@ -3287,6 +3288,12 @@ class WriterCatalog:
|
||||
nodes: list[dict[str, Any]] = field(default_factory=list)
|
||||
omitted: list[str] = field(default_factory=list)
|
||||
empty_sources: list[str] = field(default_factory=list)
|
||||
compact_rows: int = _PROMPT_COMPACT_ROWS
|
||||
|
||||
@property
|
||||
def compact(self) -> int:
|
||||
"""列出了数据、但只列了前 compact_rows 行的查询次数。"""
|
||||
return sum(int(r.get("compact") or 0) for r in self.nodes)
|
||||
|
||||
@property
|
||||
def guarded(self) -> bool:
|
||||
|
||||
@@ -189,12 +189,21 @@ async def model_window(session: Any, spec: ModelSpec, model_id: str) -> int | No
|
||||
|
||||
|
||||
def _trimmed_message(writer: WriterCatalog) -> str:
|
||||
"""写作目录有省略时的那条警告(给人看):各来源省略了几次,写作模型看不到哪些明细。"""
|
||||
per = "、".join(f"{r['title']}省略 {r['omitted']} 次" for r in writer.nodes if r["omitted"])
|
||||
shown = writer.queries - len(writer.omitted)
|
||||
return (f"证据目录超出篇幅(约 {writer.budget} 字,按写作模型的上下文窗口确定):{writer.queries} 次查询中 "
|
||||
f"{shown} 次列出了数据,{len(writer.omitted)} 次只列编号和摘要({per})。写作模型看不到这些查询的明细,"
|
||||
"报告不得据此断言没有数据")
|
||||
"""写作目录有省略时的那条警告(给人看):省略的是什么、各来源省略了几次,写作模型看不到哪些明细。
|
||||
|
||||
放不下不一定省略了查询:查询全都列出、只是有的只列前几行,或者只省略了表和字段的说明。以前一律按「省略查询」写,
|
||||
全部列出时成了「0 次只列编号和摘要()。写作模型看不到这些查询的明细」,说的不是实情。"""
|
||||
head = f"证据目录超出篇幅(约 {writer.budget} 字,按写作模型的上下文窗口确定):"
|
||||
if writer.omitted:
|
||||
per = "、".join(f"{r['title']}省略 {r['omitted']} 次" for r in writer.nodes if r["omitted"])
|
||||
shown = writer.queries - len(writer.omitted)
|
||||
return (f"{head}{writer.queries} 次查询中 {shown} 次列出了数据,{len(writer.omitted)} 次只列编号和摘要({per})。"
|
||||
"写作模型看不到这些查询的明细,报告不得据此断言没有数据")
|
||||
if writer.compact:
|
||||
per = "、".join(f"{r['title']} {r['compact']} 次" for r in writer.nodes if r.get("compact"))
|
||||
return (f"{head}{writer.queries} 次查询都列出了数据,其中 {writer.compact} 次只列前 {writer.compact_rows} 行"
|
||||
f"({per})。写作模型看不到这些查询其余的行,报告不得据此断言没有数据")
|
||||
return f"{head}{writer.queries} 次查询都列出了数据,表和字段、知识库检索的说明有省略"
|
||||
|
||||
|
||||
def report_entities(spec: GraphSpec, node: GraphNode) -> str:
|
||||
@@ -475,7 +484,8 @@ async def run_report(state: GraphState, ctx: NodeContext) -> dict[str, Any]:
|
||||
if writer.trimmed:
|
||||
ctx.emit(EventType.LOG, level="warn", code="report_catalog_trimmed", message=_trimmed_message(writer),
|
||||
budget=writer.budget, window=window, queries=writer.queries, omitted=len(writer.omitted),
|
||||
nodes=[{k: r[k] for k in ("node_id", "title", "total", "omitted")} for r in writer.nodes])
|
||||
compact=writer.compact, compact_rows=writer.compact_rows,
|
||||
nodes=[{k: r[k] for k in ("node_id", "title", "total", "omitted", "compact")} for r in writer.nodes])
|
||||
messages = _messages(ctx, state, writer, claims)
|
||||
started = time.perf_counter()
|
||||
spent: list[dict[str, Any]] = []
|
||||
|
||||
@@ -207,3 +207,45 @@ async def test_the_window_comes_from_the_provider_entry_then_the_catalog():
|
||||
finally:
|
||||
await session.execute(delete(Provider).where(Provider.id == row.id))
|
||||
await session.commit()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 警告说的是实情
|
||||
|
||||
|
||||
def _writer(nodes, omitted=()):
|
||||
from app.engine.evidence import WriterCatalog
|
||||
|
||||
return WriterCatalog(text="", budget=51200, queries=sum(r["total"] for r in nodes), trimmed=True, nodes=nodes,
|
||||
omitted=list(omitted))
|
||||
|
||||
|
||||
def _row(title, total, *, full=0, compact=0, omitted=0):
|
||||
return {"node_id": title, "title": title, "total": total, "full": full, "compact": compact, "omitted": omitted}
|
||||
|
||||
|
||||
def test_the_warning_names_omitted_queries_when_there_are_some():
|
||||
from app.engine.nodes.report import _trimmed_message
|
||||
|
||||
text = _trimmed_message(_writer([_row("会员专项", 26, full=26), _row("卡券核销", 79, full=20, omitted=59)],
|
||||
omitted=[f"Q{i}" for i in range(1, 60)]))
|
||||
assert "105 次查询中 46 次列出了数据,59 次只列编号和摘要(卡券核销省略 59 次)" in text
|
||||
assert "看不到这些查询的明细" in text
|
||||
|
||||
|
||||
def test_all_queries_listed_but_some_only_first_rows_is_not_called_omitted():
|
||||
"""放不下但查询全都列出,只是有的只列前几行:不写「0 次只列编号和摘要()」,写清是行数有省略。"""
|
||||
from app.engine.nodes.report import _trimmed_message
|
||||
|
||||
writer = _writer([_row("会员专项", 26, full=26), _row("卡券核销", 79, full=30, compact=49)])
|
||||
text = _trimmed_message(writer)
|
||||
assert writer.compact == 49
|
||||
assert f"105 次查询都列出了数据,其中 49 次只列前 {writer.compact_rows} 行(卡券核销 49 次)" in text
|
||||
assert "看不到这些查询其余的行" in text
|
||||
assert "只列编号和摘要" not in text and "()" not in text
|
||||
|
||||
|
||||
def test_only_the_table_notes_were_cut():
|
||||
from app.engine.nodes.report import _trimmed_message
|
||||
|
||||
text = _trimmed_message(_writer([_row("会员专项", 26, full=26)]))
|
||||
assert "26 次查询都列出了数据,表和字段、知识库检索的说明有省略" in text and "()" not in text
|
||||
|
||||
@@ -773,9 +773,24 @@ function explainLog(code: string | undefined, message: string, d: Record<string,
|
||||
// 写作目录超出篇幅(report.py _trimmed_message)。读结构化字段(queries、omitted、nodes),不从原话里认
|
||||
const total = typeof d.queries === 'number' ? d.queries : undefined
|
||||
const omitted = typeof d.omitted === 'number' ? d.omitted : undefined
|
||||
const compact = typeof d.compact === 'number' ? d.compact : undefined
|
||||
const rows: any[] = Array.isArray(d.nodes) ? d.nodes : []
|
||||
const per = rows.filter((r) => typeof r?.omitted === 'number' && r.omitted > 0 && str(r?.title))
|
||||
.map((r) => `${r.title} 省略 ${formatNumber(r.omitted)}/${formatNumber(r.total)} 次`)
|
||||
const all = total != null ? `${formatNumber(total)} 次查询都列出了数据` : '查询都列出了数据'
|
||||
if (omitted === 0) {
|
||||
// 放不下但一次查询都没省略:有的只列了前几行,或者只省略了表和字段的说明。不写「0 次只列了编号和摘要」
|
||||
const rowsPer = rows.filter((r) => typeof r?.compact === 'number' && r.compact > 0 && str(r?.title))
|
||||
.map((r) => `${r.title} ${formatNumber(r.compact)}/${formatNumber(r.total)} 次只列前几行`)
|
||||
return {
|
||||
title: compact ? `证据目录超出篇幅,${all},${formatNumber(compact)} 次只列了前几行`
|
||||
: compact === 0 ? `证据目录超出篇幅,${all},表和字段的说明有省略`
|
||||
: `证据目录超出篇幅,${all},部分明细有省略`,
|
||||
sub: rowsPer.length ? rowsPer.join('、') : undefined,
|
||||
next: '写作模型看到了每次查询,但有的只看到前几行;其余的行仍在证据台账里,报告不得据此断言没有数据。'
|
||||
+ '需要完整明细时,请换用上下文窗口更大的写作模型,或让 Agent 在 SQL 中先汇总再交出结果',
|
||||
}
|
||||
}
|
||||
return {
|
||||
title: omitted != null
|
||||
? `证据目录超出篇幅,${total != null ? `${formatNumber(total)} 次查询中 ` : ''}${formatNumber(omitted)} 次只列了编号和摘要`
|
||||
|
||||
@@ -1336,6 +1336,19 @@ await section('报告核对 report.checked、口径卡的台账(可点击证
|
||||
{ node_id: 'c', title: '活动内容', total: 42, omitted: 22 }] })])).find((s) => s.code === 'report_catalog_trimmed')
|
||||
check('report_catalog_trimmed:标题说几次查询只列了编号和摘要', trimmed?.title === '证据目录超出篇幅,164 次查询中 74 次只列了编号和摘要', trimmed?.title)
|
||||
check('……副标题按来源列省略了几次(没省略的来源不列)', trimmed?.sub === '卡券核销 省略 52/79 次、活动内容 省略 22/42 次', trimmed?.sub)
|
||||
// 放不下但查询全都列出、只是有的只列前几行:不写「0 次只列了编号和摘要」
|
||||
const rowsOnly = flatten(mod.decodeRun([E(1, 'node.started', 'w', { node_type: 'report', label: '写报告' }),
|
||||
E(2, 'log', 'w', { level: 'warn', code: 'report_catalog_trimmed', message: '证据目录超出篇幅', budget: 51200, window: null,
|
||||
queries: 140, omitted: 0, compact: 38, compact_rows: 3, nodes: [{ node_id: 'a', title: '会员专项', total: 60, omitted: 0, compact: 0 },
|
||||
{ node_id: 'b', title: '卡券核销', total: 80, omitted: 0, compact: 38 }] })])).find((s) => s.code === 'report_catalog_trimmed')
|
||||
check('……查询全都列出、有的只列前几行:标题照实说,不写「0 次只列了编号和摘要」',
|
||||
rowsOnly?.title === '证据目录超出篇幅,140 次查询都列出了数据,38 次只列了前几行' && !rowsOnly?.title?.includes('编号和摘要'), rowsOnly?.title)
|
||||
check('……副标题按来源列只列了前几行的次数', rowsOnly?.sub === '卡券核销 38/80 次只列前几行', rowsOnly?.sub)
|
||||
check('……下一步说其余的行仍在台账里', !!rowsOnly?.next?.includes('只看到前几行') && !!rowsOnly?.next?.includes('不得据此断言没有数据'), rowsOnly?.next)
|
||||
const notesOnly = flatten(mod.decodeRun([E(1, 'node.started', 'w', { node_type: 'report', label: '写报告' }),
|
||||
E(2, 'log', 'w', { level: 'warn', code: 'report_catalog_trimmed', message: '证据目录超出篇幅', queries: 12, omitted: 0, compact: 0,
|
||||
nodes: [{ node_id: 'a', title: '会员专项', total: 12, omitted: 0, compact: 0 }] })])).find((s) => s.code === 'report_catalog_trimmed')
|
||||
check('……只省略了表和字段的说明:照实说', notesOnly?.title === '证据目录超出篇幅,12 次查询都列出了数据,表和字段的说明有省略', notesOnly?.title)
|
||||
check('……下一步说被省略的仍在台账里、不得据此断言没有数据,以及怎么看全', !!trimmed?.next?.includes('不得据此断言没有数据')
|
||||
&& !!trimmed?.next?.includes('上下文窗口更大') && trimmed?.level === 'warn' && trimmed?.detail === TRIMMED, trimmed?.next)
|
||||
const trimmedBare = flatten(mod.decodeRun([E(1, 'log', 'w', { level: 'warn', code: 'report_catalog_trimmed', message: '目录有省略' })]))
|
||||
|
||||
Reference in New Issue
Block a user