diff --git a/backend/tests/bundle_samples.py b/backend/tests/bundle_samples.py index 7aada838..c821973f 100644 --- a/backend/tests/bundle_samples.py +++ b/backend/tests/bundle_samples.py @@ -38,6 +38,10 @@ 原样导出照样全部通过 - prose：同经营月报的口径卡和流水，正文是写作者自己写在引用旁边的普通文字（「运费统一[[m:…]]」「合计——[[m:…]]」 「SKU-[[v:…]]」「共[[m:…]]家」这类）：原样导出全部通过、总览是绿的，紧挨着核对的规则不误判正常写法 +- analysis：异动周报（分析能力一轮）。Agent 在虚构零售库 shop 上查了两周的总数（Q1）和按地区两周的销售额（Q2），做了按地区的 + 变化拆解（A1，带总数核对）、波动基线（A2，两周历史不够，算不了）、两期可比性检查（A3），结论都是 app.analysis 真算的； + 报告引用 A1 的判定语、具名的数（占比）、结果表的格，正文后面是系统写的「分析范围」（engine/analysis_scope 按虚构的数据 + 目录算出没拆的渠道）。结论和输入快照随文件附带、导出时复算过，原样导出全部通过；分析范围里的编号点得开（A2、A3 正文没引用） """ from __future__ import annotations @@ -313,6 +317,125 @@ def formats_bundle(*, card: dict[str, Any] = FORMATS_CARD, table: dict[str, Any] return build(f, events, bound) +#: 分析样本（analysis）的两份查询：虚构零售库 shop 两周的总数（Q1）、四个地区两周的销售额（Q2） +AN_TOTALS = {"columns": ["week", "gmv"], "column_types": {"week": "text", "gmv": "number"}, + "rows": [["2026-W41", 2220], ["2026-W42", 2026]], "source": "shop", + "sql": "SELECT week, SUM(amount) AS gmv FROM orders GROUP BY week ORDER BY week", "truncated": False, + "row_count": 2} +AN_REGIONS = {"columns": ["region", "week", "gmv"], "column_types": {"region": "text", "week": "text", "gmv": "number"}, + "rows": [["华东", "2026-W41", 820], ["华东", "2026-W42", 700], ["华北", "2026-W41", 600], + ["华北", "2026-W42", 560], ["华南", "2026-W41", 700], ["华南", "2026-W42", 666], + ["西南", "2026-W41", 100], ["西南", "2026-W42", 100]], "source": "shop", + "sql": "SELECT region, week, SUM(amount) AS gmv FROM orders GROUP BY region, week ORDER BY region, week", + "truncated": False, "row_count": 8} +#: Agent 的三次分析调用（工具名、请求），照分析工具交给 app.analysis.run 的写法 +AN_CALLS = [ + ("analysis_decompose", {"op": "decompose", "mode": "additive", "hypothesis": "下降是不是集中在某个地区", + "params": {"query": "Q2", "by": ["region"], "period": "week", "base": "2026-W41", + "target": "2026-W42", "measure": "gmv", "label": "销售额", + "headline": {"base": "Q1.r0.gmv", "target": "Q1.r1.gmv"}}}), + ("analysis_baseline", {"op": "baseline", "mode": None, "hypothesis": "这周的下降算不算异常", + "params": {"query": "Q1", "period": "week", "measure": "gmv", "label": "销售额"}}), + ("analysis_comparable", {"op": "comparable", "mode": None, "hypothesis": "两周能不能直接比", + "params": {"query": "Q2", "period": "week", "base": "2026-W41", "target": "2026-W42", + "measure": "gmv"}}), +] +#: 异动周报：引用 A1 的判定语、具名的数（占比）、结果表的格；A2、A3 只在系统写的分析范围里 +AN_REPORT = ("## 异动周报\n\n[[a:A1]]。华东占总变化的 [[a:A1.top_share|pct]]，华东的变化是 [[v:A1.r0.delta]]。[[see:A1]]\n\n" + "各地区的变化见下表。\n\n[[table:A1 cols=region,base,target,delta rows=0-3]]") + + +def _shop_notes() -> dict[str, Any]: + """虚构零售库 shop 的数据目录（orders 是事实表；地区、渠道是属性列，码值表列全）：分析范围按它算没拆的候选维度。""" + from app.data.catalog import make_item + + def column(label: str, measure: str, codes: tuple[str, ...] = ()) -> dict[str, Any]: + out = {"label": make_item(label, "human"), "measure": make_item(measure, "human")} + if codes: + out["codes"] = make_item({c: "" for c in codes}, "profile", "verified") + return out + + return {"orders": {"kind": make_item("fact", "human"), "columns": { + "region": column("地区", "attribute", ("华东", "华北", "华南", "西南")), + "channel": column("渠道", "attribute", ("线上", "线下")), + "week": column("周", "attribute"), "amount": column("实收金额", "flow")}}} + + +def analysis_bundle() -> dict[str, Any]: + """异动周报（analysis）的页面数据：结论按两份快照真算（app.analysis.run），Agent 节点的事件照分析工具真实的写法 + （tool.end 带结论工件和判定那一段），报告文档按写作目录真实组装，分析范围按虚构的数据目录真实算出。""" + import asyncio + from datetime import date + + from app import analysis + from app.analysis.contract import effective_settings, ledger_entry + from app.engine import analysis_scope + from app.tools.analysis import event_view, input_snapshot + + run = {**RUN, "id": "9c2e4a6b8d0f4e1a3c5b7d9f1e3a5c7b", "workflow_id": "wf-sample-anomaly", "workflow_name": "异动周报", + "started_at": "2026-10-19T01:00:00+00:00", "finished_at": "2026-10-19T01:01:30+00:00"} + graph = {"nodes": [{"id": n, "type": t, "data": {"label": label}} for n, t, label in [ + ("start", "input", "开始"), ("bot", "agent", "问数据"), ("write", "report", "写周报"), ("out", "output", "成果")]]} + files: dict[str, bytes] = {} + + def put(obj: Any) -> str: + artifact, raw = stored(obj) + files[artifact] = raw + return artifact + + def load(artifact: str) -> Any: + raw = files.get(artifact) + return json.loads(raw) if raw is not None else None + + queries = [(put(snap), snap) for snap in (AN_TOTALS, AN_REGIONS)] + # 照分析工具交给算子的写法：查询快照原样加上出处；目录事实（列的叫法、度量类型）按虚构的数据目录给，结论里记下 + snapshots = {f"Q{i}": input_snapshot(snap, ref=f"Q{i}", artifact=sha, node_id="bot") + for i, (sha, snap) in enumerate(queries, 1)} + context = {"today": date(2026, 10, 19), "catalogs": {"shop": _shop_notes()}} + ledger: list[dict[str, Any]] = [ + {"kind": "query", "node_id": "bot", "exec": 1, "artifact": sha, "call_id": f"call_{i}", "tool": "db_query__shop", + "source": "shop", "columns": snap["columns"], "rows": len(snap["rows"]), "truncated": False} + for i, (sha, snap) in enumerate(queries)] + results = [] + for n, (tool, request) in enumerate(AN_CALLS, len(queries)): + result = dict(asyncio.run(analysis.run(request, snapshots.get, settings=effective_settings(None), context=context))) + artifact = put(result) + results.append((tool, f"call_{n}", artifact, result)) + ledger.append(ledger_entry(result, node_id="bot", exec_no=1, artifact=artifact, call_id=f"call_{n}")) + catalog = build_catalog(nodes={}, ledger=ledger, loader=load) + doc = compose_doc(AN_REPORT, catalog, node_id="write", run_id=run["id"], loader=load) + doc["analysis_scope"] = analysis_scope.build(catalog, sources={"shop": {"tables": _shop_notes(), "masked": []}}, + load=load) + doc_id = put(doc) + + events: list[tuple[int, str, str | None, Any]] = [] + + def add(etype: str, node_id: str | None, data: dict[str, Any]) -> None: + events.append((len(events) + 1, etype, node_id, data)) + + add("run.started", None, {"nodes": len(graph["nodes"]), "resumed": False, "memory_scope": "default", + "collection": "default", "approval_default": None, "tool_trust": "formal", + "agent_limits": {}, "data_versions": None, "replay_protocol": 2}) + add("node.started", "bot", {"node_type": "agent", "label": "问数据"}) + for i, (sha, snap) in enumerate(queries): + add("tool.end", "bot", {"tool": "db_query__shop", "call_id": f"call_{i}", "duration_ms": 9, "query_artifact": sha, + "preview": json.dumps({"columns": snap["columns"], "rows": snap["rows"][:2]}, + ensure_ascii=False)}) + for k, (tool, call_id, artifact, result) in enumerate(results, 1): + add("tool.end", "bot", {"tool": tool, "call_id": call_id, "duration_ms": 6, "analysis_artifact": artifact, + "preview": json.dumps({"analysis_artifact": artifact}), "analysis": event_view(result, f"A{k}")}) + add("node.finished", "bot", {"duration_ms": 240, "attempt": 1, "preview": {"text": "销售额下降，主要来自华东。"}, + "evidence": ledger}) + add("llm.end", "write", {"model": "mock", "output": "（写作草稿）"}) + add("report.checked", "write", {"doc_artifact": doc_id, "ok": True}) + add("node.finished", "write", {"duration_ms": 20, "attempt": 1, "preview": {"text": "（报告正文）", "doc_artifact": doc_id}}) + add("issuance", "out", {"tier": "formal", "mode": "citations"}) + add("node.finished", "out", {"duration_ms": 2, "attempt": 1, "preview": {"周报": "（报告正文）"}}) + add("run.finished", None, {"output": {"周报": "（报告正文）"}}) + return build_bundle(run, events, manifest_seq=len(events), manifest_hash=manifest_hash(events), read=files.get, + exported_at=EXPORTED_AT, graph=graph, source_masks={}) + + def bundles() -> dict[str, dict[str, Any]]: """{页面名: 页面数据}""" out: dict[str, dict[str, Any]] = {} @@ -368,6 +491,7 @@ def bundles() -> dict[str, dict[str, Any]]: out["formats"] = formats_bundle() out["formats-unit"] = formats_bundle(card=FORMATS_UNIT_CARD) out["prose"] = formats_bundle(text=PROSE) + out["analysis"] = analysis_bundle() return out diff --git a/backend/tests/test_bundle_build.py b/backend/tests/test_bundle_build.py index 4172ade7..2afb0d42 100644 --- a/backend/tests/test_bundle_build.py +++ b/backend/tests/test_bundle_build.py @@ -470,7 +470,7 @@ def test_the_sample_pages_for_the_browser_check_are_sealed_for_real(): got = {name: json.loads(data_block(page)) for name, page in rendered.items()} assert set(got) == {"base", "two-reports", "metric-only", "legacy", "failed", "rejected", "continued", "degraded", "missing-doc", "missing-card", "nan", "stray", "masked-card", "masked-old-doc", "masked-source", - "formats", "formats-unit", "prose"} + "formats", "formats-unit", "prose", "analysis"} for name, bundle in got.items(): assert browser_manifest(bundle["events"]) == bundle["meta"]["manifest_hash"], name @@ -557,3 +557,15 @@ def test_the_sample_pages_for_the_browser_check_are_sealed_for_real(): # 口径卡单位带尾随空白：服务端把「元 」原样接在数后面，正文里的销售额带着那个空白 shown = {s.get("ref"): s["text"] for s in segments(got["formats-unit"])} assert shown["m:gmv"] == "74,613.65元 " and shown["m:gmv|万"] == "7.46万元 " + # 异动周报：三份分析结论（导出时都复算一致）和两份输入快照随文件附带；文档带着系统写的分析范围，正文没引用的 A2、A3 + # 只在范围里点名；分析工具的 tool.end 结论随文件附带了，带规范字符串 + analysed = got["analysis"] + kinds = sorted((a["kind"], a.get("node_id"), "b64" in a) for a in analysed["artifacts"]) + assert kinds == [("analysis_result", "bot", True)] * 3 + [("query", "bot", True)] * 2 + [("report_doc", "write", True)] + assert {a["recheck"]["status"] for a in analysed["artifacts"] if a["kind"] == "analysis_result"} == {"same"} + doc = json.loads(base64.b64decode(next(a["b64"] for a in analysed["artifacts"] if a["kind"] == "report_doc"))) + scope = doc["analysis_scope"] + assert [[i["alias"] for i in scope[k]] for k in ("phenomena", "decomposed", "refused")] == [["A3"], ["A1"], ["A2"]] + assert scope["not_decomposed"] == ["渠道"] and scope["decomposed"][0]["dims"] == ["地区"] + assert {s["ref"] for s in segments(analysed) if s["ref"].startswith("a:")} == {"a:A1.top_share|pct"} + assert all("canonical" in e for e in analysed["events"] if e["type"] == "tool.end") diff --git a/scripts/check-bundle.mjs b/scripts/check-bundle.mjs index 40c7ee30..3598421b 100644 --- a/scripts/check-bundle.mjs +++ b/scripts/check-bundle.mjs @@ -22,7 +22,9 @@ // 判定、服务端缺文件、裸 NaN、别的运行的文档、口径卡有输入取自遮罩列（卡的节点完成事件只给哈希）、同一张卡配上证据 // 目录里写着号码的老文档（文档只给哈希、页面写明原因）、设了遮罩的数据源上按地区汇总的查询、每种显示 // 格式各引用一处的经营月报、单位带尾随空白的经营月报、引用旁边是写作者自己写的普通文字的经营月报（「统一」「——」 -// 「SKU-」「共…家」，原样导出全绿）），逐份断言各项结果和原因文案；有一项无法核对的样本页总览是中性的，不是绿的； +// 「SKU-」「共…家」，原样导出全绿）、带分析结论和系统写的分析范围的异动周报（判定语、具名的数、结果表的格，范围里的 +// 编号点得开）），逐份断言各项结果和原因文案；有一项无法核对的样本页总览是中性的，不是绿的；异动周报上只改判定语、具名的数 +// 的显示并重算全部哈希，各自变红并点名； // 经营月报上只改文字取值、数字、换算成万的口径卡引用的显示文字并重算全部哈希，各自变红并点名；核对过的字不动、改它 // 旁边的（链接地址藏起数字、整段改成分隔线、文字片段冒用数字的 id；数字前面补一位数、加负号（半角、数学减号、负号 // 前面是汉字的）、后面接「0 万」，隔着显示时不占位置的控制字符补一位数，这几种变红并点名「正文里紧挨着这个数的字改变了 @@ -611,7 +613,7 @@ await section('样本页：各种形状', async () => { check('生成样本页（tests/bundle_samples.py）', false, String(e?.message ?? e)) return } - check('生成样本页（tests/bundle_samples.py）', Object.keys(samples).length === 18, Object.keys(samples).join('、')) + check('生成样本页（tests/bundle_samples.py）', Object.keys(samples).length === 19, Object.keys(samples).join('、')) await sample('base', async (s, page) => { check('base：有无法核对的项，总览是中性的「可核对的项全部通过，另有 N 项无法核对」，不是绿色的通过', neutral(s), @@ -904,6 +906,57 @@ await section('样本页：各种形状', async () => { check('prose：17 个数字和取值都显示成可点的数字、都通过', s.numberStates.pass === 17 && s.numbers === 17, JSON.stringify(s.numberStates)) }) + // 异动周报（分析能力一轮）：Agent 的三份分析结论（按地区的变化拆解、算不了的波动基线、两期可比性检查）和两份输入快照 + // 随文件附带、导出时服务端复算过；报告引用拆解的判定语、具名的数、结果表的格（连同一整张结果表），正文后面是系统写的 + // 「分析范围」。页面只核哈希、核判定语和数是不是结论里写的那几个字，不在浏览器里重算统计 + await sample('analysis', async (s, page) => { + check('analysis：总览「全部通过」，没有一项无法核对', s.overall === 'pass' && s.text === '全部通过' + && !s.items.some((i) => i.result !== 'pass'), `${s.overall}：${s.text}；${describe(s)}`) + const kinds = s.items.filter((i) => i.check === 'artifacts').map((i) => i.text) + check('analysis：③ 三份分析结论、两份输入快照、报告文档都随文件附带、通过', + kinds.filter((t) => t.includes('分析结论')).length === 3 && kinds.filter((t) => t.includes('查询快照')).length === 2 + && kinds.some((t) => t.includes('报告文档')), kinds.join(' | ')) + check('analysis：④ 输入快照按「已核对的分析结论的输入」认', hasItem(s, 'chain', 'pass', '分析结论（A1）的输入') + && hasItem(s, 'chain', 'pass', '分析结论（A2）的输入'), s.items.filter((i) => i.check === 'chain').map((i) => i.text).join(' | ')) + check('analysis：⑤ 分开数数字、取值、判定语', s.sums.numbers.includes('判定语 1 个'), s.sums.numbers) + const phrase = page.locator('.doc button.num[data-kind="analysis"]') + check('analysis：判定语是一个可点的数，读屏读作「判定语 …，通过」', await phrase.count() === 1 + && (await phrase.getAttribute('aria-label') ?? '').startsWith('判定语 销售额下降 194（-8.7%），主要来自华东（-120，占 61.9%），通过'), + await phrase.getAttribute('aria-label')) + const detail = await detailText(page, phrase) + check('analysis：点开判定语：判定语、判定标准、方法与版本（服务端带的叫法）', detail.includes('判定语') + && detail.includes('一段占总变化的一半以上算主要来源') && detail.includes('变化拆解（可加） · decompose.additive@1'), + detail.replace(/\s+/g, ' ').slice(0, 300)) + check('analysis：……附带说明、Agent 写的假设', detail.includes('附带说明') && detail.includes('Agent 写的假设') + && detail.includes('下降是不是集中在某个地区'), detail.replace(/\s+/g, ' ').slice(0, 400)) + check('analysis：……输入（角色、编号、各自的核对状态）', ['基期总数 · Q1', '本期总数 · Q1', '数据 · Q2'].every((t) => detail.includes(t)), + detail.replace(/\s+/g, ' ').slice(0, 500)) + check('analysis：……复算写导出时服务端复算的结论、本页不重算统计', detail.includes('复算一致：按封存的输入重新计算，结论逐项相同') + && detail.includes('本页只核哈希，不在浏览器里重算统计'), detail.replace(/\s+/g, ' ').slice(-300)) + const share = page.locator('.doc p button.num').filter({ hasText: /^61\.9%$/ }) + const shareDetail = await share.count() ? await detailText(page, share.first()) : '找不到「61.9%」' + check('analysis：具名的数（占比）：出处写分析结论 A1 的具名的数「top_share」，和结论里记的值一致', shareDetail.includes('分析结论 A1 的具名的数「top_share」') + && shareDetail.includes('与按显示规则渲染的结果一致'), shareDetail.replace(/\s+/g, ' ').slice(0, 300)) + const cell = page.locator('.doc p button.num').filter({ hasText: /^-120$/ }) + const cellDetail = await cell.count() ? await detailText(page, cell.first()) : '找不到「-120」' + check('analysis：结果表的格：出处写分析结论 A1 的结果表 · 第 1 行 · delta', cellDetail.includes('分析结论 A1 的结果表 · 第 1 行 · delta'), + cellDetail.replace(/\s+/g, ' ').slice(0, 300)) + check('analysis：结果表整表引用：表头是给人看的列名，数字列右对齐', (await page.locator('.doc table th').allTextContents()).join('、') === '地区、基期、本期、变化' + && await page.locator('.doc td.n button.num').count() === 12, (await page.locator('.doc table th').allTextContents()).join('、')) + + const scope = page.locator('[data-scope]') + const st = (await scope.innerText().catch(() => '')).replace(/\s+/g, ' ') + check('analysis：正文后面是系统写的「分析范围」：现象 A3、拆过 A1（按地区）、算不了 A2、没拆的渠道', st.includes('分析范围') + && st.includes('写作者改不了') && /现象 A3/.test(st) && /拆过 A1（按地区）/.test(st) && /算不了 A2/.test(st) && /没拆 渠道/.test(st), st) + const a2 = scope.locator('button.num').filter({ hasText: /^A2$/ }) + const a2Detail = await a2.count() ? await detailText(page, a2.first()) : '找不到 A2' + check('analysis：点范围里的 A2（正文没引用）：随文件附带，看得到算不了的判定语和复算', a2Detail.includes('波动基线没有做成：只有 0 期历史，至少要 8 期') + && a2Detail.includes('复算一致'), a2Detail.replace(/\s+/g, ' ').slice(0, 300)) + check('analysis：……范围里的编号不核「正文显示」', !a2Detail.includes('正文显示'), a2Detail.replace(/\s+/g, ' ').slice(0, 300)) + }) + await sample('analysis', async (s) => { + check('analysis：390 宽不横向溢出', s.overflow <= 0, `${s.overflow}px`) + }, { width: 390 }) // 不执行脚本（附件预览、快速查看）：封存哈希、运行 ID、工作流名照样看得到，状态写「尚未核对」，不是一直「正在核对」 { @@ -987,7 +1040,7 @@ await section('样本页：篡改', async () => { data.artifacts.push({ id: card.id.replace(/^./, (c) => (c === '0' ? '1' : '0')), kind: 'retrieval', masked: true }) return '多一份标着遮罩的检索快照' }, (s) => [...turnsRed('artifacts')(s), - ['原因写明', hasItem(s, 'artifacts', 'fail', '只有查询快照、口径卡和报告文档会因遮罩不随文件附带')]]) + ['原因写明', hasItem(s, 'artifacts', 'fail', '只有查询快照、口径卡、分析结论和报告文档会因遮罩不随文件附带')]]) // 遮罩卡在服务端被改过时，服务端照样只给 id、另标 mismatch（tests/test_bundle_masked_cards.py 守着服务端这一边）： // 不能因为不嵌内容就和正常的遮罩卡一样「可核对的项全部通过」 await tamper(samples['masked-card'], '遮罩卡标成服务端的文件与 id 不一致', (data) => { @@ -1083,6 +1136,29 @@ await section('样本页：篡改', async () => { ...turnsRed('numbers', ['events', 'manifest', 'artifacts', 'chain', 'meta'])(s), ['报告徽标不通过', s.reports[0]?.result === 'fail' && s.reports[0].parts.includes('不通过 1'), JSON.stringify(s.reports[0])], ], { seal: true }) + // 分析结论：只改正文显示的判定语、具名的数（引用不动、重算全部哈希），⑤ 里只有那一项不通过，原因点名正文显示的字 + for (const [label, ref, change] of [['判定语', 'a:A1', (t) => t.replace('华东', '华南')], ['具名的数', 'a:A1.top_share|pct', () => '71.9%']]) { + await tamper(samples.analysis, `分析结论的${label}的显示并重算全部哈希`, (data) => rewriteDoc(data, (doc) => { + const seg = doc.blocks.flatMap((b) => b.units).flatMap((u) => u.segments).find((x) => x.ref === ref) + if (!seg) return null + const text = change(seg.text) + doc.markdown = doc.markdown.replace(seg.text, text) + seg.text = text + seg.cite.rendered = text + return { what: `片段 ${seg.id}（${ref}）`, seg } + }), (s, forged) => { + const failed = s.items.filter((i) => i.check === 'numbers' && i.result === 'fail' && !i.folded) + return [ + ...turnsRed('numbers', ['events', 'manifest', 'artifacts', 'chain'])(s), + ['⑤ 里只有这一项不通过', failed.length === 1, `${failed.length} 项`], + ['原因点名正文显示的字', failed[0]?.text.includes(`正文显示「${forged.seg.text}」`), failed[0]?.text ?? '没有不通过的项'], + ...(ref === 'a:A1' ? [['判定语另写结论里的判定语，不写「引用的值是空值」', failed[0]?.text.includes('分析结论里的判定语是「销售额下降 194（-8.7%），主要来自华东') + && !failed[0]?.text.includes('空值'), failed[0]?.text ?? '']] : []), + ['正文里那一处标成不通过', s.failedButtons.length === 1 && s.failedButtons[0].includes(`${forged.seg.text}，不通过`), + s.failedButtons.join(' | ')], + ] + }, { seal: true }) + } // 只改显示文字、引用值不动、重算全部哈希（验收里的 T4、T5）：经营月报上改文字取值、千分位加货币单位的数字、换算成万的 // 口径卡引用，各自只有那一项不通过。原因点名正文显示的字和引用的值，数字另写按显示规则应显示的字 const shownForgeries = [