From 9e1ff1127842b35f0f4c4ce8eb5bafc661761c2e Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Tue, 20 Oct 2026 11:39:05 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BA=A4=E5=8F=89=E8=A1=A8=E6=94=AF=E6=8C=81?= =?UTF-8?q?=E6=8C=89=E6=96=87=E5=AD=97=E5=AF=BC=E5=85=A5=E7=9A=84=E6=8C=87?= =?UTF-8?q?=E6=A0=87=EF=BC=88=E5=A6=82=E3=80=8C=E9=AB=98=E5=B3=B0=E6=97=B6?= =?UTF-8?q?=E6=AE=B5=E3=80=8D=E4=B8=80=E8=A1=8C=E5=86=99=E7=9D=80=E3=80=8C?= =?UTF-8?q?18-19=E3=80=8D=EF=BC=89=EF=BC=8C=E5=A0=B5=E4=B8=8A=E5=8D=A0?= =?UTF-8?q?=E4=BD=8D=E7=AC=A6=E9=99=B7=E9=98=B1=EF=BC=9A=E9=85=8D=E6=96=B9?= =?UTF-8?q?=E7=9A=84=20measures=20=E5=88=86=E6=AE=B5=E5=8A=A0=20text=5Fmea?= =?UTF-8?q?sures=EF=BC=88=E6=A0=87=E7=AD=BE=20=E2=86=92=20text=20/=20hour?= =?UTF-8?q?=5Frange=EF=BC=89=EF=BC=8C=E8=90=BD=E6=88=90=E6=96=87=E5=AD=97?= =?UTF-8?q?=E5=88=97=EF=BC=88=E6=97=B6=E6=AE=B5=E5=AD=98=E8=A7=84=E8=8C=83?= =?UTF-8?q?=E5=86=99=E6=B3=95=EF=BC=89=EF=BC=8C=E7=A9=BA=E6=A0=BC=E3=80=81?= =?UTF-8?q?=E5=85=AC=E5=BC=8F=E3=80=81=E5=8D=A0=E4=BD=8D=E7=AC=A6=E7=85=A7?= =?UTF-8?q?=E5=9D=97=E7=9A=84=E5=8F=96=E5=80=BC=E8=A7=84=E5=88=99=EF=BC=8C?= =?UTF-8?q?=E6=95=B0=E5=AD=97=E3=80=81=E6=97=A5=E6=9C=9F=E3=80=81=E8=A7=A3?= =?UTF-8?q?=E6=9E=90=E4=B8=8D=E4=BA=86=E7=9A=84=E6=97=B6=E6=AE=B5=E6=8A=A5?= =?UTF-8?q?=20value=5Fnot=5Ftext=20/=20value=5Fnot=5Fhour=5Frange=EF=BC=9B?= =?UTF-8?q?=E8=A7=84=E5=88=99=E8=B5=B7=E8=8D=89=E9=81=87=E5=88=B0=E6=97=A0?= =?UTF-8?q?=E6=A0=87=E9=A2=98=E6=8C=87=E6=A0=87=E5=9D=97=E9=87=8C=E6=95=B4?= =?UTF-8?q?=E8=A1=8C=E6=98=AF=E6=96=87=E5=AD=97=E7=9A=84=E8=A1=8C=EF=BC=88?= =?UTF-8?q?=E6=A0=87=E7=AD=BE=E4=B8=8D=E5=B8=A6=E8=AF=8D=E8=A1=A8=E5=8D=95?= =?UTF-8?q?=E4=BD=8D=EF=BC=89=E6=8C=89=E6=96=87=E5=AD=97=E6=8C=87=E6=A0=87?= =?UTF-8?q?=E8=B5=B7=E8=8D=89=EF=BC=8C=E5=B9=B6=E5=87=BA=E5=8D=A1=E7=89=87?= =?UTF-8?q?=E5=92=8C=E9=97=AE=E9=A2=98=E8=A6=81=E4=BA=BA=E7=A1=AE=E8=AE=A4?= =?UTF-8?q?=EF=BC=88=E3=80=8C=E4=B8=8D=E6=98=AF=E3=80=8D=E5=B0=B1=E5=8E=BB?= =?UTF-8?q?=E6=8E=89=EF=BC=8C=E5=9B=9E=E5=88=B0=E6=8C=89=E6=95=B0=E5=AD=97?= =?UTF-8?q?=E5=AF=BC=E5=85=A5=E3=80=81=E8=AF=95=E8=BF=90=E8=A1=8C=E6=8B=92?= =?UTF-8?q?=E6=94=B6=EF=BC=89=EF=BC=8C=E7=A1=AE=E8=AE=A4=E6=B8=85=E5=8D=95?= =?UTF-8?q?=E5=8A=A0=20text=5Fmeasure=20=E9=A1=B9=EF=BC=8C=E5=89=8D?= =?UTF-8?q?=E5=90=8E=E5=AF=B9=E7=85=A7=E6=8A=A5=E7=B1=BB=E5=9E=8B=E5=92=8C?= =?UTF-8?q?=E5=AD=98=E6=B3=95=E5=8F=98=E5=8C=96=EF=BC=9B=E6=95=B0=E5=80=BC?= =?UTF-8?q?=E5=8C=BA=E4=B8=8D=E5=90=88=E6=A0=BC=E7=9A=84=E6=A0=BC=E4=B8=80?= =?UTF-8?q?=E8=A1=8C=E5=90=88=E6=88=90=E4=B8=80=E6=9D=A1=E8=B5=B7=E8=8D=89?= =?UTF-8?q?=E5=A4=B1=E8=B4=A5=EF=BC=88=E4=B8=8D=E5=86=8D=E4=B8=80=E8=A1=8C?= =?UTF-8?q?=2030=20=E6=9D=A1=EF=BC=89=E3=80=82=E9=99=B7=E9=98=B1=EF=BC=9A?= =?UTF-8?q?=E4=BB=A5=E5=89=8D=E7=85=A7=E3=80=8C=E8=AF=B7=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E5=8D=A0=E4=BD=8D=E7=AC=A6=E3=80=8D=E6=8A=8A=E3=80=8C18-19?= =?UTF-8?q?=E3=80=8D=E5=8A=A0=E6=88=90=E5=8D=A0=E4=BD=8D=E7=AC=A6=EF=BC=8C?= =?UTF-8?q?=E6=95=B4=E6=A0=BC=E9=9D=99=E9=BB=98=E5=AD=98=E6=88=90=E7=A9=BA?= =?UTF-8?q?=E5=80=BC=E3=80=81=E5=B9=B2=E8=B7=91=E7=85=A7=E6=A0=B7=E9=80=9A?= =?UTF-8?q?=E8=BF=87=E2=80=94=E2=80=94=E9=9D=99=E6=80=81=E6=A0=A1=E9=AA=8C?= =?UTF-8?q?=E6=8B=92=E6=94=B6=E5=90=AB=E6=95=B0=E5=AD=97=E7=9A=84=E5=8D=A0?= =?UTF-8?q?=E4=BD=8D=E7=AC=A6=EF=BC=88placeholder=5Fdigits=EF=BC=8C?= =?UTF-8?q?=E9=87=8D=E6=94=BE=E5=92=8C=E6=B2=BF=E7=94=A8=E7=8E=B0=E8=A1=8C?= =?UTF-8?q?=E9=85=8D=E6=96=B9=E7=9A=84=E6=94=BE=E8=A1=8C=EF=BC=89=EF=BC=8C?= =?UTF-8?q?value=5Fnot=5Fnumber=20=E7=9A=84=E4=B8=8B=E4=B8=80=E6=AD=A5?= =?UTF-8?q?=E6=8C=89=E6=A0=BC=E5=AD=90=E6=83=85=E5=86=B5=E5=86=99=EF=BC=9A?= =?UTF-8?q?=E6=95=B4=E8=A1=8C=E6=96=87=E5=AD=97=E7=9A=84=E6=8F=90=E7=A4=BA?= =?UTF-8?q?=E6=94=B9=E3=80=8C=E6=8C=87=E6=A0=87=E7=B1=BB=E5=9E=8B=E3=80=8D?= =?UTF-8?q?=E5=B9=B6=E7=BB=99=E4=BF=AE=E5=A4=8D=E6=8C=89=E9=92=AE=20declar?= =?UTF-8?q?e=5Ftext=5Fmeasure=EF=BC=8C=E5=90=AB=E6=95=B0=E5=AD=97=E7=9A=84?= =?UTF-8?q?=E4=B8=8D=E5=86=8D=E6=8F=90=E7=A4=BA=E5=8A=A0=E5=8D=A0=E4=BD=8D?= =?UTF-8?q?=E7=AC=A6=EF=BC=9B=E6=A0=87=E7=AD=BE=E5=B8=A6=E5=8D=95=E4=BD=8D?= =?UTF-8?q?=E7=9A=84=E6=8B=92=E6=8C=89=E6=96=87=E5=AD=97=E5=AF=BC=E5=85=A5?= =?UTF-8?q?=EF=BC=88text=5Fmeasure=5Funit=EF=BC=89=EF=BC=8C=E6=96=87?= =?UTF-8?q?=E5=AD=97=E5=88=97=E4=B8=8D=E8=83=BD=E5=86=99=E5=8D=95=E4=BD=8D?= =?UTF-8?q?=EF=BC=88unit=5Fon=5Ftext=EF=BC=89=E3=80=82=E9=85=8D=E6=96=B9?= =?UTF-8?q?=E9=9D=A2=E6=9D=BF=E6=AF=8F=E4=B8=AA=E6=8C=87=E6=A0=87=E5=A4=9A?= =?UTF-8?q?=E4=B8=80=E4=B8=AA=E3=80=8C=E6=8C=87=E6=A0=87=E7=B1=BB=E5=9E=8B?= =?UTF-8?q?=E3=80=8D=E4=B8=8B=E6=8B=89=EF=BC=88=E6=95=B0=E5=AD=97=20/=20?= =?UTF-8?q?=E6=97=B6=E6=AE=B5=20/=20=E6=96=87=E5=AD=97=EF=BC=89=E3=80=82?= =?UTF-8?q?=E9=BB=98=E8=AE=A4=E5=80=BC=E4=B8=8D=E8=BF=9B=E8=A7=84=E8=8C=83?= =?UTF-8?q?=E5=BD=A2=E5=BC=8F=EF=BC=8C=E5=B7=B2=E5=AD=98=E9=85=8D=E6=96=B9?= =?UTF-8?q?=E7=9A=84=E5=93=88=E5=B8=8C=E4=B8=8D=E5=8F=98=E3=80=82=E6=B5=8B?= =?UTF-8?q?=E8=AF=95=20tests/test=5Frecipe=5Ftext=5Fmeasures.py=EF=BC=88?= =?UTF-8?q?=E5=90=88=E6=88=90=E5=A4=B9=E5=85=B7=EF=BC=8C=E5=81=87=E5=90=8D?= =?UTF-8?q?=EF=BC=89=EF=BC=9Bcheck-manage=20=E5=8A=A0=E6=8C=87=E6=A0=87?= =?UTF-8?q?=E7=B1=BB=E5=9E=8B=E4=B8=89=E9=A1=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/data/recipe.py | 54 ++- backend/app/data/recipe_ai.py | 3 +- backend/app/data/recipe_confirm.py | 19 +- backend/app/data/recipe_engine.py | 132 ++++++- backend/app/data/recipe_fixes.py | 86 +++- backend/app/data/recipe_notes.py | 5 +- backend/app/data/recipe_suggest.py | 97 ++++- backend/app/data/recipe_types.py | 15 +- backend/tests/test_recipe_contract.py | 6 +- backend/tests/test_recipe_engine_messages.py | 19 +- backend/tests/test_recipe_text_measures.py | 395 +++++++++++++++++++ frontend/src/lib/terms.ts | 4 + frontend/src/pages/import/RecipePanel.tsx | 28 +- frontend/src/types.ts | 5 +- scripts/check-manage.mjs | 27 +- 15 files changed, 861 insertions(+), 34 deletions(-) create mode 100644 backend/tests/test_recipe_text_measures.py diff --git a/backend/app/data/recipe.py b/backend/app/data/recipe.py index 5bc4cad5..37da06c7 100644 --- a/backend/app/data/recipe.py +++ b/backend/app/data/recipe.py @@ -265,7 +265,7 @@ _FIELD_LABEL = { "meaning": "含义", "blank": "空格的处理", "text_number": "文本数字的处理", "formula": "公式的处理", "segments": "分段", "role": "分段类型", "table": "表", "locate": "定位方式", "by": "定位依据", "title": "分段标题", "segment": "紧跟的分段", "labels": "标签", "expect": "期望的标签", - "measures": "指标列名对照", "dim": "维度", "derive": "派生列", "value": "值列", "const": "常量列", + "measures": "指标列名对照", "text_measures": "按文字导入的指标", "dim": "维度", "derive": "派生列", "value": "值列", "const": "常量列", "pick": "取值", "stop_parser": "结束本段的标签", "labels_parser": "标签解析方式", "verify": "核对方式", "against_table": "核对的基表", "keep_as": "另存", "header_rows": "表头行数", "after_title": "表头上方的标题", "columns": "列", "header": "表头", "store": "存储写法", "extra_columns": "多出的列", @@ -815,6 +815,21 @@ class _Checker: if key in seen: self.add(lp, "label_duplicate", f"标签「{label}」与「{seen[key]}」是同一个标签") seen.setdefault(key, label) + if isinstance(seg, MeasuresSegment): + self.check_text_measures(sp, seg) + + def check_text_measures(self, sp: tuple[Any, ...], seg: MeasuresSegment) -> None: + """文字指标的键与 measures 一样逐字对 labels.expect；标签末尾写着词表里的单位的是数字指标，不能按文字导入。""" + for label in seg.text_measures: + tp = sp + ("text_measures", label) + if label not in seg.labels.expect: + self.add(tp, "text_measure_unknown", + f"按文字导入的「{label}」不在期望的标签中（必须与标签逐字相同）") + continue + unit = split_unit_suffix(label)[1] + if unit is not None and unit in UNITS: + self.add(tp, "text_measure_unit", + f"标签「{label}」写着单位「{unit}」，是数字指标，不能按文字导入") @staticmethod def label_key(seg: Any, label: str) -> str | None: @@ -880,13 +895,25 @@ class _Checker: # ---- 3. 占位符只能映射空值，不能像数 ---- def check_placeholders(self) -> None: + """占位符不能像数；也不能含数字（origin 不是 replay 时）。「18-19」「约3000」不像一个数，却是数据本身： + 当成占位符，整格的内容就静默存成空值，干跑照样通过。表示无数据的符号（「·」「—」「/」「无」）不含数字； + 已提交的配方里沿用下来的（同一块、canon 相同）放行，重放也不报，免得期 3 之前存下的配方在下一期被拒。""" + literal = self.origin != "replay" + carried = self.base_placeholders() if literal else {} for si, sheet in enumerate(self.r.sheets): for bi, block in enumerate(sheet.blocks): for n, ph in enumerate(block.values.placeholders): + pp = ("sheets", si, "blocks", bi, "values", "placeholders", n, "text") if looks_numeric_text(ph.text): - self.add(("sheets", si, "blocks", bi, "values", "placeholders", n, "text"), - "placeholder_numeric", + self.add(pp, "placeholder_numeric", f"占位符「{ph.text}」看起来是一个数，不能当作占位符（占位符一律存为空值）") + elif (literal and any(ch.isdigit() for ch in ph.text) + and canon(ph.text) not in carried.get(block.id, set())): + what = ("；如果这一行写的都是文字（如时段、名称），请把这个指标的「指标类型」改为「文字」或「时段」" + if isinstance(block, CrosstabBlock) else "") + self.add(pp, "placeholder_digits", + f"占位符「{ph.text}」含数字：表示无数据的符号不含数字（如「·」「—」「无」），" + f"含数字的通常是数据本身，当作占位符会整格存成空值{what}") for si, bi, block in self.lists: total = block.rows.total_row if total is not None and any(ch.isdigit() for ch in total.pick): @@ -897,6 +924,14 @@ class _Checker: self.add(("sheets", si, "blocks", bi, "rows", "subtotals", "pick"), "period_literal", f"小计行的开头词「{sub.pick}」不能含数字（配方里不能写死年份或期次）") + def base_placeholders(self) -> dict[str, set[str]]: + """现行配方（base）里每个块的占位符（canon 后）：块 id → 集合。没有 base 时为空。""" + out: dict[str, set[str]] = {} + for sheet in self.base.sheets if self.base is not None else []: + for block in sheet.blocks: + out.setdefault(block.id, set()).update(canon(p.text) for p in block.values.placeholders) + return out + # ---- 4、9、13. 单位：键是本表列名，值在词表里，与原表标签、表头里的单位一致 ---- def check_units(self) -> None: @@ -915,7 +950,12 @@ class _Checker: for si, bi, _gi, _block, seg in self.segs: if isinstance(seg, MeasuresSegment): for label in seg.labels.expect: - if label in seg.measures: + if label not in seg.measures: + continue + if label in seg.text_measures: + # 标签写着单位的已报 text_measure_unit；这里只拦文字列上写的单位 + self.no_unit_on_text(seg.table, seg.measures[label]) + else: self.unit_against(seg.table, seg.measures[label], label, "标签") elif isinstance(seg, DimensionSegment) and seg.locate.by == "section_title" and seg.locate.title: self.unit_against(seg.table, seg.value, seg.locate.title, "分段标题") @@ -938,6 +978,12 @@ class _Checker: return next((s for sj, bj, _gi, _block, s in self.segs if (sj, bj) == (si, bi) and isinstance(s, DimensionSegment) and s.id == seg.locate.segment), None) + def no_unit_on_text(self, table: str, column: str) -> None: + i = self.declared.get(table) + if i is None or not column or column not in self.r.tables[i].units: + return + self.add(("tables", i, "units", column), "unit_on_text", f"列「{column}」按文字导入，不能写单位") + def unit_against(self, table: str, column: str, source: str, what: str) -> None: i = self.declared.get(table) if i is None or not column or (table in self.broken and column not in self.col_names(table)): diff --git a/backend/app/data/recipe_ai.py b/backend/app/data/recipe_ai.py index 5cb8b42f..0f585f08 100644 --- a/backend/app/data/recipe_ai.py +++ b/backend/app/data/recipe_ai.py @@ -121,7 +121,8 @@ RECIPE_DRAFT_SYSTEM = ( "2. 配方语言是封闭的：没有正则，不写单元格坐标；只能用 Schema 里有的字段和取值。\n" "3. measures 的键必须与 labels.expect 里的标签逐字相同、一一对应。\n" "4. const 的 pick 只能从「分段标题候选词」里选。\n" - "5. 占位符只能表示空值（无数据或不适用）。\n" + "5. 占位符只能表示空值（无数据或不适用），不含数字。指标行每格写的都是文字（如时段「18-19」）时，" + "在该 measures 分段的 text_measures 里声明（时段用 hour_range，其余用 text），不要当成占位符。\n" "6. 单位只能取自「单位词表」；标签或表头末尾括号里的单位拆进 tables[].units，列名去掉单位，" "单位必须与括号里写的一致；括号里的单位不在词表里时，列名保留括号、不写单位。\n" "7. 合计、小计行用 role=derived 核对，并用 keep_as 另存成一张表。\n" diff --git a/backend/app/data/recipe_confirm.py b/backend/app/data/recipe_confirm.py index 015abf54..b2e31775 100644 --- a/backend/app/data/recipe_confirm.py +++ b/backend/app/data/recipe_confirm.py @@ -214,7 +214,8 @@ def _column_sources(recipe: Recipe) -> dict[tuple[str, str], str]: def _text_stores(recipe: Recipe) -> dict[tuple[str, str], tuple[str, bool]]: - """(表, 列表的 TEXT 列) → (实际生效的存法, 是否显式写了)。缺省时主键列 canonical、其余 raw。""" + """(表, 列表的 TEXT 列) → (实际生效的存法, 是否显式写了)。缺省时主键列 canonical、其余 raw。 + 交叉表按文字导入的指标也算：按时段解析的存规范写法，按文字的存原文，都是显式写的。""" grains = {t.name: set(t.grain) for t in recipe.tables} out: dict[tuple[str, str], tuple[str, bool]] = {} for sheet in recipe.sheets: @@ -224,6 +225,13 @@ def _text_stores(recipe: Recipe) -> dict[tuple[str, str], tuple[str, bool]]: if c.type == "TEXT": eff = c.store or ("canonical" if c.name in grains.get(block.table, set()) else "raw") out[(block.table, c.name)] = (eff, c.store is not None) + continue + for seg in block.segments: + if isinstance(seg, MeasuresSegment): + for label, tm in seg.text_measures.items(): + if seg.measures.get(label): + out[(seg.table, seg.measures[label])] = ( + "canonical" if tm.parser == "hour_range" else "raw", True) return out @@ -516,6 +524,9 @@ def _signatures(recipe: Recipe, sheet_name: Callable[[SheetRecipe], str]) -> dic if isinstance(seg, DerivedSegment): keep = seg.keep_as.table if seg.keep_as else None out[f"derived:{seg.id}"] = _sig(keep, sorted(match_key(x) for x in seg.labels.expect)) + elif isinstance(seg, MeasuresSegment): + for label, tm in seg.text_measures.items(): + out[f"text_measure:{seg.id}:{label}"] = _sig(tm.parser, seg.measures.get(label)) else: out[f"merged_fill:{block.id}"] = _sig(block.merged_data) out[f"ignored_columns:{block.id}"] = _sig(block.extra_columns, ignore_cols) @@ -1048,6 +1059,12 @@ def _recipe_items(bag: _Bag, recipe: Recipe, ex: dict[str, Any], facts: Any, *, prefix = rows_text(seg_rows.get(seg.id)) or f"分段「{seg.id}」的合计行" keep = f"，原值另存表「{seg.keep_as.table}」" if seg.keep_as else "，原值不另存" bag.add(f"derived:{seg.id}", f"{prefix}改作核对{keep}") + elif isinstance(seg, MeasuresSegment): + # 文字指标：这一行不再按数字核对类型，写错成别的文字也照样入库，所以配置了就要人确认 + for label, tm in seg.text_measures.items(): + how = "按时段导入，存规范写法（如 18-19）" if tm.parser == "hour_range" else "按文字原样导入" + bag.add(f"text_measure:{seg.id}:{label}", + f"「{label}」{how}，不当数字：表「{seg.table}」的列「{seg.measures.get(label, '')}」是文字列") derived_rows: dict[str, set[int]] = {} for d in ex.get("derived") or []: if isinstance(d, dict) and d.get("kind") == "column_sum": diff --git a/backend/app/data/recipe_engine.py b/backend/app/data/recipe_engine.py index 7f0ac181..132c4e41 100644 --- a/backend/app/data/recipe_engine.py +++ b/backend/app/data/recipe_engine.py @@ -67,10 +67,10 @@ from app.data import engine as _data_engine from app.data import tabular, xlsx_cells from app.data.names import canon from app.data.recipe_parsers import ( - TOTAL_WORDS, AxisDate, HourRange, HourTotal, Period, classify_outside, cn_date_range, cn_year_month, + TOTAL_WORDS, UNITS, AxisDate, HourRange, HourTotal, Period, classify_outside, cn_date_range, cn_year_month, filename_period, hour_range, hour_range_total, is_pure_period_cell, looks_numeric_text, match_key, month_day_or_date, - period_residue, text_label, thousands_number, + period_residue, split_unit_suffix, text_label, thousands_number, ) from app.data.recipe_types import ( SUBTOTAL_KEY, AxisOut, CanonEntry, CheckResult, ColumnOut, ContextSpec, CrosstabBlock, DerivedItem, @@ -116,7 +116,7 @@ PROBLEM_CODES: dict[str, str] = { "value_not_integer": "structure", "value_text_number": "structure", "value_blank": "structure", "value_error": "structure", "value_not_number": "structure", "value_not_date": "structure", "value_formula": "structure", "formula_uncached": "structure", "formula_full_calc": "structure", - "pk_duplicate": "structure", + "pk_duplicate": "structure", "value_not_text": "structure", "value_not_hour_range": "structure", } #: 修复按钮的种类（契约 Problem.fix，取值见 FIX_KINDS）。参数 fix_args 的形状见 P3-SPEC 3.2，由报问题的地方随问题 @@ -520,6 +520,9 @@ class _Issue: row: int | None = None #: value_not_number：能当占位符提议的文字（canon 后）→ 格数，最多收 PLACEHOLDER_TEXTS_MAX + 1 种 texts: Counter[str] = field(default_factory=Counter) + #: value_not_number：measures 段里整行都是文字的指标（declare_text_measure 的参数）： + #: {"segment", "label"（期望标签原文）, "cell"（标签格）, "parsers"（能用的解析方式）} + text_rows: list[dict[str, Any]] = field(default_factory=list) #: 合并类问题的说明：code → (原因（{n} 是个数）, 下一步)。下一步写配方面板上的「字段」和「选项」原文 @@ -530,8 +533,13 @@ _ISSUE_TEXT: dict[str, tuple[str, str]] = { "。如需按数字导入，请在配方面板的「文本写的数字」中选择「千分位写法按数字保存」"), "value_blank": ("{n} 个单元格为空", "。如原表确实为空，请在配方面板的「空格」中选择「存为空值」"), "value_error": ("{n} 个单元格是错误值", "。请在 Excel 中修正后重新上传"), + # 下一步按格子的情况另写（_not_number_tail）：整行是文字的提示改「指标类型」，能当占位符的才提示加占位符 "value_not_number": ("{n} 个单元格不是数字", "。如果它表示无数据，请在配方面板中添加占位符"), "value_not_date": ("{n} 个单元格不是日期", "。请检查原表，或在配方面板中把这一列的「类型」改为「文字」"), + "value_not_text": ("{n} 个单元格不是文字", + "。这个指标按文字导入：请检查原表，或在配方面板中把它的「指标类型」改为「数字」"), + "value_not_hour_range": ("{n} 个单元格无法按时段解析", + "。这个指标按时段导入：请检查原表，或在配方面板中把它的「指标类型」改为「文字」"), "value_formula": ("{n} 个数据格是公式", "。数据区的公式通常是没有识别出来的合计；如确为数据，请在配方面板的「公式」中选择「按保存值导入」"), "formula_uncached": ("{n} 个公式格没有保存计算结果", "，文件未经计算保存。请在 Excel 中打开并保存后重新上传"), @@ -548,6 +556,21 @@ _PK_PLACEHOLDER = ("主键列有 {n} 个单元格是占位符", "。占位符按 _PK_NULL = ("有 {n} 处为空值", "。主键不能为空：请检查原表，或在配方面板的「主键」中去掉这一列") +def _not_number_tail(it: _Issue) -> str: + """value_not_number 的下一步。整行写的都是文字的指标，提示改「指标类型」；有能当占位符的文字（不含数字的短符号） + 才提示加占位符：「18-19」这种写法当占位符会整格存成空值，干跑照样通过（静态校验 placeholder_digits 也拦）。""" + hint: list[str] = [] + if it.text_rows: + hint.append(f"{_quoted([x['label'] for x in it.text_rows])}这一行写的都是文字：如果本来就是文字（如时段、名称），" + "请在配方面板中把它的「指标类型」改为「文字」或「时段」") + if it.texts: + hint.append("如果其中有表示无数据的符号，请在配方面板中添加占位符" if hint + else "如果它表示无数据，请在配方面板中添加占位符") + if not hint: + return "。数值区只能是数字或表示无数据的符号（如「·」「—」，不含数字）：请检查原表" + return "。" + "；".join(hint) + + class _Problems: def __init__(self) -> None: self.items: list[tuple[str | None, Problem]] = [] @@ -592,6 +615,12 @@ class _Problems: if sample is not None and (sample in it.texts or len(it.texts) <= PLACEHOLDER_TEXTS_MAX): it.texts[sample] += 1 + def text_row(self, sheet: str, where: str, row: dict[str, Any]) -> None: + """把整行是文字的指标记到这一段的 value_not_number 上（那一行的格转换时已经各自报过）。""" + it = self.issues.get(("value_not_number", sheet, where + "\x00")) + if it is not None and all(x["label"] != row["label"] for x in it.text_rows): + it.text_rows.append(row) + def finish(self, failed_sheets: set[str], args_for: Callable[[_Issue], dict[str, Any] | None] | None = None) -> list[Problem]: """合并类问题出成 Problem；args_for 给有修复按钮的合并类问题（outside_number、value_not_number）算 fix_args。 @@ -602,7 +631,8 @@ class _Problems: ex = f"（如 {'；'.join(it.examples)}）" if it.examples else f"（{_join(shown, 5)}）" model_ex = f"（{_join(shown, 5)}）" args = args_for(it) if args_for is not None and it.code in PROBLEM_FIX else None - self.add(it.code, f"{it.where}：{reason}{ex}{it.tail}", f"{it.where}：{reason}{model_ex}", it.cells, + tail = _not_number_tail(it) if it.code == "value_not_number" else it.tail + self.add(it.code, f"{it.where}：{reason}{ex}{tail}", f"{it.where}：{reason}{model_ex}", it.cells, sheet=it.sheet, fix_args=args) for (code, sheet), n in self._over.items(): where = f"工作表「{sheet}」" if sheet else "本文件" @@ -1482,10 +1512,22 @@ class _Crosstab: eng.headers[(seg.table, seg.measures.get(label, ""))] = _text(self._label(r).value) # type: ignore[union-attr] # 值保留 _FAIL（已报过问题），交给 _Sink 按空值存；没找到的标签（已报 label_missing）也按 _FAIL values: dict[tuple[str, int], Any] = {} + pol = self.block.values for label, r in rows_by_label.items(): excluded = self.exclude_hidden and sh.hidden_row(r) + text = seg.text_measures.get(label) + column = seg.measures.get(label, "") for c in self._axis_cols(): - values[(label, c)] = None if excluded else self._value(r, c, where) + if excluded: + values[(label, c)] = None + elif text is not None: + values[(label, c)] = eng.convert_text(self.grid.get(r, c), text.parser, pol, sh.name, where, r, c, + seg.table, column) + else: + values[(label, c)] = self._value(r, c, where) + if text is None and not excluded and (parsers := self._text_parsers(r, label)): + eng.problems.text_row(sh.name, where, {"segment": seg.id, "label": label, + "cell": sh.coord(r, self.label_c), "parsers": parsers}) if self.dates is None: return for c in self._axis_cols(): @@ -1498,6 +1540,31 @@ class _Crosstab: cells.append((seg.measures.get(label, ""), r, c)) sink.add(row, sheet=sh.name, at=(self.axis_row, c), cells=cells, fails=True) + def _text_parsers(self, r: int, label: str) -> list[str] | None: + """这一行能不能改按文字指标导入（declare_text_measure 的提议）：轴列上除了占位符和空格，全是文字（不是数字、 + 日期、公式、错误值，也不是写成文字的数），至少一格。全都能按时段解析时给两种解析方式，时段在前。标签写着 + 词表里的单位的是数字指标（静态校验 text_measure_unit 也拒），不提议。""" + unit = split_unit_suffix(label)[1] + if unit is not None and unit in UNITS: + return None + ph = self.eng._placeholders(self.block.values) + texts: list[str] = [] + for c in self._axis_cols(): + cell = self.grid.get(r, c) + if cell is None: + continue + if cell.formula is not None or not isinstance(cell.value, str): + return None + t = cell.value.strip() + if not t or canon(t) in ph: + continue + if t in tabular.EXCEL_ERRORS or thousands_number(t) is not None or looks_numeric_text(t): + return None + texts.append(t) + if not texts: + return None + return ["hour_range", "text"] if all(hour_range(t) is not None for t in texts) else ["text"] + def _derive_values(self, derive: dict[str, str], hr: Any) -> list[Any]: ordered = sorted(derive.items(), key=lambda kv: (kv[1] != "start", kv[0])) if not isinstance(hr, (HourRange, HourTotal)): @@ -2441,6 +2508,55 @@ class _Engine: return v if vtype == "REAL" else int(v) return self._number(v, vtype, pol, sheet, where, r, c, pk) + def convert_text(self, cell: GridCell | None, parser: str, pol: Any, sheet: str, where: str, r: int, c: int, + table: str, column: str) -> Any: + """文字指标的一格（MeasuresSegment.text_measures）→ 入库值。空格、公式、占位符照块的取值规则；text 存原文 + （去首尾空白），hour_range 存时段的规范写法。数字、日期不是文字（不替人把 8-9 被 Excel 改成的日期认回去）， + 解析不了的时段也是问题：都按 _FAIL 返回、已记问题。""" + p = self.problems + if cell is None: + if pol.blank == "null": + return None + p.issue("value_blank", sheet, where, _coord(sheet, r, c)) + return _FAIL + v = cell.value + if cell.formula is not None: + v = self._accept_formula(cell, pol, sheet, where, r, c) + if v is _FAIL: + return _FAIL + if not isinstance(v, str): + p.issue("value_not_text", sheet, where, _coord(sheet, r, c), f"{cell_ref(r, c)}「{_shown(v)}」") + return _FAIL + t = v.strip() + ph = self._placeholders(pol) + key = canon(t) + if key in ph: + name = ph[key] + self.ex.placeholders[name] = self.ex.placeholders.get(name, 0) + 1 + return None + if not t: + if pol.blank == "null": + return None + p.issue("value_blank", sheet, where, _coord(sheet, r, c)) + return _FAIL + if t in tabular.EXCEL_ERRORS: + p.issue("value_error", sheet, where, _coord(sheet, r, c), f"{cell_ref(r, c)}「{_shown(v)}」") + return _FAIL + if parser == "text": + return t + hr = hour_range(t) + if hr is None: + p.issue("value_not_hour_range", sheet, where, _coord(sheet, r, c), f"{cell_ref(r, c)}「{_shown(v)}」") + return _FAIL + if hr.canonical != t: + log = (table, column, t, hr.canonical) + hit = self.canon_log.get(log) + if hit is not None: + hit.count += 1 + elif len(self.canon_log) < CANON_MAX: + self.canon_log[log] = CanonEntry(table, column, t, hr.canonical, 1, _coord(sheet, r, c)) + return hr.canonical + def _accept_formula(self, cell: GridCell, pol: Any, sheet: str, where: str, r: int, c: int) -> Any: """数据区的公式格：拒收、无缓存、打开时重算都是问题；否则按保存值导入并计数。""" p = self.problems @@ -2675,7 +2791,11 @@ class _Engine: if it.block is None: return None texts = list(it.texts)[:PLACEHOLDER_TEXTS_MAX] - return {"block": it.block, "texts": texts, "other": it.count - sum(it.texts[t] for t in texts)} + args = {"block": it.block, "texts": texts, "other": it.count - sum(it.texts[t] for t in texts)} + if it.text_rows: + # 只有标签原文、标签格坐标和解析方式，没有数据格的内容 + args["text_rows"] = [dict(x) for x in it.text_rows] + return args if it.code == "outside_number" and it.row is not None: sh = next((x for x in self.sheets if x.name == it.sheet), None) if sh is None: diff --git a/backend/app/data/recipe_fixes.py b/backend/app/data/recipe_fixes.py index 5921ebfb..b8579c70 100644 --- a/backend/app/data/recipe_fixes.py +++ b/backend/app/data/recipe_fixes.py @@ -27,10 +27,12 @@ from pydantic import ValidationError from app.data.names import canon, collide_key from app.data.recipe import apply_patch, recipe_sha256 from app.data.recipe_parsers import ( + UNITS, candidate_words, looks_numeric_text, match_key, month_day_or_date, + split_unit_suffix, ) from app.data.recipe_suggest import _measure_column from app.data.recipe_types import ( @@ -255,6 +257,8 @@ def _key(kind: str, target: dict[str, Any]) -> str: return f"rename_sheet:{t['sheet_id']}" if kind == "declare_subtotals": return f"declare_subtotals:{t['block']}" + if kind == "declare_text_measure": + return f"declare_text_measure:{t['segment']}:{t['label']}" raise KeyError(kind) @@ -308,6 +312,8 @@ def _o_remove_label(recipe: dict, p: FixProposal, opt: FixOption, reason: str | cols = [] for e in removed: col = seg["measures"].get(e) + if e in (seg.get("text_measures") or {}): + ops.append({"op": "remove", "path": ptr(*sp, "text_measures", e)}) if col is None: continue cols.append(col) @@ -967,6 +973,67 @@ def _o_declare_placeholder(recipe: dict, p: FixProposal, opt: FixOption, reason: return ops, [f"块「{block['id']}」把{quoted(p.target['texts'])}声明为{meaning}（存为空值）"] +# ========================================================================== +# ⑪ 按文字导入整行是文字的指标（value_not_number 的另一个出口：「高峰时段」一行写的是「18-19」） +# ========================================================================== + +_TEXT_PARSER_OPTION = { + "hour_range": ("按时段导入（存规范写法，如 18-19）", "以后各期这一行每格都要能按时段解析，否则拒收"), + "text": ("按文字原样导入", "以后各期这一行按原文存，不再要求是数字"), +} + + +def _p_declare_text_measure(recipe: dict, prob: Any, args: dict, anchor: FixAnchor) -> list[FixProposal]: + """执行器在 fix_args.text_rows 里给出整行都是文字的指标（标签原文、标签格、能用的解析方式）。每个指标一条提议； + 标签写着词表里的单位的不提议（静态校验 text_measure_unit 会拒），已经按文字导入的不再提议。""" + out: list[FixProposal] = [] + for row in args.get("text_rows") or []: + found = find_segment(recipe, row.get("segment")) + label = row.get("label") + if found is None or not isinstance(label, str): + continue + seg = found[4] + if (seg.get("role") != "measures" or label not in (seg.get("labels") or {}).get("expect", []) + or label in (seg.get("text_measures") or {}) or not seg.get("measures", {}).get(label)): + continue + unit = split_unit_suffix(label)[1] + if unit is not None and unit in UNITS: + continue + parsers = [p for p in row.get("parsers") or [] if p in _TEXT_PARSER_OPTION] + if not parsers: + continue + target = {"segment": seg["id"], "label": label} + col = seg["measures"][label] + cell = row.get("cell") + out.append(_proposal( + "declare_text_measure", _key("declare_text_measure", target), code=_get(prob, "code"), + title=f"「{label}」这一行写的都是文字，不是数字", + cells=[cell] if isinstance(cell, str) else [], target=target, + options=[FixOption(p, _TEXT_PARSER_OPTION[p][0], + detail=f"表「{seg.get('table')}」的列「{col}」改成文字列；{_TEXT_PARSER_OPTION[p][1]}", + breaking=True) for p in parsers], + anchor=anchor)) + return out + + +def _o_declare_text_measure(recipe: dict, p: FixProposal, opt: FixOption, reason: str | None) -> tuple[list, list[str]]: + found = find_segment(recipe, p.target.get("segment")) + label = p.target.get("label") + if found is None or found[4].get("role") != "measures" or label not in found[4].get("measures", {}): + raise _Stale + si, bi, gi, _block, seg = found + sp = ("sheets", si, "blocks", bi, "segments", gi) + ops: list[dict[str, Any]] = [] + if "text_measures" not in seg: + ops.append({"op": "add", "path": ptr(*sp, "text_measures"), "value": {}}) + ops.append({"op": "add", "path": ptr(*sp, "text_measures", label), "value": {"parser": opt.value}}) + col = seg["measures"][label] + ti = table_index(recipe, seg.get("table")) + if ti is not None and col in (recipe["tables"][ti].get("units") or {}): + ops.append({"op": "remove", "path": ptr("tables", ti, "units", col)}) + return ops, [f"分段「{seg['id']}」的「{label}」{opt.label}：表「{seg.get('table')}」的列「{col}」是文字列"] + + # ========================================================================== # ⑧ 声明隐藏行列怎么处理 # ========================================================================== @@ -1062,7 +1129,12 @@ _OPS: dict[str, Callable[[dict, FixProposal, FixOption, str | None], tuple[list, "remove_label": _o_remove_label, "add_label": _o_add_label, "edit_members": _o_edit_members, "rename_title": _o_rename_title, "declare_total": _o_declare_total, "ignore_cells": _o_ignore_cells, "declare_placeholder": _o_declare_placeholder, "declare_hidden": _o_declare_hidden, "rename_sheet": _o_rename_sheet, - "declare_subtotals": _o_declare_subtotals, + "declare_subtotals": _o_declare_subtotals, "declare_text_measure": _o_declare_text_measure, +} +#: 同一条问题的另一种出口（问题的 fix 字段只写一种，界面按 fix_ids 放按钮，所以能挂几个提议）：value_not_number 的 +#: 格子是整行文字时，除了声明占位符，还可以把这个指标改按文字导入 +_EXTRA_BUILDERS: dict[str, Callable[[dict, Any, dict, FixAnchor], list[FixProposal]]] = { + "value_not_number": _p_declare_text_measure, } @@ -1096,7 +1168,15 @@ def propose_fixes(recipe: dict[str, Any], problems: list[dict[str, Any]], recipe try: push(builder(recipe, prob, args, FixAnchor("problem", i))) except (KeyError, TypeError, ValueError, IndexError, AttributeError): - continue # fix_args 的形状不对（执行器的契约问题）：不出提议，不拖垮 staging_out + pass # fix_args 的形状不对（执行器的契约问题）：不出提议，不拖垮 staging_out + extra = _EXTRA_BUILDERS.get(_get(prob, "code") or "") + if extra is None: + continue + try: + for p in extra(recipe, prob, args, FixAnchor("problem", i)): + push(p) + except (KeyError, TypeError, ValueError, IndexError, AttributeError): + continue fl = _facts(facts) for i, rp in enumerate(recipe_problems or []): try: @@ -1125,6 +1205,8 @@ def _edit_title(p: FixProposal, opt: FixOption) -> str: return opt.label if p.kind == "declare_subtotals": return f"列表「{t.get('block')}」中间「{t.get('pick')}」开头的行{opt.label}" + if p.kind == "declare_text_measure": + return f"分段「{t.get('segment')}」的「{t.get('label')}」{opt.label}" return p.title diff --git a/backend/app/data/recipe_notes.py b/backend/app/data/recipe_notes.py index 48c8da20..885cd94a 100644 --- a/backend/app/data/recipe_notes.py +++ b/backend/app/data/recipe_notes.py @@ -696,8 +696,9 @@ def _period_column_key(col: ColumnOut) -> str | None: def _null_eligible(ctx: _Ctx, w: _Writers, table: str, col: ColumnOut) -> bool: - """这一列要不要考虑空值说明：值列；主键不会是空值；表内合计表的空值已由表说明（未能核对）交代。""" - value_col = col.role in ("measure", "value") and col.type in ("INTEGER", "REAL") + """这一列要不要考虑空值说明：值列（交叉表按文字导入的指标也是，它的占位符、空格同样存成空值）；主键不会是空值； + 表内合计表的空值已由表说明（未能核对）交代。""" + value_col = col.role == "measure" or (col.role == "value" and col.type in ("INTEGER", "REAL")) is_total = bool(w.kept_from or w.list_totals or w.list_subtotals) return value_col and not is_total and col.name not in set(ctx.grains.get(table, [])) diff --git a/backend/app/data/recipe_suggest.py b/backend/app/data/recipe_suggest.py index 9a176511..a3eb9037 100644 --- a/backend/app/data/recipe_suggest.py +++ b/backend/app/data/recipe_suggest.py @@ -359,6 +359,7 @@ class _Seg: title: str | None = None title_row: int | None = None parser: str | None = None # dimension：hour_range / text + text: dict[str, str] = field(default_factory=dict) # measures：整行是文字的标签原文 → 解析方式 # 以下由命名阶段填 id: str = "" table: str = "" @@ -417,6 +418,41 @@ def _sums_above(cell: GridCell | None, r: int) -> bool: return bool(m) and m.group(1) == m.group(3) and int(m.group(4)) == r - 1 +def _text_row(grid: Grid, r: int, c1: int, c2: int, label: str) -> str | None: + """指标行是不是整行文字（「高峰时段」每格写着「18-19」）：是的话返回解析方式（全都能按时段解析的 hour_range， + 否则 text），不是返回 None。轴列上除了占位符候选和空格全是文字，至少一格；有一格是数、公式、错误值、写成文字的 + 数就不是。标签末尾写着词表里的单位（「时段人数（人次）」）的是数字指标，不算。""" + unit = P.split_unit_suffix(label)[1] + if unit is not None and P.unit_known(unit): + return None + texts: list[str] = [] + for c in range(c1, c2 + 1): + cell = grid.get(r, c) + kind = _cell_kind(cell) + if kind in ("blank", "ph"): + continue + if kind != "text": + return None + t = cell.value.strip() # type: ignore[union-attr] + if P.looks_numeric_text(t) or P.thousands_number(t) is not None: + return None + texts.append(t) + if not texts: + return None + return "hour_range" if all(P.hour_range(t) is not None for t in texts) else "text" + + +def _value_failure(sheet: str, r: int, label: str, why: str, cells: list[tuple[int, str]]) -> _Failure: + """数值区一行里同一种不合格的格合成一条（一行 30 格不再报 30 条）。给模型的那条只有坐标，没有格子里的字。""" + if len(cells) == 1: + c, shown = cells[0] + return _Failure(f"工作表「{sheet}」：{_a1(r, c)} 的「{shown}」{why}", f"工作表「{sheet}」：{_a1(r, c)} {why}") + examples = "、".join(f"{_a1(r, c)}「{shown}」" for c, shown in cells[:3]) + where = _a1_list([(r, c) for c, _ in cells], 3) + return _Failure(f"工作表「{sheet}」：第 {r} 行「{label}」有 {len(cells)} 个单元格{why}（如 {examples}）", + f"工作表「{sheet}」：第 {r} 行有 {len(cells)} 个单元格{why}（{where}）") + + def _analyze_crosstab(grid: Grid, axis_row: int, date_cols: list[int]) -> _Cross: sheet = grid.sheet c1, c2 = min(date_cols), max(date_cols) @@ -544,8 +580,13 @@ def _analyze_crosstab(grid: Grid, axis_row: int, date_cols: list[int]) -> _Cross # ---- 数值区：类型、占位符、空格、公式（P2-SPEC 6.1 第 9 条） ints = True for seg in x.segments: - for r in seg.rows: + for r, label in zip(seg.rows, seg.labels): + # 指标行整行是文字（「高峰时段」每格写着「18-19」）：按文字指标起草，不当数字报错；要人在问题里确认 + text = _text_row(grid, r, c1, c2, label) if seg.role == "measures" else None + if text is not None: + seg.text[label] = text row_vals: list[int | float | None] = [] + bad: dict[str, list[tuple[int, str]]] = {} for c in range(c1, c2 + 1): cell = grid.get(r, c) kind = _cell_kind(cell) @@ -565,13 +606,16 @@ def _analyze_crosstab(grid: Grid, axis_row: int, date_cols: list[int]) -> _Cross entry[1] += 1 entry[2].append((r, c)) x.nonnumeric[cell.value.strip()] = x.nonnumeric.get(cell.value.strip(), 0) + 1 + elif kind == "text" and text is not None: + continue elif kind in ("text", "error", "other"): shown = _shown(cell.value) if isinstance(cell.value, str): x.nonnumeric[shown] = x.nonnumeric.get(shown, 0) + 1 - why = "是错误值" if kind == "error" else "不是数字，也不像表示无数据的符号" - fail(_Failure(f"工作表「{sheet}」：{_a1(r, c)} 的「{shown}」{why}", - f"工作表「{sheet}」：{_a1(r, c)} {why}")) + bad.setdefault("是错误值" if kind == "error" else "不是数字，也不像表示无数据的符号", + []).append((c, shown)) + for why, cells in bad.items(): + fail(_value_failure(sheet, r, label, why, cells)) x.values[r] = row_vals x.value_type = "INTEGER" if ints else "REAL" @@ -1436,8 +1480,11 @@ def _prefer_prefix(period: list[tuple[int, int, str]]) -> str | None: def _segment_dict(s: _Seg) -> dict[str, Any]: if s.role == "measures": - return {"id": s.id, "role": "measures", "table": s.table, "locate": {"by": "labels"}, - "labels": {"expect": list(s.labels)}, "measures": dict(s.columns)} + out = {"id": s.id, "role": "measures", "table": s.table, "locate": {"by": "labels"}, + "labels": {"expect": list(s.labels)}, "measures": dict(s.columns)} + if s.text: + out["text_measures"] = {label: {"parser": p} for label, p in s.text.items()} + return out if s.role == "dimension": out: dict[str, Any] = { "id": s.id, "role": "dimension", "table": s.table, @@ -2427,6 +2474,18 @@ def _unit_cards(labels: list[tuple[str, str]], sheet_id: str) -> list[Card]: return out +def _label_row(grid: Grid | None, loc: _CrossLoc | None, seg_id: str, label: str) -> int | None: + """分段里标签是 label 的那一行（按 match_key 比）；定位不到时 None。""" + if grid is None or loc is None: + return None + key = P.match_key(label) + for r in loc.seg_rows.get(seg_id, []): + cell = grid.get(r, loc.label_col) + if cell is not None and isinstance(cell.value, str) and P.match_key(cell.value) == key: + return r + return None + + def _crosstab_cards(si: int, bi: int, sheet: Any, block: CrosstabBlock, loc: _CrossLoc | None, grid: Grid | None, name: str, recipe: dict[str, Any], parsed: Recipe, sheet_facts: list[Fact], relation_question: Callable[[Fact], Question], ext: Extraction | None, @@ -2539,6 +2598,32 @@ def _crosstab_cards(si: int, bi: int, sheet: Any, block: CrosstabBlock, loc: _Cr cards.extend(ph_cards) questions.extend(reversed(ph_qs)) + # 6b 按文字导入的指标：要人确认。「不是」去掉这一条，这一行回到按块的数值类型导入，试运行会拒收 + for gi, s in enumerate(block.segments): + if not isinstance(s, MeasuresSegment): + continue + for label, tm in s.text_measures.items(): + r = _label_row(grid, loc, s.id, label) + spots: list[tuple[int, int]] = [] + if r is not None and loc is not None and grid is not None: + spots = [(r, loc.label_col)] + [(r, cc) for cc in range(loc.c1, loc.c2 + 1) + if grid.get(r, cc) is not None] + where = f"第 {r} 行" if r is not None else f"分段「{s.id}」中" + how = "时段" if tm.parser == "hour_range" else "文字" + col = s.measures.get(label, "") + qid = _qid(f"q_text_measure:{s.id}:{label}", block.id, used) + reason = (f"这一行每格写的都是{'时段（如「18-19」）' if tm.parser == 'hour_range' else '文字'}，不是数字：" + f"确认后列「{col}」存成文字列" + f"{'（时段存规范写法，「18–19」存成「18-19」）' if tm.parser == 'hour_range' else '（存原文）'}，" + "不参与数值核对；如果它应该是数字，这份文件无法按当前配方导入") + cards.append(Card(f"text_measure:{s.id}:{label}", f"{where}「{label}」按{how}导入", reason, + _coords(name, spots), qid)) + path = ["sheets", si, "blocks", bi, "segments", gi, "text_measures", label] + questions.append(Question(qid, f"「{label}」这一行按{how}导入吗", + [{"value": "text", "label": f"是，按{how}导入", "needs_reason": False}, + {"value": "reject", "label": "不是，不导入", "needs_reason": False}], + None, {"text": [], "reject": [{"op": "remove", "path": _ptr(*path)}]})) + # 7 合计段 for s in block.segments: if not isinstance(s, DerivedSegment): diff --git a/backend/app/data/recipe_types.py b/backend/app/data/recipe_types.py index 215cd7f3..357b187b 100644 --- a/backend/app/data/recipe_types.py +++ b/backend/app/data/recipe_types.py @@ -154,6 +154,14 @@ class DimSpec(_M): derive: dict[str, Literal["start", "end"]] = Field(default_factory=dict) +class TextMeasure(_M): + """按文字导入的指标：这一行每格写的是文字（如「高峰时段」写着「18-19」），落成文字列。空格、公式、占位符照块的 + 取值规则；块的数值类型、千分位规则对它不适用。""" + + #: text：存原文（去首尾空白）；hour_range：按时段解析，存规范写法（「18–19」存成「18-19」），解析不了的格是问题 + parser: Literal["text", "hour_range"] = "text" + + class MeasuresSegment(_M): """每一行是一个指标：落成宽表，一个指标一列。""" @@ -166,6 +174,8 @@ class MeasuresSegment(_M): #: derive_tables 按原文查，查不到报 measures_keys_mismatch，绝不产出空列名）。文件里标签的写法变化由 #: 执行器按 match_key 认，与这里无关 measures: dict[str, str] + #: 按文字导入的指标：标签原文（与 measures 的键逐字相同）→ 解析方式。不在这里的指标按块的数值类型导入 + text_measures: dict[str, TextMeasure] = Field(default_factory=dict) class DimensionSegment(_M): @@ -479,7 +489,7 @@ class Problem: #: 修复按钮的种类（P3-SPEC 3.2 ①–⑨；期 5 加 declare_subtotals：列表中间像小计的行改作小计核对）。Problem.fix、 #: FixProposal.kind、确认项 fix:<目标键> 的 kind 都取自这里 FIX_KINDS = ("remove_label", "add_label", "edit_members", "rename_title", "declare_total", "ignore_cells", - "declare_placeholder", "declare_hidden", "rename_sheet", "declare_subtotals") + "declare_placeholder", "declare_hidden", "rename_sheet", "declare_subtotals", "declare_text_measure") @dataclass @@ -1135,7 +1145,8 @@ def derive_tables(recipe: Recipe) -> tuple[dict[str, list[ColumnOut]], list[Reci problems.append(RecipeProblem(f"{path}/measures", "measures_keys_mismatch", f"分段「{seg.id}」：" + "；".join(parts))) cols = [ColumnOut(axis_name, "TEXT", role="axis")] - cols += [ColumnOut(seg.measures.get(label, ""), vtype, header=label, role="measure") + cols += [ColumnOut(seg.measures.get(label, ""), "TEXT" if label in seg.text_measures else vtype, + header=label, role="measure") for label in seg.labels.expect] put(seg.table, cols, path) elif isinstance(seg, DimensionSegment): diff --git a/backend/tests/test_recipe_contract.py b/backend/tests/test_recipe_contract.py index 03827e7e..fd729e91 100644 --- a/backend/tests/test_recipe_contract.py +++ b/backend/tests/test_recipe_contract.py @@ -488,9 +488,9 @@ def test_closed_value_sets(): "header", "row_label", "section_title", "total_word", "axis", "after_title", "outside_text") assert _literal(T.EditResult, "kind") == ("fix", "selection") assert _literal(T.ChangeClass, "change") == ("same", "compatible", "retire", "semantic") - assert T.FIX_KINDS[-1] == "declare_subtotals" # 期 5 加在末尾 - assert T.FIX_KINDS[:-1] == ("remove_label", "add_label", "edit_members", "rename_title", "declare_total", - "ignore_cells", "declare_placeholder", "declare_hidden", "rename_sheet") + assert T.FIX_KINDS[-2:] == ("declare_subtotals", "declare_text_measure") # 期 5、文字指标依次加在末尾 + assert T.FIX_KINDS[:-2] == ("remove_label", "add_label", "edit_members", "rename_title", "declare_total", + "ignore_cells", "declare_placeholder", "declare_hidden", "rename_sheet") assert T.SELECTION_AS == ("list", "crosstab", "segment", "derived", "section_title", "ignore_rows", "ignore_columns", "ignore_outside") assert T.ACCUMULATE_ACTIONS == ("replace", "first", "append", "replace_period", "restart", "rejected") diff --git a/backend/tests/test_recipe_engine_messages.py b/backend/tests/test_recipe_engine_messages.py index c2013ab1..e948cef1 100644 --- a/backend/tests/test_recipe_engine_messages.py +++ b/backend/tests/test_recipe_engine_messages.py @@ -362,6 +362,20 @@ def c_value_not_number(mp): return _run(_flow(_set(C11=f"约{N}")), FLOW), [f"约{N}", N] +def _text_measure(parser: str) -> dict[str, Any]: + data = recipe_dict() + data["sheets"][0]["blocks"][0]["segments"][0]["text_measures"] = {"分区甲（人次）": {"parser": parser}} + return data + + +def c_value_not_text(mp): + return _run(_flow(_set(C6=N)), _text_measure("text")), [N] + + +def c_value_not_hour_range(mp): + return _run(_flow(_set(C6=f"约{N}", D6="18-19", E6="19-20")), _text_measure("hour_range")), [f"约{N}", N] + + def c_value_not_date(mp): return _run(sales_book(edit=lambda ws: ws.cell(4, 3, f"{N}日")), LIST), [f"{N}日", N] @@ -438,7 +452,10 @@ NEXT_STEPS = { "value_text_number": "在配方面板的「文本写的数字」中选择「千分位写法按数字保存」", "value_blank": "在配方面板的「空格」中选择「存为空值」", "value_formula": "在配方面板的「公式」中选择「按保存值导入」", - "value_not_number": "在配方面板中添加占位符", + # 「约987654」含数字、不是整行文字：既不提示加占位符，也不提示改指标类型（另见 test_recipe_text_measures） + "value_not_number": "数值区只能是数字或表示无数据的符号", + "value_not_text": "在配方面板中把它的「指标类型」改为「数字」", + "value_not_hour_range": "在配方面板中把它的「指标类型」改为「文字」", "value_not_integer": "在配方面板中把「类型」改为「小数」", "value_not_date": "在配方面板中把这一列的「类型」改为「文字」", "sheet_unexpected": "在配方面板的「配方没有列出的其他工作表」中选择「记入回执，需确认」", diff --git a/backend/tests/test_recipe_text_measures.py b/backend/tests/test_recipe_text_measures.py new file mode 100644 index 00000000..01fcf21b --- /dev/null +++ b/backend/tests/test_recipe_text_measures.py @@ -0,0 +1,395 @@ +"""交叉表里按文字导入的指标（MeasuresSegment.text_measures）：契约、静态校验、执行、规则起草、修复按钮、确认项、说明。 + +起因：一份客流导出的顶部指标块里夹着「高峰时段」一行，每格写的是当天客流最多的时段（「18-19」），不是数字。 +以前起草逐格报「不是数字」（一行 30 条），提示去加占位符；照做的话「18-19」整格静默存成空值、干跑照样通过； +从分段里去掉这一行再「忽略这些行」又不生效（按标签定位的分段认领连续的数据行）。 + +夹具在这里用 openpyxl 现造，名字全是假名（分区甲 / 分区乙）：B2 统计期、B3 标题、第 4 行日期、第 5–9 行无标题的 +指标块（全日、分区甲、分区乙、高峰时段、时段人数）、空行、日间和夜间两段时段客流，没有合计行。第 4 天的时段全是「·」， +那天的高峰时段、时段人数也是「·」；第 2 天的高峰时段用 en dash 写（「18–19」）。 +""" +from __future__ import annotations + +import copy +import datetime as dt +import io +import random +import sqlite3 +from typing import Any, Callable + +import pytest +from openpyxl import Workbook +from openpyxl.utils import get_column_letter + +from app.data import recipe_engine, xlsx_cells, xlsx_scan +from app.data import recipe_fixes as F +from app.data import recipe_suggest as S +from app.data.recipe import apply_patch, canonical_recipe, validate_recipe +from app.data.recipe_confirm import ConfirmContext, confirm_items, table_changes +from app.data.recipe_types import Recipe, derive_tables + +SHEET = "客流汇总" +START = dt.date(2026, 7, 1) +DAYS = 8 # 至少 tabular.DATE_HEADER_MIN（7）格日期才按交叉表起草 +DAY_HOURS = (8, 9, 10, 11) +NIGHT_HOURS = (18, 19, 20) +GAP = 3 # 第 4 天：时段全是「·」 +PEAK, PEAK_N = "高峰时段", "时段人数（人次）" + + +def peak_book(edit: Callable[[Any], None] | None = None) -> tuple[bytes, str]: + rnd = random.Random(7) + wb = Workbook() + ws = wb.active + ws.title = SHEET + end = START + dt.timedelta(days=DAYS - 1) + last = get_column_letter(2 + DAYS) + ws["B2"] = f"统计时间范围：{START.year}年{START.month}月{START.day}日至{end.year}年{end.month}月{end.day}日" + ws.merge_cells(f"B2:{last}2") + ws["B3"] = "某地区客流汇总表" + ws.merge_cells(f"B3:{last}3") + for j in range(DAYS): + d = START + dt.timedelta(days=j) + ws.cell(4, 3 + j, f"{d.month}月{d.day}日") + hours: dict[int, list[Any]] = {h: [rnd.randint(100, 900) for _ in range(DAYS)] for h in DAY_HOURS + NIGHT_HOURS} + zone_a = [rnd.randint(3000, 9000) for _ in range(DAYS)] + zone_b = [rnd.randint(3000, 9000) for _ in range(DAYS)] + peak: list[Any] = [] + peak_n: list[Any] = [] + for j in range(DAYS): + if j == GAP: + for h in hours: + hours[h][j] = "·" + peak.append("·") + peak_n.append("·") + continue + h = max(hours, key=lambda k: hours[k][j]) + peak.append(f"{h}–{h + 1}" if j == 1 else f"{h}-{h + 1}") + peak_n.append(hours[h][j]) + rows: list[tuple[str, list[Any]]] = [ + ("全日客流（人次）", [a + b for a, b in zip(zone_a, zone_b)]), ("分区甲（人次）", zone_a), + ("分区乙（人次）", zone_b), (PEAK, peak), (PEAK_N, peak_n)] + r = 5 + for label, values in rows: + ws.cell(r, 2, label) + for j, v in enumerate(values): + ws.cell(r, 3 + j, v) + r += 1 + ws.merge_cells(f"B{r}:{last}{r}") # 第 10 行空行 + r += 1 + for title, hs in (("日间时段客流（人次）", DAY_HOURS), ("夜间时段客流（人次）", NIGHT_HOURS)): + ws.cell(r, 2, title) + ws.merge_cells(f"B{r}:{last}{r}") + r += 1 + for h in hs: + ws.cell(r, 2, f"{h}-{h + 1}") + for j, v in enumerate(hours[h]): + ws.cell(r, 3 + j, v) + r += 1 + if edit is not None: + edit(ws) + bio = io.BytesIO() + wb.save(bio) + return bio.getvalue(), f"月报导出_{START.isoformat()}_{end.isoformat()}.xlsx" + + +class Book: + def __init__(self, edit: Callable[[Any], None] | None = None): + self.raw, self.filename = peak_book(edit) + self.scan = xlsx_scan.scan(self.raw) + self.grids = {s.name: xlsx_cells.read_grid(self.raw, self.scan, s.name, max_rows=S.DRAFT_MAX_ROWS) + for s in self.scan.sheets if s.state == "visible" and s.bounds is not None and s.nonempty} + + def run(self, recipe: dict[str, Any] | Recipe, db: str | None = None): + parsed = recipe if isinstance(recipe, Recipe) else Recipe.model_validate(recipe) + return recipe_engine.execute(parsed, self.raw, self.filename, self.scan, db) + + def draft(self): + return S.draft(self.scan, self.grids, self.filename, + validate=lambda r, f, o: validate_recipe(r, facts=f, origin=o), dry_run=self.run) + + +@pytest.fixture(scope="module") +def drafted(): + book = Book() + d = book.draft() + return book, d + + +def measures_seg(recipe: dict[str, Any]) -> dict[str, Any]: + return next(s for s in recipe["sheets"][0]["blocks"][0]["segments"] if s["role"] == "measures") + + +def full(recipe: dict[str, Any]) -> dict[str, Any]: + return Recipe.model_validate(recipe).model_dump(mode="json") + + +def codes(problems: list[Any]) -> list[str]: + return [p.code for p in problems] + + +# ========================================================================== +# 规则起草 +# ========================================================================== + + +def test_rule_draft_turns_the_text_row_into_a_text_measure(drafted): + _book, d = drafted + assert d.complete, d.failures + assert d.failures == [] + seg = measures_seg(d.recipe) + assert seg["text_measures"] == {PEAK: {"parser": "hour_range"}} + assert PEAK in seg["labels"]["expect"] and PEAK_N in seg["labels"]["expect"] + # 文字列不写单位；「时段人数（人次）」是数字指标，单位照常拆 + units = next(t["units"] for t in d.recipe["tables"] if t["name"] == seg["table"]) + assert PEAK not in units and units.get("时段人数") == "人次" + + +def test_rule_draft_asks_to_confirm_the_text_measure(drafted): + _book, d = drafted + seg = measures_seg(d.recipe) + card = next(c for c in d.cards if c.id == f"text_measure:{seg['id']}:{PEAK}") + q = next(q for q in d.questions if q.id == card.question) + assert card.title == f"第 8 行「{PEAK}」按时段导入" + assert f"{SHEET}!B8" in card.cells and f"{SHEET}!C8" in card.cells + assert [o["value"] for o in q.options] == ["text", "reject"] + assert q.effects["text"] == [] + # 答「不是」：去掉这一条，这一行回到按数字导入，干跑拒收（不会静默存成空值） + rejected = apply_patch(full(d.recipe), q.effects["reject"]) + assert PEAK not in measures_seg(rejected)["text_measures"] + book = Book() + ex = book.run(rejected) + assert not ex.ok and "value_not_number" in codes(ex.problems) + + +def test_rule_draft_merges_bad_cells_of_one_row_into_one_failure(): + # 时段人数一行混进两格文字：标签写着单位，是数字指标，照旧起草失败，但一行只报一条，给模型的那条没有格子里的字 + d = Book(lambda ws: (ws.cell(9, 3, "约3000"), ws.cell(9, 4, "约4000"))).draft() + assert not d.complete + lines = [f for f in d.failures if f.startswith(f"工作表「{SHEET}」：") and ("第 9 行" in f or "C9" in f)] + assert lines == [f"工作表「{SHEET}」：第 9 行「{PEAK_N}」有 2 个单元格不是数字，也不像表示无数据的符号" + "（如 C9「约3000」、D9「约4000」）"] + assert not any("约3000" in f for f in d.failures_for_model) + # 干跑另有一条按分段合并的，下一步不再叫人把「约3000」加成占位符 + dry = next(f for f in d.failures if "的分段" in f and "C9" in f) + assert "添加占位符" not in dry and "请检查原表" in dry + assert PEAK not in str(d.failures) # 高峰时段照样按文字指标起草，不报错 + single = Book(lambda ws: ws.cell(9, 3, "约3000")).draft() + assert f"工作表「{SHEET}」：C9 的「约3000」不是数字，也不像表示无数据的符号" in single.failures + + +def test_text_measure_lands_as_a_text_column(drafted, tmp_path): + book, d = drafted + db = str(tmp_path / "t.db") + ex = book.run(d.recipe, db) + assert ex.ok, [(p.code, p.message) for p in ex.problems] + seg = measures_seg(d.recipe) + table = next(t for t in ex.tables if t.name == seg["table"]) + types = {c.name: c.type for c in table.columns} + assert types["高峰时段"] == "TEXT" and types["时段人数"] == "INTEGER" and types["全日客流"] == "INTEGER" + con = sqlite3.connect(db) + try: + decl = {row[1]: row[2] for row in con.execute(f'PRAGMA table_info("{seg["table"]}")')} + rows = con.execute(f'SELECT "日期", "高峰时段", "时段人数" FROM "{seg["table"]}" ORDER BY "日期"').fetchall() + hour_rows = con.execute('SELECT COUNT(*) FROM "时段客流"').fetchone()[0] + finally: + con.close() + assert decl["高峰时段"] == "TEXT" + assert len(rows) == DAYS and hour_rows == DAYS * (len(DAY_HOURS) + len(NIGHT_HOURS)) + # en dash 写的存规范写法；整天没有时段数的那天两列都是空值 + assert rows[1][1] is not None and "–" not in rows[1][1] and rows[1][1].count("-") == 1 + assert rows[GAP][1:] == (None, None) + assert all(isinstance(r[2], int) for i, r in enumerate(rows) if i != GAP) + # 高峰时段能和时段客流按规范写法对上 + con = sqlite3.connect(db) + try: + hit = con.execute(f'SELECT COUNT(*) FROM "{seg["table"]}" d JOIN "时段客流" h ON h."日期" = d."日期" ' + 'AND h."时段" = d."高峰时段" AND h."客流" = d."时段人数"').fetchone()[0] + finally: + con.close() + assert hit == DAYS - 1 + assert any(c.column == "高峰时段" and "–" in c.raw for c in ex.canonicalized) + + +# ========================================================================== +# 执行器：文字指标的取值、报错 +# ========================================================================== + + +def test_number_in_a_text_measure_is_value_not_text(drafted): + _, d = drafted + book = Book(lambda ws: ws.cell(8, 3, 18)) + ex = book.run(d.recipe) + p = next(p for p in ex.problems if p.code == "value_not_text") + assert p.category == "structure" and f"{SHEET}!C8" in p.cells + assert "「指标类型」改为「数字」" in p.message + assert "18" not in p.model_message.replace(SHEET, "").replace("C8", "") + + +def test_unparsable_hour_range_and_plain_text_parser(drafted): + _, d = drafted + book = Book(lambda ws: ws.cell(8, 3, "傍晚")) + ex = book.run(d.recipe) + p = next(p for p in ex.problems if p.code == "value_not_hour_range") + assert "「指标类型」改为「文字」" in p.message and "傍晚" not in p.model_message + as_text = full(d.recipe) + measures_seg(as_text)["text_measures"][PEAK] = {"parser": "text"} + ex2 = book.run(as_text) + assert ex2.ok, [(p.code, p.message) for p in ex2.problems] + + +# ========================================================================== +# 没声明成文字指标时：报错的下一步和修复按钮 +# ========================================================================== + + +def _without_text_measure(d: Any) -> dict[str, Any]: + r = full(d.recipe) + measures_seg(r)["text_measures"] = {} + return r + + +def test_not_number_on_a_text_row_points_at_measure_kind_not_placeholder(drafted): + book, d = drafted + ex = book.run(_without_text_measure(d)) + p = next(p for p in ex.problems if p.code == "value_not_number") + assert f"「{PEAK}」这一行写的都是文字" in p.message + assert "「指标类型」改为「文字」或「时段」" in p.message + assert "添加占位符" not in p.message + seg = measures_seg(d.recipe) + assert p.fix_args["texts"] == [] + assert p.fix_args["text_rows"] == [{"segment": seg["id"], "label": PEAK, "cell": f"{SHEET}!B8", + "parsers": ["hour_range", "text"]}] + + +def test_not_number_tail_depends_on_what_the_cells_are(drafted): + _, d = drafted + # 不含数字的短符号：照旧提示加占位符 + ex = Book(lambda ws: ws.cell(5, 3, "—")).run(d.recipe) + p = next(p for p in ex.problems if p.code == "value_not_number") + assert p.message.endswith("如果它表示无数据，请在配方面板中添加占位符") + # 含数字、又不是整行文字：不再提示加占位符 + ex = Book(lambda ws: ws.cell(5, 3, "约3000")).run(d.recipe) + p = next(p for p in ex.problems if p.code == "value_not_number") + assert "占位符" not in p.message.split("。", 1)[1].replace("表示无数据的符号", "") + assert "请检查原表" in p.message and "text_rows" not in p.fix_args + # 标签写着单位的指标整行是文字：是数字指标，不提示改指标类型 + ex = Book(lambda ws: [ws.cell(9, c, "18-19") for c in range(3, 3 + DAYS)]).run(d.recipe) + p = next(p for p in ex.problems if p.code == "value_not_number") + assert "指标类型" not in p.message and "text_rows" not in p.fix_args + + +def test_fix_button_declares_the_text_measure(drafted): + book, d = drafted + work = _without_text_measure(d) + ex = book.run(work) + problems = [p.__dict__ for p in ex.problems] + props = F.propose_fixes(work, problems, [], d.facts) + p = next(x for x in props if x.kind == "declare_text_measure") + assert [o.value for o in p.options] == ["hour_range", "text"] + assert all(o.breaking for o in p.options) + assert p.anchor.index == next(i for i, x in enumerate(problems) if x["code"] == "value_not_number") + res = F.fix_ops(work, p, "hour_range", None) + assert res.ok and "按时段导入" in res.title + new = apply_patch(work, res.ops) + assert measures_seg(new)["text_measures"] == {PEAK: {"parser": "hour_range"}} + parsed, probs = validate_recipe(new, facts=d.facts, origin="manual") + assert probs == [] and parsed is not None + assert book.run(parsed).ok + # 已经按文字导入的不再提议 + assert not [x for x in F.propose_fixes(new, problems, [], d.facts) if x.kind == "declare_text_measure"] + + +def test_remove_label_also_drops_the_text_measure(drafted): + _, d = drafted + work = full(d.recipe) + seg = measures_seg(work) + prob = {"code": "label_missing", "fix": "remove_label", "cells": [], + "fix_args": {"segment": seg["id"], "labels": [PEAK], "role": "measures"}} + p = next(x for x in F.propose_fixes(work, [prob], [], d.facts) if x.kind == "remove_label") + new = apply_patch(work, F.fix_ops(work, p, p.options[0].value, None).ops) + assert PEAK not in measures_seg(new)["text_measures"] + _parsed, probs = validate_recipe(new, facts=d.facts, origin="manual") + assert "text_measure_unknown" not in codes(probs) + + +# ========================================================================== +# 静态校验、契约 +# ========================================================================== + + +def test_static_checks_on_text_measures(drafted): + _, d = drafted + r = full(d.recipe) + seg = measures_seg(r) + seg["text_measures"]["不存在的指标"] = {"parser": "text"} + seg["text_measures"][PEAK_N] = {"parser": "text"} + table = next(t for t in r["tables"] if t["name"] == seg["table"]) + table["units"]["高峰时段"] = "人次" + _parsed, probs = validate_recipe(r, facts=d.facts, origin="manual") + got = {(p.code, p.path) for p in probs} + sp = "/sheets/0/blocks/0/segments/0" + assert ("text_measure_unknown", f"{sp}/text_measures/不存在的指标") in got + assert ("text_measure_unit", f"{sp}/text_measures/{PEAK_N}") in got + assert any(code == "unit_on_text" and path.endswith("/units/高峰时段") for code, path in got) + + +def test_placeholder_with_digits_is_rejected_except_on_replay_or_when_carried(drafted): + _, d = drafted + r = full(d.recipe) + measures_seg(r)["text_measures"] = {} + r["sheets"][0]["blocks"][0]["values"]["placeholders"].append({"text": "18-19", "meaning": "无数据"}) + _parsed, probs = validate_recipe(r, facts=d.facts, origin="manual") + p = next(p for p in probs if p.code == "placeholder_digits") + assert p.path == "/sheets/0/blocks/0/values/placeholders/1/text" + assert "「指标类型」改为「文字」或「时段」" in p.message + assert "placeholder_digits" not in codes(validate_recipe(r, origin="replay")[1]) + base = Recipe.model_validate(copy.deepcopy(r)) + assert "placeholder_digits" not in codes(validate_recipe(r, facts=d.facts, origin="manual", base=base)[1]) + + +def test_contract_text_measures_are_text_columns_and_hash_compatible(drafted): + _, d = drafted + parsed = Recipe.model_validate(d.recipe) + cols, problems = derive_tables(parsed) + seg = measures_seg(d.recipe) + assert problems == [] + assert {c.name: c.type for c in cols[seg["table"]]}["高峰时段"] == "TEXT" + # 默认值（没有文字指标）不进规范形式：已存配方的哈希和构建 id 不变 + plain = full(d.recipe) + measures_seg(plain)["text_measures"] = {} + assert "text_measures" not in str(canonical_recipe(Recipe.model_validate(plain))) + + +# ========================================================================== +# 确认项、前后对照、说明 +# ========================================================================== + + +def test_confirm_item_and_breaking_changes(drafted): + book, d = drafted + ex = book.run(d.recipe) + seg = measures_seg(d.recipe) + items = {i.id: i for i in confirm_items(ConfirmContext(kind="first", recipe=d.recipe, extraction=ex, checks=[]))} + item = items[f"text_measure:{seg['id']}:{PEAK}"] + assert "按时段导入" in item.label and "文字列" in item.label + as_text = full(d.recipe) + measures_seg(as_text)["text_measures"][PEAK] = {"parser": "text"} + kinds = {(c.kind, c.column) for c in table_changes(d.recipe, as_text)} + assert ("store", "高峰时段") in kinds + plain = full(d.recipe) + measures_seg(plain)["text_measures"] = {} + assert ("type", "高峰时段") in {(c.kind, c.column) for c in table_changes(d.recipe, plain)} + + +def test_column_note_explains_nulls_of_the_text_measure(drafted, tmp_path): + from app.data.recipe_checks import run_checks + from app.data.recipe_notes import build_notes + + book, d = drafted + db = str(tmp_path / "n.db") + ex = book.run(d.recipe, db) + parsed = Recipe.model_validate(d.recipe) + notes = build_notes(parsed, ex, run_checks(db, ex, parsed), []) + seg = measures_seg(d.recipe) + assert "空值" in notes.tables[seg["table"]].columns.get("高峰时段", "") diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index 22c3c606..b6149d7f 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -1627,6 +1627,8 @@ export const RECIPE_TEXT = { derivedSegment: (id: string) => `合计段「${id}」`, tableName: '表名', columnName: '列名', + /** measures 段每个指标的「指标类型」：数字按块的「类型」导入；文字、时段落成文字列（执行器的报错里写的就是这个字段名） */ + measureKind: '指标类型', unit: '单位', noUnit: '无单位', grain: '主键', @@ -1909,6 +1911,7 @@ export const FIX_KIND_LABEL: Record = { rename_title: '改分段标题', declare_total: '改作合计核对', declare_subtotals: '改作小计核对', + declare_text_measure: '按文字导入这一行', ignore_cells: '按文字忽略', declare_placeholder: '声明占位符', declare_hidden: '设置隐藏行处理', @@ -2152,6 +2155,7 @@ export const RECIPE_ORIGIN_LABEL: Record = { export const RECIPE_CHOICE_LABEL = { locateBy: { labels: '按标签', section_title: '按分段标题' } as Record, dimParser: { hour_range: '时段（如 7-8）', text: '文字' } as Record, + measureKind: { '': '数字', hour_range: '时段', text: '文字' } as Record, valueType: { INTEGER: '整数', REAL: '小数' } as Record, blankCross: { reject: '拒收', null: '存为空值（需确认）' } as Record, blankList: { null: '存为空值', reject: '拒收' } as Record, diff --git a/frontend/src/pages/import/RecipePanel.tsx b/frontend/src/pages/import/RecipePanel.tsx index 003f029b..a791d61a 100644 --- a/frontend/src/pages/import/RecipePanel.tsx +++ b/frontend/src/pages/import/RecipePanel.tsx @@ -130,8 +130,9 @@ export function tableColumns(recipe: Recipe | null): Map for (const seg of arr(block.segments)) { if (!isObj(seg)) continue if (seg.role === 'measures') { + const text = isObj(seg.text_measures) ? seg.text_measures : {} put(seg.table, [axis, ...arr(seg.labels?.expect).map((label: string) => ({ - name: String(seg.measures?.[label] ?? ''), type: vtype, role: 'measure', header: label, + name: String(seg.measures?.[label] ?? ''), type: isObj(text[label]) ? 'TEXT' : vtype, role: 'measure', header: label, }))]) } else if (seg.role === 'dimension') { put(seg.table, [ @@ -568,6 +569,23 @@ function DeriveNames({ path, derive, onRename }: { path: string; derive: unknown ) } +/** + * 指标改按文字（text / hour_range）或改回数字（空串）。文字列不能有单位：改成文字时去掉这一列的单位，改回数字时按标签 + * 末尾的括号补回（与加标签时一致） + */ +function setMeasureKind(r: Recipe, x: any, label: string, kind: string, units: string[]) { + const col = x.measures?.[label] + const table = arr(r.tables).find((t) => t?.name === x.table) + if (kind) { + x.text_measures = { ...(isObj(x.text_measures) ? x.text_measures : {}), [label]: { parser: kind } } + if (table && col && isObj(table.units)) delete table.units[col] + return + } + if (isObj(x.text_measures)) delete x.text_measures[label] + const [, unit] = splitUnit(label, units) + if (unit && table && col) table.units = { ...(isObj(table.units) ? table.units : {}), [col]: unit } +} + function SegmentEditor({ s, b, g, seg, edit, candidates, labelOptions, units }: { s: number; b: number; g: number; seg: any; edit: Edit candidates: Record; labelOptions: string[]; units: string[] @@ -596,6 +614,7 @@ function SegmentEditor({ s, b, g, seg, edit, candidates, labelOptions, units }: if (x.role === 'measures' && isObj(x.measures)) { const col = x.measures[label] delete x.measures[label] + if (isObj(x.text_measures)) delete x.text_measures[label] const table = arr(r.tables).find((t) => t?.name === x.table) if (table && col) { if (isObj(table.units)) delete table.units[col] @@ -644,7 +663,14 @@ function SegmentEditor({ s, b, g, seg, edit, candidates, labelOptions, units }: if (old) renameColumn(r, x.table, old, v) else x.measures = { ...(isObj(x.measures) ? x.measures : {}), [label]: v } })} /> + + ))} diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 85aa7987..513e64c7 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -800,10 +800,11 @@ export interface CurrentRecipe { // 和接口的 JSON 同名（Selection 的 as 除外：后端 dataclass 叫 as_，JSON 里是 as） // --------------------------------------------------------------------------- -/** 修复按钮的九种（契约 FIX_KINDS） */ +/** 修复按钮的种类（契约 FIX_KINDS） */ export type FixKind = | 'remove_label' | 'add_label' | 'edit_members' | 'rename_title' | 'declare_total' | 'ignore_cells' - | 'declare_placeholder' | 'declare_hidden' | 'rename_sheet' | (string & {}) + | 'declare_placeholder' | 'declare_hidden' | 'rename_sheet' | 'declare_subtotals' | 'declare_text_measure' + | (string & {}) /** 修复提议的一个封闭选项。效果由服务端算：界面只发 fix_id、value 和理由 */ export interface FixOption { diff --git a/scripts/check-manage.mjs b/scripts/check-manage.mjs index 005eb0ac..16feed64 100644 --- a/scripts/check-manage.mjs +++ b/scripts/check-manage.mjs @@ -2528,13 +2528,34 @@ await section('数据 · 表格：按配方导入', async () => { check('勾选「没有小计行」：发 PUT，no_subtotals 为 true、没有配小计', noSub?.no_subtotals === true && !noSub?.subtotals, JSON.stringify(noSub ?? {})) await page.waitForTimeout(300) + // 指标类型：整行是文字的指标（如「高峰时段」写着「18-19」）改按时段 / 文字导入，落成文字列、去掉单位；改回数字时按标签补回单位 + const kindSel = panel.locator('[data-field="/sheets/0/blocks/0/segments/0/text_measures/分区乙（人次）"]') + const kindOptions = await kindSel.locator('option').evaluateAll((os) => os.map((o) => `${o.value}:${o.textContent}`)) + check('指标类型是下拉框：数字、时段、文字', JSON.stringify(kindOptions) === JSON.stringify([':数字', 'hour_range:时段', 'text:文字']), + kindOptions.join(',')) + replies.recipe.push(echo()) + await kindSel.selectOption('hour_range') + await until(async () => puts().length === 7, 4000) + const k1 = puts()[6]?.body?.recipe + check('改成「时段」：发 PUT，text_measures 写上 hour_range，这一列的单位去掉', + k1?.sheets?.[0]?.blocks?.[0]?.segments?.[0]?.text_measures?.['分区乙（人次）']?.parser === 'hour_range' + && !('分区乙' in (k1?.tables?.[0]?.units ?? {})), JSON.stringify(k1?.tables?.[0]?.units ?? {})) + await page.waitForTimeout(300) + replies.recipe.push(echo()) + await kindSel.selectOption('') + await until(async () => puts().length === 8, 4000) + const k2 = puts()[7]?.body?.recipe + check('改回「数字」：发 PUT，text_measures 去掉这一项，单位按标签补回「人次」', + !('分区乙（人次）' in (k2?.sheets?.[0]?.blocks?.[0]?.segments?.[0]?.text_measures ?? {})) + && k2?.tables?.[0]?.units?.['分区乙'] === '人次', JSON.stringify(k2?.tables?.[0]?.units ?? {})) + await page.waitForTimeout(300) // 有问题的配方：一条对得上字段（单位），一条对不上（挂到面板顶部） replies.recipe.push(echo({ recipe_problems: [ { path: '/tables/0/units/分区甲', code: 'unit_label_conflict', message: '表「日客流量」：列「分区甲」的单位与标签「分区甲（人次）」不一致' }, { path: '/sheets/0/blocks/0/label_offset', code: 'schema', message: '第 1 个工作表的交叉表：标签列偏移只能是紧挨日期的左侧一列' }, ] })) await panel.locator('[data-field="/tables/0/units/分区甲"]').selectOption('人') - await until(async () => puts().length === 7, 4000) + await until(async () => puts().length === 9, 4000) await page.waitForTimeout(300) const trialBtn = wizard().locator('[data-trial-run]') check('配方有问题时「试运行」禁用', await trialBtn.isDisabled()) @@ -2548,8 +2569,8 @@ await section('数据 · 表格：按配方导入', async () => { await panel.locator('[data-recipe-paste-toggle]').click() await panel.locator('[data-recipe-paste] textarea').fill(JSON.stringify(FLOW_RECIPE)) await panel.locator('[data-recipe-paste] button', { hasText: '保存配方' }).click() - await until(async () => puts().length === 8, 4000) - check('「粘贴配方」发 PUT，body 就是粘贴的配方', JSON.stringify(puts()[7]?.body?.recipe) === JSON.stringify(FLOW_RECIPE)) + await until(async () => puts().length === 10, 4000) + check('「粘贴配方」发 PUT，body 就是粘贴的配方', JSON.stringify(puts()[9]?.body?.recipe) === JSON.stringify(FLOW_RECIPE)) await page.waitForTimeout(300) check('……问题清掉以后「试运行」可用', await trialBtn.isEnabled()) // 连续修改：上一次 PUT 还没回来就接着改。上一次回来时，排在后面的修改不能被服务端那份盖掉，