合入:交叉表支持按文字导入的指标(「高峰时段」一行写着「18-19」时落成文字列,规则起草按文字指标起草并要人确认,配方面板加「指标类型」);堵上占位符陷阱(拒收含数字的占位符,「不是数字」的下一步按格子情况写,整行文字给「按文字导入这一行」修复按钮)。验收:配方相关测试 2195 条、新增 15 条全过;check-copy 通过;check-manage 718 项全过;真实文件本机重跑起草完整、按日 31 行、时段 527 行;泄露扫描干净

This commit is contained in:
ThearchyHelios
2026-10-20 11:41:44 +08:00
15 changed files with 861 additions and 34 deletions
+50 -4
View File
@@ -265,7 +265,7 @@ _FIELD_LABEL = {
"meaning": "含义", "blank": "空格的处理", "text_number": "文本数字的处理", "formula": "公式的处理",
"segments": "分段", "role": "分段类型", "table": "表", "locate": "定位方式", "by": "定位依据",
"title": "分段标题", "segment": "紧跟的分段", "labels": "标签", "expect": "期望的标签",
"measures": "指标列名对照", "dim": "维度", "derive": "派生列", "value": "值列", "const": "常量列",
"measures": "指标列名对照", "text_measures": "按文字导入的指标", "dim": "维度", "derive": "派生列", "value": "值列", "const": "常量列",
"pick": "取值", "stop_parser": "结束本段的标签", "labels_parser": "标签解析方式", "verify": "核对方式",
"against_table": "核对的基表", "keep_as": "另存", "header_rows": "表头行数", "after_title": "表头上方的标题",
"columns": "列", "header": "表头", "store": "存储写法", "extra_columns": "多出的列",
@@ -815,6 +815,21 @@ class _Checker:
if key in seen:
self.add(lp, "label_duplicate", f"标签「{label}」与「{seen[key]}」是同一个标签")
seen.setdefault(key, label)
if isinstance(seg, MeasuresSegment):
self.check_text_measures(sp, seg)
def check_text_measures(self, sp: tuple[Any, ...], seg: MeasuresSegment) -> None:
"""文字指标的键与 measures 一样逐字对 labels.expect;标签末尾写着词表里的单位的是数字指标,不能按文字导入。"""
for label in seg.text_measures:
tp = sp + ("text_measures", label)
if label not in seg.labels.expect:
self.add(tp, "text_measure_unknown",
f"按文字导入的「{label}」不在期望的标签中(必须与标签逐字相同)")
continue
unit = split_unit_suffix(label)[1]
if unit is not None and unit in UNITS:
self.add(tp, "text_measure_unit",
f"标签「{label}」写着单位「{unit}」,是数字指标,不能按文字导入")
@staticmethod
def label_key(seg: Any, label: str) -> str | None:
@@ -880,13 +895,25 @@ class _Checker:
# ---- 3. 占位符只能映射空值,不能像数 ----
def check_placeholders(self) -> None:
"""占位符不能像数;也不能含数字(origin 不是 replay 时)。「18-19」「约3000」不像一个数,却是数据本身:
当成占位符,整格的内容就静默存成空值,干跑照样通过。表示无数据的符号(「·」「—」「/」「无」)不含数字;
已提交的配方里沿用下来的(同一块、canon 相同)放行,重放也不报,免得期 3 之前存下的配方在下一期被拒。"""
literal = self.origin != "replay"
carried = self.base_placeholders() if literal else {}
for si, sheet in enumerate(self.r.sheets):
for bi, block in enumerate(sheet.blocks):
for n, ph in enumerate(block.values.placeholders):
pp = ("sheets", si, "blocks", bi, "values", "placeholders", n, "text")
if looks_numeric_text(ph.text):
self.add(("sheets", si, "blocks", bi, "values", "placeholders", n, "text"),
"placeholder_numeric",
self.add(pp, "placeholder_numeric",
f"占位符「{ph.text}」看起来是一个数,不能当作占位符(占位符一律存为空值)")
elif (literal and any(ch.isdigit() for ch in ph.text)
and canon(ph.text) not in carried.get(block.id, set())):
what = (";如果这一行写的都是文字(如时段、名称),请把这个指标的「指标类型」改为「文字」或「时段」"
if isinstance(block, CrosstabBlock) else "")
self.add(pp, "placeholder_digits",
f"占位符「{ph.text}」含数字:表示无数据的符号不含数字(如「·」「—」「无」),"
f"含数字的通常是数据本身,当作占位符会整格存成空值{what}")
for si, bi, block in self.lists:
total = block.rows.total_row
if total is not None and any(ch.isdigit() for ch in total.pick):
@@ -897,6 +924,14 @@ class _Checker:
self.add(("sheets", si, "blocks", bi, "rows", "subtotals", "pick"), "period_literal",
f"小计行的开头词「{sub.pick}」不能含数字(配方里不能写死年份或期次)")
def base_placeholders(self) -> dict[str, set[str]]:
"""现行配方(base)里每个块的占位符(canon 后):块 id → 集合。没有 base 时为空。"""
out: dict[str, set[str]] = {}
for sheet in self.base.sheets if self.base is not None else []:
for block in sheet.blocks:
out.setdefault(block.id, set()).update(canon(p.text) for p in block.values.placeholders)
return out
# ---- 4、9、13. 单位:键是本表列名,值在词表里,与原表标签、表头里的单位一致 ----
def check_units(self) -> None:
@@ -915,7 +950,12 @@ class _Checker:
for si, bi, _gi, _block, seg in self.segs:
if isinstance(seg, MeasuresSegment):
for label in seg.labels.expect:
if label in seg.measures:
if label not in seg.measures:
continue
if label in seg.text_measures:
# 标签写着单位的已报 text_measure_unit;这里只拦文字列上写的单位
self.no_unit_on_text(seg.table, seg.measures[label])
else:
self.unit_against(seg.table, seg.measures[label], label, "标签")
elif isinstance(seg, DimensionSegment) and seg.locate.by == "section_title" and seg.locate.title:
self.unit_against(seg.table, seg.value, seg.locate.title, "分段标题")
@@ -938,6 +978,12 @@ class _Checker:
return next((s for sj, bj, _gi, _block, s in self.segs
if (sj, bj) == (si, bi) and isinstance(s, DimensionSegment) and s.id == seg.locate.segment), None)
def no_unit_on_text(self, table: str, column: str) -> None:
i = self.declared.get(table)
if i is None or not column or column not in self.r.tables[i].units:
return
self.add(("tables", i, "units", column), "unit_on_text", f"列「{column}」按文字导入,不能写单位")
def unit_against(self, table: str, column: str, source: str, what: str) -> None:
i = self.declared.get(table)
if i is None or not column or (table in self.broken and column not in self.col_names(table)):
+2 -1
View File
@@ -121,7 +121,8 @@ RECIPE_DRAFT_SYSTEM = (
"2. 配方语言是封闭的:没有正则,不写单元格坐标;只能用 Schema 里有的字段和取值。\n"
"3. measures 的键必须与 labels.expect 里的标签逐字相同、一一对应。\n"
"4. const 的 pick 只能从「分段标题候选词」里选。\n"
"5. 占位符只能表示空值(无数据或不适用)。\n"
"5. 占位符只能表示空值(无数据或不适用),不含数字。指标行每格写的都是文字(如时段「18-19」)时,"
"在该 measures 分段的 text_measures 里声明(时段用 hour_range,其余用 text),不要当成占位符。\n"
"6. 单位只能取自「单位词表」;标签或表头末尾括号里的单位拆进 tables[].units,列名去掉单位,"
"单位必须与括号里写的一致;括号里的单位不在词表里时,列名保留括号、不写单位。\n"
"7. 合计、小计行用 role=derived 核对,并用 keep_as 另存成一张表。\n"
+18 -1
View File
@@ -214,7 +214,8 @@ def _column_sources(recipe: Recipe) -> dict[tuple[str, str], str]:
def _text_stores(recipe: Recipe) -> dict[tuple[str, str], tuple[str, bool]]:
"""(表, 列表的 TEXT 列) → (实际生效的存法, 是否显式写了)。缺省时主键列 canonical、其余 raw。"""
"""(表, 列表的 TEXT 列) → (实际生效的存法, 是否显式写了)。缺省时主键列 canonical、其余 raw。
交叉表按文字导入的指标也算:按时段解析的存规范写法,按文字的存原文,都是显式写的。"""
grains = {t.name: set(t.grain) for t in recipe.tables}
out: dict[tuple[str, str], tuple[str, bool]] = {}
for sheet in recipe.sheets:
@@ -224,6 +225,13 @@ def _text_stores(recipe: Recipe) -> dict[tuple[str, str], tuple[str, bool]]:
if c.type == "TEXT":
eff = c.store or ("canonical" if c.name in grains.get(block.table, set()) else "raw")
out[(block.table, c.name)] = (eff, c.store is not None)
continue
for seg in block.segments:
if isinstance(seg, MeasuresSegment):
for label, tm in seg.text_measures.items():
if seg.measures.get(label):
out[(seg.table, seg.measures[label])] = (
"canonical" if tm.parser == "hour_range" else "raw", True)
return out
@@ -516,6 +524,9 @@ def _signatures(recipe: Recipe, sheet_name: Callable[[SheetRecipe], str]) -> dic
if isinstance(seg, DerivedSegment):
keep = seg.keep_as.table if seg.keep_as else None
out[f"derived:{seg.id}"] = _sig(keep, sorted(match_key(x) for x in seg.labels.expect))
elif isinstance(seg, MeasuresSegment):
for label, tm in seg.text_measures.items():
out[f"text_measure:{seg.id}:{label}"] = _sig(tm.parser, seg.measures.get(label))
else:
out[f"merged_fill:{block.id}"] = _sig(block.merged_data)
out[f"ignored_columns:{block.id}"] = _sig(block.extra_columns, ignore_cols)
@@ -1048,6 +1059,12 @@ def _recipe_items(bag: _Bag, recipe: Recipe, ex: dict[str, Any], facts: Any, *,
prefix = rows_text(seg_rows.get(seg.id)) or f"分段「{seg.id}」的合计行"
keep = f",原值另存表「{seg.keep_as.table}」" if seg.keep_as else ",原值不另存"
bag.add(f"derived:{seg.id}", f"{prefix}改作核对{keep}")
elif isinstance(seg, MeasuresSegment):
# 文字指标:这一行不再按数字核对类型,写错成别的文字也照样入库,所以配置了就要人确认
for label, tm in seg.text_measures.items():
how = "按时段导入,存规范写法(如 18-19)" if tm.parser == "hour_range" else "按文字原样导入"
bag.add(f"text_measure:{seg.id}:{label}",
f"「{label}」{how},不当数字:表「{seg.table}」的列「{seg.measures.get(label, '')}」是文字列")
derived_rows: dict[str, set[int]] = {}
for d in ex.get("derived") or []:
if isinstance(d, dict) and d.get("kind") == "column_sum":
+126 -6
View File
@@ -67,10 +67,10 @@ from app.data import engine as _data_engine
from app.data import tabular, xlsx_cells
from app.data.names import canon
from app.data.recipe_parsers import (
TOTAL_WORDS, AxisDate, HourRange, HourTotal, Period, classify_outside, cn_date_range, cn_year_month,
TOTAL_WORDS, UNITS, AxisDate, HourRange, HourTotal, Period, classify_outside, cn_date_range, cn_year_month,
filename_period,
hour_range, hour_range_total, is_pure_period_cell, looks_numeric_text, match_key, month_day_or_date,
period_residue, text_label, thousands_number,
period_residue, split_unit_suffix, text_label, thousands_number,
)
from app.data.recipe_types import (
SUBTOTAL_KEY, AxisOut, CanonEntry, CheckResult, ColumnOut, ContextSpec, CrosstabBlock, DerivedItem,
@@ -116,7 +116,7 @@ PROBLEM_CODES: dict[str, str] = {
"value_not_integer": "structure", "value_text_number": "structure", "value_blank": "structure",
"value_error": "structure", "value_not_number": "structure", "value_not_date": "structure",
"value_formula": "structure", "formula_uncached": "structure", "formula_full_calc": "structure",
"pk_duplicate": "structure",
"pk_duplicate": "structure", "value_not_text": "structure", "value_not_hour_range": "structure",
}
#: 修复按钮的种类(契约 Problem.fix,取值见 FIX_KINDS)。参数 fix_args 的形状见 P3-SPEC 3.2,由报问题的地方随问题
@@ -520,6 +520,9 @@ class _Issue:
row: int | None = None
#: value_not_number:能当占位符提议的文字(canon 后)→ 格数,最多收 PLACEHOLDER_TEXTS_MAX + 1 种
texts: Counter[str] = field(default_factory=Counter)
#: value_not_number:measures 段里整行都是文字的指标(declare_text_measure 的参数):
#: {"segment", "label"(期望标签原文), "cell"(标签格), "parsers"(能用的解析方式)}
text_rows: list[dict[str, Any]] = field(default_factory=list)
#: 合并类问题的说明:code → (原因({n} 是个数), 下一步)。下一步写配方面板上的「字段」和「选项」原文
@@ -530,8 +533,13 @@ _ISSUE_TEXT: dict[str, tuple[str, str]] = {
"。如需按数字导入,请在配方面板的「文本写的数字」中选择「千分位写法按数字保存」"),
"value_blank": ("{n} 个单元格为空", "。如原表确实为空,请在配方面板的「空格」中选择「存为空值」"),
"value_error": ("{n} 个单元格是错误值", "。请在 Excel 中修正后重新上传"),
# 下一步按格子的情况另写(_not_number_tail):整行是文字的提示改「指标类型」,能当占位符的才提示加占位符
"value_not_number": ("{n} 个单元格不是数字", "。如果它表示无数据,请在配方面板中添加占位符"),
"value_not_date": ("{n} 个单元格不是日期", "。请检查原表,或在配方面板中把这一列的「类型」改为「文字」"),
"value_not_text": ("{n} 个单元格不是文字",
"。这个指标按文字导入:请检查原表,或在配方面板中把它的「指标类型」改为「数字」"),
"value_not_hour_range": ("{n} 个单元格无法按时段解析",
"。这个指标按时段导入:请检查原表,或在配方面板中把它的「指标类型」改为「文字」"),
"value_formula": ("{n} 个数据格是公式",
"。数据区的公式通常是没有识别出来的合计;如确为数据,请在配方面板的「公式」中选择「按保存值导入」"),
"formula_uncached": ("{n} 个公式格没有保存计算结果", ",文件未经计算保存。请在 Excel 中打开并保存后重新上传"),
@@ -548,6 +556,21 @@ _PK_PLACEHOLDER = ("主键列有 {n} 个单元格是占位符", "。占位符按
_PK_NULL = ("有 {n} 处为空值", "。主键不能为空:请检查原表,或在配方面板的「主键」中去掉这一列")
def _not_number_tail(it: _Issue) -> str:
"""value_not_number 的下一步。整行写的都是文字的指标,提示改「指标类型」;有能当占位符的文字(不含数字的短符号)
才提示加占位符:「18-19」这种写法当占位符会整格存成空值,干跑照样通过(静态校验 placeholder_digits 也拦)。"""
hint: list[str] = []
if it.text_rows:
hint.append(f"{_quoted([x['label'] for x in it.text_rows])}这一行写的都是文字:如果本来就是文字(如时段、名称),"
"请在配方面板中把它的「指标类型」改为「文字」或「时段」")
if it.texts:
hint.append("如果其中有表示无数据的符号,请在配方面板中添加占位符" if hint
else "如果它表示无数据,请在配方面板中添加占位符")
if not hint:
return "。数值区只能是数字或表示无数据的符号(如「·」「—」,不含数字):请检查原表"
return "。" + ";".join(hint)
class _Problems:
def __init__(self) -> None:
self.items: list[tuple[str | None, Problem]] = []
@@ -592,6 +615,12 @@ class _Problems:
if sample is not None and (sample in it.texts or len(it.texts) <= PLACEHOLDER_TEXTS_MAX):
it.texts[sample] += 1
def text_row(self, sheet: str, where: str, row: dict[str, Any]) -> None:
"""把整行是文字的指标记到这一段的 value_not_number 上(那一行的格转换时已经各自报过)。"""
it = self.issues.get(("value_not_number", sheet, where + "\x00"))
if it is not None and all(x["label"] != row["label"] for x in it.text_rows):
it.text_rows.append(row)
def finish(self, failed_sheets: set[str],
args_for: Callable[[_Issue], dict[str, Any] | None] | None = None) -> list[Problem]:
"""合并类问题出成 Problem;args_for 给有修复按钮的合并类问题(outside_number、value_not_number)算 fix_args。
@@ -602,7 +631,8 @@ class _Problems:
ex = f"(如 {';'.join(it.examples)})" if it.examples else f"({_join(shown, 5)})"
model_ex = f"({_join(shown, 5)})"
args = args_for(it) if args_for is not None and it.code in PROBLEM_FIX else None
self.add(it.code, f"{it.where}:{reason}{ex}{it.tail}", f"{it.where}:{reason}{model_ex}", it.cells,
tail = _not_number_tail(it) if it.code == "value_not_number" else it.tail
self.add(it.code, f"{it.where}:{reason}{ex}{tail}", f"{it.where}:{reason}{model_ex}", it.cells,
sheet=it.sheet, fix_args=args)
for (code, sheet), n in self._over.items():
where = f"工作表「{sheet}」" if sheet else "本文件"
@@ -1482,10 +1512,22 @@ class _Crosstab:
eng.headers[(seg.table, seg.measures.get(label, ""))] = _text(self._label(r).value) # type: ignore[union-attr]
# 值保留 _FAIL(已报过问题),交给 _Sink 按空值存;没找到的标签(已报 label_missing)也按 _FAIL
values: dict[tuple[str, int], Any] = {}
pol = self.block.values
for label, r in rows_by_label.items():
excluded = self.exclude_hidden and sh.hidden_row(r)
text = seg.text_measures.get(label)
column = seg.measures.get(label, "")
for c in self._axis_cols():
values[(label, c)] = None if excluded else self._value(r, c, where)
if excluded:
values[(label, c)] = None
elif text is not None:
values[(label, c)] = eng.convert_text(self.grid.get(r, c), text.parser, pol, sh.name, where, r, c,
seg.table, column)
else:
values[(label, c)] = self._value(r, c, where)
if text is None and not excluded and (parsers := self._text_parsers(r, label)):
eng.problems.text_row(sh.name, where, {"segment": seg.id, "label": label,
"cell": sh.coord(r, self.label_c), "parsers": parsers})
if self.dates is None:
return
for c in self._axis_cols():
@@ -1498,6 +1540,31 @@ class _Crosstab:
cells.append((seg.measures.get(label, ""), r, c))
sink.add(row, sheet=sh.name, at=(self.axis_row, c), cells=cells, fails=True)
def _text_parsers(self, r: int, label: str) -> list[str] | None:
"""这一行能不能改按文字指标导入(declare_text_measure 的提议):轴列上除了占位符和空格,全是文字(不是数字、
日期、公式、错误值,也不是写成文字的数),至少一格。全都能按时段解析时给两种解析方式,时段在前。标签写着
词表里的单位的是数字指标(静态校验 text_measure_unit 也拒),不提议。"""
unit = split_unit_suffix(label)[1]
if unit is not None and unit in UNITS:
return None
ph = self.eng._placeholders(self.block.values)
texts: list[str] = []
for c in self._axis_cols():
cell = self.grid.get(r, c)
if cell is None:
continue
if cell.formula is not None or not isinstance(cell.value, str):
return None
t = cell.value.strip()
if not t or canon(t) in ph:
continue
if t in tabular.EXCEL_ERRORS or thousands_number(t) is not None or looks_numeric_text(t):
return None
texts.append(t)
if not texts:
return None
return ["hour_range", "text"] if all(hour_range(t) is not None for t in texts) else ["text"]
def _derive_values(self, derive: dict[str, str], hr: Any) -> list[Any]:
ordered = sorted(derive.items(), key=lambda kv: (kv[1] != "start", kv[0]))
if not isinstance(hr, (HourRange, HourTotal)):
@@ -2441,6 +2508,55 @@ class _Engine:
return v if vtype == "REAL" else int(v)
return self._number(v, vtype, pol, sheet, where, r, c, pk)
def convert_text(self, cell: GridCell | None, parser: str, pol: Any, sheet: str, where: str, r: int, c: int,
table: str, column: str) -> Any:
"""文字指标的一格(MeasuresSegment.text_measures)→ 入库值。空格、公式、占位符照块的取值规则;text 存原文
(去首尾空白),hour_range 存时段的规范写法。数字、日期不是文字(不替人把 8-9 被 Excel 改成的日期认回去),
解析不了的时段也是问题:都按 _FAIL 返回、已记问题。"""
p = self.problems
if cell is None:
if pol.blank == "null":
return None
p.issue("value_blank", sheet, where, _coord(sheet, r, c))
return _FAIL
v = cell.value
if cell.formula is not None:
v = self._accept_formula(cell, pol, sheet, where, r, c)
if v is _FAIL:
return _FAIL
if not isinstance(v, str):
p.issue("value_not_text", sheet, where, _coord(sheet, r, c), f"{cell_ref(r, c)}「{_shown(v)}」")
return _FAIL
t = v.strip()
ph = self._placeholders(pol)
key = canon(t)
if key in ph:
name = ph[key]
self.ex.placeholders[name] = self.ex.placeholders.get(name, 0) + 1
return None
if not t:
if pol.blank == "null":
return None
p.issue("value_blank", sheet, where, _coord(sheet, r, c))
return _FAIL
if t in tabular.EXCEL_ERRORS:
p.issue("value_error", sheet, where, _coord(sheet, r, c), f"{cell_ref(r, c)}「{_shown(v)}」")
return _FAIL
if parser == "text":
return t
hr = hour_range(t)
if hr is None:
p.issue("value_not_hour_range", sheet, where, _coord(sheet, r, c), f"{cell_ref(r, c)}「{_shown(v)}」")
return _FAIL
if hr.canonical != t:
log = (table, column, t, hr.canonical)
hit = self.canon_log.get(log)
if hit is not None:
hit.count += 1
elif len(self.canon_log) < CANON_MAX:
self.canon_log[log] = CanonEntry(table, column, t, hr.canonical, 1, _coord(sheet, r, c))
return hr.canonical
def _accept_formula(self, cell: GridCell, pol: Any, sheet: str, where: str, r: int, c: int) -> Any:
"""数据区的公式格:拒收、无缓存、打开时重算都是问题;否则按保存值导入并计数。"""
p = self.problems
@@ -2675,7 +2791,11 @@ class _Engine:
if it.block is None:
return None
texts = list(it.texts)[:PLACEHOLDER_TEXTS_MAX]
return {"block": it.block, "texts": texts, "other": it.count - sum(it.texts[t] for t in texts)}
args = {"block": it.block, "texts": texts, "other": it.count - sum(it.texts[t] for t in texts)}
if it.text_rows:
# 只有标签原文、标签格坐标和解析方式,没有数据格的内容
args["text_rows"] = [dict(x) for x in it.text_rows]
return args
if it.code == "outside_number" and it.row is not None:
sh = next((x for x in self.sheets if x.name == it.sheet), None)
if sh is None:
+84 -2
View File
@@ -27,10 +27,12 @@ from pydantic import ValidationError
from app.data.names import canon, collide_key
from app.data.recipe import apply_patch, recipe_sha256
from app.data.recipe_parsers import (
UNITS,
candidate_words,
looks_numeric_text,
match_key,
month_day_or_date,
split_unit_suffix,
)
from app.data.recipe_suggest import _measure_column
from app.data.recipe_types import (
@@ -255,6 +257,8 @@ def _key(kind: str, target: dict[str, Any]) -> str:
return f"rename_sheet:{t['sheet_id']}"
if kind == "declare_subtotals":
return f"declare_subtotals:{t['block']}"
if kind == "declare_text_measure":
return f"declare_text_measure:{t['segment']}:{t['label']}"
raise KeyError(kind)
@@ -308,6 +312,8 @@ def _o_remove_label(recipe: dict, p: FixProposal, opt: FixOption, reason: str |
cols = []
for e in removed:
col = seg["measures"].get(e)
if e in (seg.get("text_measures") or {}):
ops.append({"op": "remove", "path": ptr(*sp, "text_measures", e)})
if col is None:
continue
cols.append(col)
@@ -967,6 +973,67 @@ def _o_declare_placeholder(recipe: dict, p: FixProposal, opt: FixOption, reason:
return ops, [f"块「{block['id']}」把{quoted(p.target['texts'])}声明为{meaning}(存为空值)"]
# ==========================================================================
# ⑪ 按文字导入整行是文字的指标(value_not_number 的另一个出口:「高峰时段」一行写的是「18-19」)
# ==========================================================================
_TEXT_PARSER_OPTION = {
"hour_range": ("按时段导入(存规范写法,如 18-19)", "以后各期这一行每格都要能按时段解析,否则拒收"),
"text": ("按文字原样导入", "以后各期这一行按原文存,不再要求是数字"),
}
def _p_declare_text_measure(recipe: dict, prob: Any, args: dict, anchor: FixAnchor) -> list[FixProposal]:
"""执行器在 fix_args.text_rows 里给出整行都是文字的指标(标签原文、标签格、能用的解析方式)。每个指标一条提议;
标签写着词表里的单位的不提议(静态校验 text_measure_unit 会拒),已经按文字导入的不再提议。"""
out: list[FixProposal] = []
for row in args.get("text_rows") or []:
found = find_segment(recipe, row.get("segment"))
label = row.get("label")
if found is None or not isinstance(label, str):
continue
seg = found[4]
if (seg.get("role") != "measures" or label not in (seg.get("labels") or {}).get("expect", [])
or label in (seg.get("text_measures") or {}) or not seg.get("measures", {}).get(label)):
continue
unit = split_unit_suffix(label)[1]
if unit is not None and unit in UNITS:
continue
parsers = [p for p in row.get("parsers") or [] if p in _TEXT_PARSER_OPTION]
if not parsers:
continue
target = {"segment": seg["id"], "label": label}
col = seg["measures"][label]
cell = row.get("cell")
out.append(_proposal(
"declare_text_measure", _key("declare_text_measure", target), code=_get(prob, "code"),
title=f"「{label}」这一行写的都是文字,不是数字",
cells=[cell] if isinstance(cell, str) else [], target=target,
options=[FixOption(p, _TEXT_PARSER_OPTION[p][0],
detail=f"表「{seg.get('table')}」的列「{col}」改成文字列;{_TEXT_PARSER_OPTION[p][1]}",
breaking=True) for p in parsers],
anchor=anchor))
return out
def _o_declare_text_measure(recipe: dict, p: FixProposal, opt: FixOption, reason: str | None) -> tuple[list, list[str]]:
found = find_segment(recipe, p.target.get("segment"))
label = p.target.get("label")
if found is None or found[4].get("role") != "measures" or label not in found[4].get("measures", {}):
raise _Stale
si, bi, gi, _block, seg = found
sp = ("sheets", si, "blocks", bi, "segments", gi)
ops: list[dict[str, Any]] = []
if "text_measures" not in seg:
ops.append({"op": "add", "path": ptr(*sp, "text_measures"), "value": {}})
ops.append({"op": "add", "path": ptr(*sp, "text_measures", label), "value": {"parser": opt.value}})
col = seg["measures"][label]
ti = table_index(recipe, seg.get("table"))
if ti is not None and col in (recipe["tables"][ti].get("units") or {}):
ops.append({"op": "remove", "path": ptr("tables", ti, "units", col)})
return ops, [f"分段「{seg['id']}」的「{label}」{opt.label}:表「{seg.get('table')}」的列「{col}」是文字列"]
# ==========================================================================
# ⑧ 声明隐藏行列怎么处理
# ==========================================================================
@@ -1062,7 +1129,12 @@ _OPS: dict[str, Callable[[dict, FixProposal, FixOption, str | None], tuple[list,
"remove_label": _o_remove_label, "add_label": _o_add_label, "edit_members": _o_edit_members,
"rename_title": _o_rename_title, "declare_total": _o_declare_total, "ignore_cells": _o_ignore_cells,
"declare_placeholder": _o_declare_placeholder, "declare_hidden": _o_declare_hidden, "rename_sheet": _o_rename_sheet,
"declare_subtotals": _o_declare_subtotals,
"declare_subtotals": _o_declare_subtotals, "declare_text_measure": _o_declare_text_measure,
}
#: 同一条问题的另一种出口(问题的 fix 字段只写一种,界面按 fix_ids 放按钮,所以能挂几个提议):value_not_number 的
#: 格子是整行文字时,除了声明占位符,还可以把这个指标改按文字导入
_EXTRA_BUILDERS: dict[str, Callable[[dict, Any, dict, FixAnchor], list[FixProposal]]] = {
"value_not_number": _p_declare_text_measure,
}
@@ -1096,7 +1168,15 @@ def propose_fixes(recipe: dict[str, Any], problems: list[dict[str, Any]], recipe
try:
push(builder(recipe, prob, args, FixAnchor("problem", i)))
except (KeyError, TypeError, ValueError, IndexError, AttributeError):
continue # fix_args 的形状不对(执行器的契约问题):不出提议,不拖垮 staging_out
pass # fix_args 的形状不对(执行器的契约问题):不出提议,不拖垮 staging_out
extra = _EXTRA_BUILDERS.get(_get(prob, "code") or "")
if extra is None:
continue
try:
for p in extra(recipe, prob, args, FixAnchor("problem", i)):
push(p)
except (KeyError, TypeError, ValueError, IndexError, AttributeError):
continue
fl = _facts(facts)
for i, rp in enumerate(recipe_problems or []):
try:
@@ -1125,6 +1205,8 @@ def _edit_title(p: FixProposal, opt: FixOption) -> str:
return opt.label
if p.kind == "declare_subtotals":
return f"列表「{t.get('block')}」中间「{t.get('pick')}」开头的行{opt.label}"
if p.kind == "declare_text_measure":
return f"分段「{t.get('segment')}」的「{t.get('label')}」{opt.label}"
return p.title
+3 -2
View File
@@ -696,8 +696,9 @@ def _period_column_key(col: ColumnOut) -> str | None:
def _null_eligible(ctx: _Ctx, w: _Writers, table: str, col: ColumnOut) -> bool:
"""这一列要不要考虑空值说明:值列;主键不会是空值;表内合计表的空值已由表说明(未能核对)交代。"""
value_col = col.role in ("measure", "value") and col.type in ("INTEGER", "REAL")
"""这一列要不要考虑空值说明:值列(交叉表按文字导入的指标也是,它的占位符、空格同样存成空值);主键不会是空值;
表内合计表的空值已由表说明(未能核对)交代。"""
value_col = col.role == "measure" or (col.role == "value" and col.type in ("INTEGER", "REAL"))
is_total = bool(w.kept_from or w.list_totals or w.list_subtotals)
return value_col and not is_total and col.name not in set(ctx.grains.get(table, []))
+91 -6
View File
@@ -359,6 +359,7 @@ class _Seg:
title: str | None = None
title_row: int | None = None
parser: str | None = None # dimension:hour_range / text
text: dict[str, str] = field(default_factory=dict) # measures:整行是文字的标签原文 → 解析方式
# 以下由命名阶段填
id: str = ""
table: str = ""
@@ -417,6 +418,41 @@ def _sums_above(cell: GridCell | None, r: int) -> bool:
return bool(m) and m.group(1) == m.group(3) and int(m.group(4)) == r - 1
def _text_row(grid: Grid, r: int, c1: int, c2: int, label: str) -> str | None:
"""指标行是不是整行文字(「高峰时段」每格写着「18-19」):是的话返回解析方式(全都能按时段解析的 hour_range,
否则 text),不是返回 None。轴列上除了占位符候选和空格全是文字,至少一格;有一格是数、公式、错误值、写成文字的
数就不是。标签末尾写着词表里的单位(「时段人数(人次)」)的是数字指标,不算。"""
unit = P.split_unit_suffix(label)[1]
if unit is not None and P.unit_known(unit):
return None
texts: list[str] = []
for c in range(c1, c2 + 1):
cell = grid.get(r, c)
kind = _cell_kind(cell)
if kind in ("blank", "ph"):
continue
if kind != "text":
return None
t = cell.value.strip() # type: ignore[union-attr]
if P.looks_numeric_text(t) or P.thousands_number(t) is not None:
return None
texts.append(t)
if not texts:
return None
return "hour_range" if all(P.hour_range(t) is not None for t in texts) else "text"
def _value_failure(sheet: str, r: int, label: str, why: str, cells: list[tuple[int, str]]) -> _Failure:
"""数值区一行里同一种不合格的格合成一条(一行 30 格不再报 30 条)。给模型的那条只有坐标,没有格子里的字。"""
if len(cells) == 1:
c, shown = cells[0]
return _Failure(f"工作表「{sheet}」:{_a1(r, c)} 的「{shown}」{why}", f"工作表「{sheet}」:{_a1(r, c)} {why}")
examples = "、".join(f"{_a1(r, c)}「{shown}」" for c, shown in cells[:3])
where = _a1_list([(r, c) for c, _ in cells], 3)
return _Failure(f"工作表「{sheet}」:第 {r} 行「{label}」有 {len(cells)} 个单元格{why}(如 {examples})",
f"工作表「{sheet}」:第 {r} 行有 {len(cells)} 个单元格{why}({where})")
def _analyze_crosstab(grid: Grid, axis_row: int, date_cols: list[int]) -> _Cross:
sheet = grid.sheet
c1, c2 = min(date_cols), max(date_cols)
@@ -544,8 +580,13 @@ def _analyze_crosstab(grid: Grid, axis_row: int, date_cols: list[int]) -> _Cross
# ---- 数值区:类型、占位符、空格、公式(P2-SPEC 6.1 第 9 条)
ints = True
for seg in x.segments:
for r in seg.rows:
for r, label in zip(seg.rows, seg.labels):
# 指标行整行是文字(「高峰时段」每格写着「18-19」):按文字指标起草,不当数字报错;要人在问题里确认
text = _text_row(grid, r, c1, c2, label) if seg.role == "measures" else None
if text is not None:
seg.text[label] = text
row_vals: list[int | float | None] = []
bad: dict[str, list[tuple[int, str]]] = {}
for c in range(c1, c2 + 1):
cell = grid.get(r, c)
kind = _cell_kind(cell)
@@ -565,13 +606,16 @@ def _analyze_crosstab(grid: Grid, axis_row: int, date_cols: list[int]) -> _Cross
entry[1] += 1
entry[2].append((r, c))
x.nonnumeric[cell.value.strip()] = x.nonnumeric.get(cell.value.strip(), 0) + 1
elif kind == "text" and text is not None:
continue
elif kind in ("text", "error", "other"):
shown = _shown(cell.value)
if isinstance(cell.value, str):
x.nonnumeric[shown] = x.nonnumeric.get(shown, 0) + 1
why = "是错误值" if kind == "error" else "不是数字,也不像表示无数据的符号"
fail(_Failure(f"工作表「{sheet}」:{_a1(r, c)} 的「{shown}」{why}",
f"工作表「{sheet}」:{_a1(r, c)} {why}"))
bad.setdefault("是错误值" if kind == "error" else "不是数字,也不像表示无数据的符号",
[]).append((c, shown))
for why, cells in bad.items():
fail(_value_failure(sheet, r, label, why, cells))
x.values[r] = row_vals
x.value_type = "INTEGER" if ints else "REAL"
@@ -1436,8 +1480,11 @@ def _prefer_prefix(period: list[tuple[int, int, str]]) -> str | None:
def _segment_dict(s: _Seg) -> dict[str, Any]:
if s.role == "measures":
return {"id": s.id, "role": "measures", "table": s.table, "locate": {"by": "labels"},
"labels": {"expect": list(s.labels)}, "measures": dict(s.columns)}
out = {"id": s.id, "role": "measures", "table": s.table, "locate": {"by": "labels"},
"labels": {"expect": list(s.labels)}, "measures": dict(s.columns)}
if s.text:
out["text_measures"] = {label: {"parser": p} for label, p in s.text.items()}
return out
if s.role == "dimension":
out: dict[str, Any] = {
"id": s.id, "role": "dimension", "table": s.table,
@@ -2427,6 +2474,18 @@ def _unit_cards(labels: list[tuple[str, str]], sheet_id: str) -> list[Card]:
return out
def _label_row(grid: Grid | None, loc: _CrossLoc | None, seg_id: str, label: str) -> int | None:
"""分段里标签是 label 的那一行(按 match_key 比);定位不到时 None。"""
if grid is None or loc is None:
return None
key = P.match_key(label)
for r in loc.seg_rows.get(seg_id, []):
cell = grid.get(r, loc.label_col)
if cell is not None and isinstance(cell.value, str) and P.match_key(cell.value) == key:
return r
return None
def _crosstab_cards(si: int, bi: int, sheet: Any, block: CrosstabBlock, loc: _CrossLoc | None, grid: Grid | None,
name: str, recipe: dict[str, Any], parsed: Recipe, sheet_facts: list[Fact],
relation_question: Callable[[Fact], Question], ext: Extraction | None,
@@ -2539,6 +2598,32 @@ def _crosstab_cards(si: int, bi: int, sheet: Any, block: CrosstabBlock, loc: _Cr
cards.extend(ph_cards)
questions.extend(reversed(ph_qs))
# 6b 按文字导入的指标:要人确认。「不是」去掉这一条,这一行回到按块的数值类型导入,试运行会拒收
for gi, s in enumerate(block.segments):
if not isinstance(s, MeasuresSegment):
continue
for label, tm in s.text_measures.items():
r = _label_row(grid, loc, s.id, label)
spots: list[tuple[int, int]] = []
if r is not None and loc is not None and grid is not None:
spots = [(r, loc.label_col)] + [(r, cc) for cc in range(loc.c1, loc.c2 + 1)
if grid.get(r, cc) is not None]
where = f"第 {r} 行" if r is not None else f"分段「{s.id}」中"
how = "时段" if tm.parser == "hour_range" else "文字"
col = s.measures.get(label, "")
qid = _qid(f"q_text_measure:{s.id}:{label}", block.id, used)
reason = (f"这一行每格写的都是{'时段(如「18-19」)' if tm.parser == 'hour_range' else '文字'},不是数字:"
f"确认后列「{col}」存成文字列"
f"{'(时段存规范写法,「18–19」存成「18-19」)' if tm.parser == 'hour_range' else '(存原文)'},"
"不参与数值核对;如果它应该是数字,这份文件无法按当前配方导入")
cards.append(Card(f"text_measure:{s.id}:{label}", f"{where}「{label}」按{how}导入", reason,
_coords(name, spots), qid))
path = ["sheets", si, "blocks", bi, "segments", gi, "text_measures", label]
questions.append(Question(qid, f"「{label}」这一行按{how}导入吗",
[{"value": "text", "label": f"是,按{how}导入", "needs_reason": False},
{"value": "reject", "label": "不是,不导入", "needs_reason": False}],
None, {"text": [], "reject": [{"op": "remove", "path": _ptr(*path)}]}))
# 7 合计段
for s in block.segments:
if not isinstance(s, DerivedSegment):
+13 -2
View File
@@ -154,6 +154,14 @@ class DimSpec(_M):
derive: dict[str, Literal["start", "end"]] = Field(default_factory=dict)
class TextMeasure(_M):
"""按文字导入的指标:这一行每格写的是文字(如「高峰时段」写着「18-19」),落成文字列。空格、公式、占位符照块的
取值规则;块的数值类型、千分位规则对它不适用。"""
#: text:存原文(去首尾空白);hour_range:按时段解析,存规范写法(「18–19」存成「18-19」),解析不了的格是问题
parser: Literal["text", "hour_range"] = "text"
class MeasuresSegment(_M):
"""每一行是一个指标:落成宽表,一个指标一列。"""
@@ -166,6 +174,8 @@ class MeasuresSegment(_M):
#: derive_tables 按原文查,查不到报 measures_keys_mismatch,绝不产出空列名)。文件里标签的写法变化由
#: 执行器按 match_key 认,与这里无关
measures: dict[str, str]
#: 按文字导入的指标:标签原文(与 measures 的键逐字相同)→ 解析方式。不在这里的指标按块的数值类型导入
text_measures: dict[str, TextMeasure] = Field(default_factory=dict)
class DimensionSegment(_M):
@@ -479,7 +489,7 @@ class Problem:
#: 修复按钮的种类(P3-SPEC 3.2 ①–⑨;期 5 加 declare_subtotals:列表中间像小计的行改作小计核对)。Problem.fix、
#: FixProposal.kind、确认项 fix:<目标键> 的 kind 都取自这里
FIX_KINDS = ("remove_label", "add_label", "edit_members", "rename_title", "declare_total", "ignore_cells",
"declare_placeholder", "declare_hidden", "rename_sheet", "declare_subtotals")
"declare_placeholder", "declare_hidden", "rename_sheet", "declare_subtotals", "declare_text_measure")
@dataclass
@@ -1135,7 +1145,8 @@ def derive_tables(recipe: Recipe) -> tuple[dict[str, list[ColumnOut]], list[Reci
problems.append(RecipeProblem(f"{path}/measures", "measures_keys_mismatch",
f"分段「{seg.id}」:" + ";".join(parts)))
cols = [ColumnOut(axis_name, "TEXT", role="axis")]
cols += [ColumnOut(seg.measures.get(label, ""), vtype, header=label, role="measure")
cols += [ColumnOut(seg.measures.get(label, ""), "TEXT" if label in seg.text_measures else vtype,
header=label, role="measure")
for label in seg.labels.expect]
put(seg.table, cols, path)
elif isinstance(seg, DimensionSegment):
+3 -3
View File
@@ -488,9 +488,9 @@ def test_closed_value_sets():
"header", "row_label", "section_title", "total_word", "axis", "after_title", "outside_text")
assert _literal(T.EditResult, "kind") == ("fix", "selection")
assert _literal(T.ChangeClass, "change") == ("same", "compatible", "retire", "semantic")
assert T.FIX_KINDS[-1] == "declare_subtotals" # 期 5 加在末尾
assert T.FIX_KINDS[:-1] == ("remove_label", "add_label", "edit_members", "rename_title", "declare_total",
"ignore_cells", "declare_placeholder", "declare_hidden", "rename_sheet")
assert T.FIX_KINDS[-2:] == ("declare_subtotals", "declare_text_measure") # 期 5、文字指标依次加在末尾
assert T.FIX_KINDS[:-2] == ("remove_label", "add_label", "edit_members", "rename_title", "declare_total",
"ignore_cells", "declare_placeholder", "declare_hidden", "rename_sheet")
assert T.SELECTION_AS == ("list", "crosstab", "segment", "derived", "section_title", "ignore_rows",
"ignore_columns", "ignore_outside")
assert T.ACCUMULATE_ACTIONS == ("replace", "first", "append", "replace_period", "restart", "rejected")
+18 -1
View File
@@ -362,6 +362,20 @@ def c_value_not_number(mp):
return _run(_flow(_set(C11=f"约{N}")), FLOW), [f"约{N}", N]
def _text_measure(parser: str) -> dict[str, Any]:
data = recipe_dict()
data["sheets"][0]["blocks"][0]["segments"][0]["text_measures"] = {"分区甲(人次)": {"parser": parser}}
return data
def c_value_not_text(mp):
return _run(_flow(_set(C6=N)), _text_measure("text")), [N]
def c_value_not_hour_range(mp):
return _run(_flow(_set(C6=f"约{N}", D6="18-19", E6="19-20")), _text_measure("hour_range")), [f"约{N}", N]
def c_value_not_date(mp):
return _run(sales_book(edit=lambda ws: ws.cell(4, 3, f"{N}日")), LIST), [f"{N}日", N]
@@ -438,7 +452,10 @@ NEXT_STEPS = {
"value_text_number": "在配方面板的「文本写的数字」中选择「千分位写法按数字保存」",
"value_blank": "在配方面板的「空格」中选择「存为空值」",
"value_formula": "在配方面板的「公式」中选择「按保存值导入」",
"value_not_number": "在配方面板中添加占位符",
# 「约987654」含数字、不是整行文字:既不提示加占位符,也不提示改指标类型(另见 test_recipe_text_measures)
"value_not_number": "数值区只能是数字或表示无数据的符号",
"value_not_text": "在配方面板中把它的「指标类型」改为「数字」",
"value_not_hour_range": "在配方面板中把它的「指标类型」改为「文字」",
"value_not_integer": "在配方面板中把「类型」改为「小数」",
"value_not_date": "在配方面板中把这一列的「类型」改为「文字」",
"sheet_unexpected": "在配方面板的「配方没有列出的其他工作表」中选择「记入回执,需确认」",
+395
View File
@@ -0,0 +1,395 @@
"""交叉表里按文字导入的指标(MeasuresSegment.text_measures):契约、静态校验、执行、规则起草、修复按钮、确认项、说明。
起因:一份客流导出的顶部指标块里夹着「高峰时段」一行,每格写的是当天客流最多的时段(「18-19」),不是数字。
以前起草逐格报「不是数字」(一行 30 条),提示去加占位符;照做的话「18-19」整格静默存成空值、干跑照样通过;
从分段里去掉这一行再「忽略这些行」又不生效(按标签定位的分段认领连续的数据行)。
夹具在这里用 openpyxl 现造,名字全是假名(分区甲 / 分区乙):B2 统计期、B3 标题、第 4 行日期、第 5–9 行无标题的
指标块(全日、分区甲、分区乙、高峰时段、时段人数)、空行、日间和夜间两段时段客流,没有合计行。第 4 天的时段全是「·」,
那天的高峰时段、时段人数也是「·」;第 2 天的高峰时段用 en dash 写(「18–19」)。
"""
from __future__ import annotations
import copy
import datetime as dt
import io
import random
import sqlite3
from typing import Any, Callable
import pytest
from openpyxl import Workbook
from openpyxl.utils import get_column_letter
from app.data import recipe_engine, xlsx_cells, xlsx_scan
from app.data import recipe_fixes as F
from app.data import recipe_suggest as S
from app.data.recipe import apply_patch, canonical_recipe, validate_recipe
from app.data.recipe_confirm import ConfirmContext, confirm_items, table_changes
from app.data.recipe_types import Recipe, derive_tables
SHEET = "客流汇总"
START = dt.date(2026, 7, 1)
DAYS = 8 # 至少 tabular.DATE_HEADER_MIN(7)格日期才按交叉表起草
DAY_HOURS = (8, 9, 10, 11)
NIGHT_HOURS = (18, 19, 20)
GAP = 3 # 第 4 天:时段全是「·」
PEAK, PEAK_N = "高峰时段", "时段人数(人次)"
def peak_book(edit: Callable[[Any], None] | None = None) -> tuple[bytes, str]:
rnd = random.Random(7)
wb = Workbook()
ws = wb.active
ws.title = SHEET
end = START + dt.timedelta(days=DAYS - 1)
last = get_column_letter(2 + DAYS)
ws["B2"] = f"统计时间范围:{START.year}年{START.month}月{START.day}日至{end.year}年{end.month}月{end.day}日"
ws.merge_cells(f"B2:{last}2")
ws["B3"] = "某地区客流汇总表"
ws.merge_cells(f"B3:{last}3")
for j in range(DAYS):
d = START + dt.timedelta(days=j)
ws.cell(4, 3 + j, f"{d.month}月{d.day}日")
hours: dict[int, list[Any]] = {h: [rnd.randint(100, 900) for _ in range(DAYS)] for h in DAY_HOURS + NIGHT_HOURS}
zone_a = [rnd.randint(3000, 9000) for _ in range(DAYS)]
zone_b = [rnd.randint(3000, 9000) for _ in range(DAYS)]
peak: list[Any] = []
peak_n: list[Any] = []
for j in range(DAYS):
if j == GAP:
for h in hours:
hours[h][j] = "·"
peak.append("·")
peak_n.append("·")
continue
h = max(hours, key=lambda k: hours[k][j])
peak.append(f"{h}–{h + 1}" if j == 1 else f"{h}-{h + 1}")
peak_n.append(hours[h][j])
rows: list[tuple[str, list[Any]]] = [
("全日客流(人次)", [a + b for a, b in zip(zone_a, zone_b)]), ("分区甲(人次)", zone_a),
("分区乙(人次)", zone_b), (PEAK, peak), (PEAK_N, peak_n)]
r = 5
for label, values in rows:
ws.cell(r, 2, label)
for j, v in enumerate(values):
ws.cell(r, 3 + j, v)
r += 1
ws.merge_cells(f"B{r}:{last}{r}") # 第 10 行空行
r += 1
for title, hs in (("日间时段客流(人次)", DAY_HOURS), ("夜间时段客流(人次)", NIGHT_HOURS)):
ws.cell(r, 2, title)
ws.merge_cells(f"B{r}:{last}{r}")
r += 1
for h in hs:
ws.cell(r, 2, f"{h}-{h + 1}")
for j, v in enumerate(hours[h]):
ws.cell(r, 3 + j, v)
r += 1
if edit is not None:
edit(ws)
bio = io.BytesIO()
wb.save(bio)
return bio.getvalue(), f"月报导出_{START.isoformat()}_{end.isoformat()}.xlsx"
class Book:
def __init__(self, edit: Callable[[Any], None] | None = None):
self.raw, self.filename = peak_book(edit)
self.scan = xlsx_scan.scan(self.raw)
self.grids = {s.name: xlsx_cells.read_grid(self.raw, self.scan, s.name, max_rows=S.DRAFT_MAX_ROWS)
for s in self.scan.sheets if s.state == "visible" and s.bounds is not None and s.nonempty}
def run(self, recipe: dict[str, Any] | Recipe, db: str | None = None):
parsed = recipe if isinstance(recipe, Recipe) else Recipe.model_validate(recipe)
return recipe_engine.execute(parsed, self.raw, self.filename, self.scan, db)
def draft(self):
return S.draft(self.scan, self.grids, self.filename,
validate=lambda r, f, o: validate_recipe(r, facts=f, origin=o), dry_run=self.run)
@pytest.fixture(scope="module")
def drafted():
book = Book()
d = book.draft()
return book, d
def measures_seg(recipe: dict[str, Any]) -> dict[str, Any]:
return next(s for s in recipe["sheets"][0]["blocks"][0]["segments"] if s["role"] == "measures")
def full(recipe: dict[str, Any]) -> dict[str, Any]:
return Recipe.model_validate(recipe).model_dump(mode="json")
def codes(problems: list[Any]) -> list[str]:
return [p.code for p in problems]
# ==========================================================================
# 规则起草
# ==========================================================================
def test_rule_draft_turns_the_text_row_into_a_text_measure(drafted):
_book, d = drafted
assert d.complete, d.failures
assert d.failures == []
seg = measures_seg(d.recipe)
assert seg["text_measures"] == {PEAK: {"parser": "hour_range"}}
assert PEAK in seg["labels"]["expect"] and PEAK_N in seg["labels"]["expect"]
# 文字列不写单位;「时段人数(人次)」是数字指标,单位照常拆
units = next(t["units"] for t in d.recipe["tables"] if t["name"] == seg["table"])
assert PEAK not in units and units.get("时段人数") == "人次"
def test_rule_draft_asks_to_confirm_the_text_measure(drafted):
_book, d = drafted
seg = measures_seg(d.recipe)
card = next(c for c in d.cards if c.id == f"text_measure:{seg['id']}:{PEAK}")
q = next(q for q in d.questions if q.id == card.question)
assert card.title == f"第 8 行「{PEAK}」按时段导入"
assert f"{SHEET}!B8" in card.cells and f"{SHEET}!C8" in card.cells
assert [o["value"] for o in q.options] == ["text", "reject"]
assert q.effects["text"] == []
# 答「不是」:去掉这一条,这一行回到按数字导入,干跑拒收(不会静默存成空值)
rejected = apply_patch(full(d.recipe), q.effects["reject"])
assert PEAK not in measures_seg(rejected)["text_measures"]
book = Book()
ex = book.run(rejected)
assert not ex.ok and "value_not_number" in codes(ex.problems)
def test_rule_draft_merges_bad_cells_of_one_row_into_one_failure():
# 时段人数一行混进两格文字:标签写着单位,是数字指标,照旧起草失败,但一行只报一条,给模型的那条没有格子里的字
d = Book(lambda ws: (ws.cell(9, 3, "约3000"), ws.cell(9, 4, "约4000"))).draft()
assert not d.complete
lines = [f for f in d.failures if f.startswith(f"工作表「{SHEET}」:") and ("第 9 行" in f or "C9" in f)]
assert lines == [f"工作表「{SHEET}」:第 9 行「{PEAK_N}」有 2 个单元格不是数字,也不像表示无数据的符号"
"(如 C9「约3000」、D9「约4000」)"]
assert not any("约3000" in f for f in d.failures_for_model)
# 干跑另有一条按分段合并的,下一步不再叫人把「约3000」加成占位符
dry = next(f for f in d.failures if "的分段" in f and "C9" in f)
assert "添加占位符" not in dry and "请检查原表" in dry
assert PEAK not in str(d.failures) # 高峰时段照样按文字指标起草,不报错
single = Book(lambda ws: ws.cell(9, 3, "约3000")).draft()
assert f"工作表「{SHEET}」:C9 的「约3000」不是数字,也不像表示无数据的符号" in single.failures
def test_text_measure_lands_as_a_text_column(drafted, tmp_path):
book, d = drafted
db = str(tmp_path / "t.db")
ex = book.run(d.recipe, db)
assert ex.ok, [(p.code, p.message) for p in ex.problems]
seg = measures_seg(d.recipe)
table = next(t for t in ex.tables if t.name == seg["table"])
types = {c.name: c.type for c in table.columns}
assert types["高峰时段"] == "TEXT" and types["时段人数"] == "INTEGER" and types["全日客流"] == "INTEGER"
con = sqlite3.connect(db)
try:
decl = {row[1]: row[2] for row in con.execute(f'PRAGMA table_info("{seg["table"]}")')}
rows = con.execute(f'SELECT "日期", "高峰时段", "时段人数" FROM "{seg["table"]}" ORDER BY "日期"').fetchall()
hour_rows = con.execute('SELECT COUNT(*) FROM "时段客流"').fetchone()[0]
finally:
con.close()
assert decl["高峰时段"] == "TEXT"
assert len(rows) == DAYS and hour_rows == DAYS * (len(DAY_HOURS) + len(NIGHT_HOURS))
# en dash 写的存规范写法;整天没有时段数的那天两列都是空值
assert rows[1][1] is not None and "–" not in rows[1][1] and rows[1][1].count("-") == 1
assert rows[GAP][1:] == (None, None)
assert all(isinstance(r[2], int) for i, r in enumerate(rows) if i != GAP)
# 高峰时段能和时段客流按规范写法对上
con = sqlite3.connect(db)
try:
hit = con.execute(f'SELECT COUNT(*) FROM "{seg["table"]}" d JOIN "时段客流" h ON h."日期" = d."日期" '
'AND h."时段" = d."高峰时段" AND h."客流" = d."时段人数"').fetchone()[0]
finally:
con.close()
assert hit == DAYS - 1
assert any(c.column == "高峰时段" and "–" in c.raw for c in ex.canonicalized)
# ==========================================================================
# 执行器:文字指标的取值、报错
# ==========================================================================
def test_number_in_a_text_measure_is_value_not_text(drafted):
_, d = drafted
book = Book(lambda ws: ws.cell(8, 3, 18))
ex = book.run(d.recipe)
p = next(p for p in ex.problems if p.code == "value_not_text")
assert p.category == "structure" and f"{SHEET}!C8" in p.cells
assert "「指标类型」改为「数字」" in p.message
assert "18" not in p.model_message.replace(SHEET, "").replace("C8", "")
def test_unparsable_hour_range_and_plain_text_parser(drafted):
_, d = drafted
book = Book(lambda ws: ws.cell(8, 3, "傍晚"))
ex = book.run(d.recipe)
p = next(p for p in ex.problems if p.code == "value_not_hour_range")
assert "「指标类型」改为「文字」" in p.message and "傍晚" not in p.model_message
as_text = full(d.recipe)
measures_seg(as_text)["text_measures"][PEAK] = {"parser": "text"}
ex2 = book.run(as_text)
assert ex2.ok, [(p.code, p.message) for p in ex2.problems]
# ==========================================================================
# 没声明成文字指标时:报错的下一步和修复按钮
# ==========================================================================
def _without_text_measure(d: Any) -> dict[str, Any]:
r = full(d.recipe)
measures_seg(r)["text_measures"] = {}
return r
def test_not_number_on_a_text_row_points_at_measure_kind_not_placeholder(drafted):
book, d = drafted
ex = book.run(_without_text_measure(d))
p = next(p for p in ex.problems if p.code == "value_not_number")
assert f"「{PEAK}」这一行写的都是文字" in p.message
assert "「指标类型」改为「文字」或「时段」" in p.message
assert "添加占位符" not in p.message
seg = measures_seg(d.recipe)
assert p.fix_args["texts"] == []
assert p.fix_args["text_rows"] == [{"segment": seg["id"], "label": PEAK, "cell": f"{SHEET}!B8",
"parsers": ["hour_range", "text"]}]
def test_not_number_tail_depends_on_what_the_cells_are(drafted):
_, d = drafted
# 不含数字的短符号:照旧提示加占位符
ex = Book(lambda ws: ws.cell(5, 3, "—")).run(d.recipe)
p = next(p for p in ex.problems if p.code == "value_not_number")
assert p.message.endswith("如果它表示无数据,请在配方面板中添加占位符")
# 含数字、又不是整行文字:不再提示加占位符
ex = Book(lambda ws: ws.cell(5, 3, "约3000")).run(d.recipe)
p = next(p for p in ex.problems if p.code == "value_not_number")
assert "占位符" not in p.message.split("。", 1)[1].replace("表示无数据的符号", "")
assert "请检查原表" in p.message and "text_rows" not in p.fix_args
# 标签写着单位的指标整行是文字:是数字指标,不提示改指标类型
ex = Book(lambda ws: [ws.cell(9, c, "18-19") for c in range(3, 3 + DAYS)]).run(d.recipe)
p = next(p for p in ex.problems if p.code == "value_not_number")
assert "指标类型" not in p.message and "text_rows" not in p.fix_args
def test_fix_button_declares_the_text_measure(drafted):
book, d = drafted
work = _without_text_measure(d)
ex = book.run(work)
problems = [p.__dict__ for p in ex.problems]
props = F.propose_fixes(work, problems, [], d.facts)
p = next(x for x in props if x.kind == "declare_text_measure")
assert [o.value for o in p.options] == ["hour_range", "text"]
assert all(o.breaking for o in p.options)
assert p.anchor.index == next(i for i, x in enumerate(problems) if x["code"] == "value_not_number")
res = F.fix_ops(work, p, "hour_range", None)
assert res.ok and "按时段导入" in res.title
new = apply_patch(work, res.ops)
assert measures_seg(new)["text_measures"] == {PEAK: {"parser": "hour_range"}}
parsed, probs = validate_recipe(new, facts=d.facts, origin="manual")
assert probs == [] and parsed is not None
assert book.run(parsed).ok
# 已经按文字导入的不再提议
assert not [x for x in F.propose_fixes(new, problems, [], d.facts) if x.kind == "declare_text_measure"]
def test_remove_label_also_drops_the_text_measure(drafted):
_, d = drafted
work = full(d.recipe)
seg = measures_seg(work)
prob = {"code": "label_missing", "fix": "remove_label", "cells": [],
"fix_args": {"segment": seg["id"], "labels": [PEAK], "role": "measures"}}
p = next(x for x in F.propose_fixes(work, [prob], [], d.facts) if x.kind == "remove_label")
new = apply_patch(work, F.fix_ops(work, p, p.options[0].value, None).ops)
assert PEAK not in measures_seg(new)["text_measures"]
_parsed, probs = validate_recipe(new, facts=d.facts, origin="manual")
assert "text_measure_unknown" not in codes(probs)
# ==========================================================================
# 静态校验、契约
# ==========================================================================
def test_static_checks_on_text_measures(drafted):
_, d = drafted
r = full(d.recipe)
seg = measures_seg(r)
seg["text_measures"]["不存在的指标"] = {"parser": "text"}
seg["text_measures"][PEAK_N] = {"parser": "text"}
table = next(t for t in r["tables"] if t["name"] == seg["table"])
table["units"]["高峰时段"] = "人次"
_parsed, probs = validate_recipe(r, facts=d.facts, origin="manual")
got = {(p.code, p.path) for p in probs}
sp = "/sheets/0/blocks/0/segments/0"
assert ("text_measure_unknown", f"{sp}/text_measures/不存在的指标") in got
assert ("text_measure_unit", f"{sp}/text_measures/{PEAK_N}") in got
assert any(code == "unit_on_text" and path.endswith("/units/高峰时段") for code, path in got)
def test_placeholder_with_digits_is_rejected_except_on_replay_or_when_carried(drafted):
_, d = drafted
r = full(d.recipe)
measures_seg(r)["text_measures"] = {}
r["sheets"][0]["blocks"][0]["values"]["placeholders"].append({"text": "18-19", "meaning": "无数据"})
_parsed, probs = validate_recipe(r, facts=d.facts, origin="manual")
p = next(p for p in probs if p.code == "placeholder_digits")
assert p.path == "/sheets/0/blocks/0/values/placeholders/1/text"
assert "「指标类型」改为「文字」或「时段」" in p.message
assert "placeholder_digits" not in codes(validate_recipe(r, origin="replay")[1])
base = Recipe.model_validate(copy.deepcopy(r))
assert "placeholder_digits" not in codes(validate_recipe(r, facts=d.facts, origin="manual", base=base)[1])
def test_contract_text_measures_are_text_columns_and_hash_compatible(drafted):
_, d = drafted
parsed = Recipe.model_validate(d.recipe)
cols, problems = derive_tables(parsed)
seg = measures_seg(d.recipe)
assert problems == []
assert {c.name: c.type for c in cols[seg["table"]]}["高峰时段"] == "TEXT"
# 默认值(没有文字指标)不进规范形式:已存配方的哈希和构建 id 不变
plain = full(d.recipe)
measures_seg(plain)["text_measures"] = {}
assert "text_measures" not in str(canonical_recipe(Recipe.model_validate(plain)))
# ==========================================================================
# 确认项、前后对照、说明
# ==========================================================================
def test_confirm_item_and_breaking_changes(drafted):
book, d = drafted
ex = book.run(d.recipe)
seg = measures_seg(d.recipe)
items = {i.id: i for i in confirm_items(ConfirmContext(kind="first", recipe=d.recipe, extraction=ex, checks=[]))}
item = items[f"text_measure:{seg['id']}:{PEAK}"]
assert "按时段导入" in item.label and "文字列" in item.label
as_text = full(d.recipe)
measures_seg(as_text)["text_measures"][PEAK] = {"parser": "text"}
kinds = {(c.kind, c.column) for c in table_changes(d.recipe, as_text)}
assert ("store", "高峰时段") in kinds
plain = full(d.recipe)
measures_seg(plain)["text_measures"] = {}
assert ("type", "高峰时段") in {(c.kind, c.column) for c in table_changes(d.recipe, plain)}
def test_column_note_explains_nulls_of_the_text_measure(drafted, tmp_path):
from app.data.recipe_checks import run_checks
from app.data.recipe_notes import build_notes
book, d = drafted
db = str(tmp_path / "n.db")
ex = book.run(d.recipe, db)
parsed = Recipe.model_validate(d.recipe)
notes = build_notes(parsed, ex, run_checks(db, ex, parsed), [])
seg = measures_seg(d.recipe)
assert "空值" in notes.tables[seg["table"]].columns.get("高峰时段", "")
+4
View File
@@ -1627,6 +1627,8 @@ export const RECIPE_TEXT = {
derivedSegment: (id: string) => `合计段「${id}」`,
tableName: '表名',
columnName: '列名',
/** measures 段每个指标的「指标类型」:数字按块的「类型」导入;文字、时段落成文字列(执行器的报错里写的就是这个字段名) */
measureKind: '指标类型',
unit: '单位',
noUnit: '无单位',
grain: '主键',
@@ -1909,6 +1911,7 @@ export const FIX_KIND_LABEL: Record<string, string> = {
rename_title: '改分段标题',
declare_total: '改作合计核对',
declare_subtotals: '改作小计核对',
declare_text_measure: '按文字导入这一行',
ignore_cells: '按文字忽略',
declare_placeholder: '声明占位符',
declare_hidden: '设置隐藏行处理',
@@ -2152,6 +2155,7 @@ export const RECIPE_ORIGIN_LABEL: Record<string, string> = {
export const RECIPE_CHOICE_LABEL = {
locateBy: { labels: '按标签', section_title: '按分段标题' } as Record<string, string>,
dimParser: { hour_range: '时段(如 7-8)', text: '文字' } as Record<string, string>,
measureKind: { '': '数字', hour_range: '时段', text: '文字' } as Record<string, string>,
valueType: { INTEGER: '整数', REAL: '小数' } as Record<string, string>,
blankCross: { reject: '拒收', null: '存为空值(需确认)' } as Record<string, string>,
blankList: { null: '存为空值', reject: '拒收' } as Record<string, string>,
+27 -1
View File
@@ -130,8 +130,9 @@ export function tableColumns(recipe: Recipe | null): Map<string, RecipeColumn[]>
for (const seg of arr(block.segments)) {
if (!isObj(seg)) continue
if (seg.role === 'measures') {
const text = isObj(seg.text_measures) ? seg.text_measures : {}
put(seg.table, [axis, ...arr(seg.labels?.expect).map((label: string) => ({
name: String(seg.measures?.[label] ?? ''), type: vtype, role: 'measure', header: label,
name: String(seg.measures?.[label] ?? ''), type: isObj(text[label]) ? 'TEXT' : vtype, role: 'measure', header: label,
}))])
} else if (seg.role === 'dimension') {
put(seg.table, [
@@ -568,6 +569,23 @@ function DeriveNames({ path, derive, onRename }: { path: string; derive: unknown
)
}
/**
* 指标改按文字(text / hour_range)或改回数字(空串)。文字列不能有单位:改成文字时去掉这一列的单位,改回数字时按标签
* 末尾的括号补回(与加标签时一致)
*/
function setMeasureKind(r: Recipe, x: any, label: string, kind: string, units: string[]) {
const col = x.measures?.[label]
const table = arr(r.tables).find((t) => t?.name === x.table)
if (kind) {
x.text_measures = { ...(isObj(x.text_measures) ? x.text_measures : {}), [label]: { parser: kind } }
if (table && col && isObj(table.units)) delete table.units[col]
return
}
if (isObj(x.text_measures)) delete x.text_measures[label]
const [, unit] = splitUnit(label, units)
if (unit && table && col) table.units = { ...(isObj(table.units) ? table.units : {}), [col]: unit }
}
function SegmentEditor({ s, b, g, seg, edit, candidates, labelOptions, units }: {
s: number; b: number; g: number; seg: any; edit: Edit
candidates: Record<string, string[]>; labelOptions: string[]; units: string[]
@@ -596,6 +614,7 @@ function SegmentEditor({ s, b, g, seg, edit, candidates, labelOptions, units }:
if (x.role === 'measures' && isObj(x.measures)) {
const col = x.measures[label]
delete x.measures[label]
if (isObj(x.text_measures)) delete x.text_measures[label]
const table = arr(r.tables).find((t) => t?.name === x.table)
if (table && col) {
if (isObj(table.units)) delete table.units[col]
@@ -644,7 +663,14 @@ function SegmentEditor({ s, b, g, seg, edit, candidates, labelOptions, units }:
if (old) renameColumn(r, x.table, old, v)
else x.measures = { ...(isObj(x.measures) ? x.measures : {}), [label]: v }
})} />
<select className="field !w-20" data-field={`${base}/text_measures/${ptr(label)}`}
value={String(seg.text_measures?.[label]?.parser ?? '')}
aria-label={`${RECIPE_TEXT.measureKind}:${label}`} title={RECIPE_TEXT.measureKind}
onChange={(e) => edit((r) => { setMeasureKind(r, at(r), label, e.target.value, units) })}>
{Object.entries(RECIPE_CHOICE_LABEL.measureKind).map(([v, l]) => <option key={v} value={v}>{l}</option>)}
</select>
<FieldProblems path={`${base}/measures/${ptr(label)}`} />
<FieldProblems path={`${base}/text_measures/${ptr(label)}`} />
</div>
))}
</div>
+3 -2
View File
@@ -800,10 +800,11 @@ export interface CurrentRecipe {
// 和接口的 JSON 同名(Selection 的 as 除外:后端 dataclass 叫 as_,JSON 里是 as)
// ---------------------------------------------------------------------------
/** 修复按钮的九种(契约 FIX_KINDS) */
/** 修复按钮的种类(契约 FIX_KINDS) */
export type FixKind =
| 'remove_label' | 'add_label' | 'edit_members' | 'rename_title' | 'declare_total' | 'ignore_cells'
| 'declare_placeholder' | 'declare_hidden' | 'rename_sheet' | (string & {})
| 'declare_placeholder' | 'declare_hidden' | 'rename_sheet' | 'declare_subtotals' | 'declare_text_measure'
| (string & {})
/** 修复提议的一个封闭选项。效果由服务端算:界面只发 fix_id、value 和理由 */
export interface FixOption {
+24 -3
View File
@@ -2528,13 +2528,34 @@ await section('数据 · 表格:按配方导入', async () => {
check('勾选「没有小计行」:发 PUT,no_subtotals 为 true、没有配小计', noSub?.no_subtotals === true && !noSub?.subtotals,
JSON.stringify(noSub ?? {}))
await page.waitForTimeout(300)
// 指标类型:整行是文字的指标(如「高峰时段」写着「18-19」)改按时段 / 文字导入,落成文字列、去掉单位;改回数字时按标签补回单位
const kindSel = panel.locator('[data-field="/sheets/0/blocks/0/segments/0/text_measures/分区乙(人次)"]')
const kindOptions = await kindSel.locator('option').evaluateAll((os) => os.map((o) => `${o.value}:${o.textContent}`))
check('指标类型是下拉框:数字、时段、文字', JSON.stringify(kindOptions) === JSON.stringify([':数字', 'hour_range:时段', 'text:文字']),
kindOptions.join(','))
replies.recipe.push(echo())
await kindSel.selectOption('hour_range')
await until(async () => puts().length === 7, 4000)
const k1 = puts()[6]?.body?.recipe
check('改成「时段」:发 PUT,text_measures 写上 hour_range,这一列的单位去掉',
k1?.sheets?.[0]?.blocks?.[0]?.segments?.[0]?.text_measures?.['分区乙(人次)']?.parser === 'hour_range'
&& !('分区乙' in (k1?.tables?.[0]?.units ?? {})), JSON.stringify(k1?.tables?.[0]?.units ?? {}))
await page.waitForTimeout(300)
replies.recipe.push(echo())
await kindSel.selectOption('')
await until(async () => puts().length === 8, 4000)
const k2 = puts()[7]?.body?.recipe
check('改回「数字」:发 PUT,text_measures 去掉这一项,单位按标签补回「人次」',
!('分区乙(人次)' in (k2?.sheets?.[0]?.blocks?.[0]?.segments?.[0]?.text_measures ?? {}))
&& k2?.tables?.[0]?.units?.['分区乙'] === '人次', JSON.stringify(k2?.tables?.[0]?.units ?? {}))
await page.waitForTimeout(300)
// 有问题的配方:一条对得上字段(单位),一条对不上(挂到面板顶部)
replies.recipe.push(echo({ recipe_problems: [
{ path: '/tables/0/units/分区甲', code: 'unit_label_conflict', message: '表「日客流量」:列「分区甲」的单位与标签「分区甲(人次)」不一致' },
{ path: '/sheets/0/blocks/0/label_offset', code: 'schema', message: '第 1 个工作表的交叉表:标签列偏移只能是紧挨日期的左侧一列' },
] }))
await panel.locator('[data-field="/tables/0/units/分区甲"]').selectOption('人')
await until(async () => puts().length === 7, 4000)
await until(async () => puts().length === 9, 4000)
await page.waitForTimeout(300)
const trialBtn = wizard().locator('[data-trial-run]')
check('配方有问题时「试运行」禁用', await trialBtn.isDisabled())
@@ -2548,8 +2569,8 @@ await section('数据 · 表格:按配方导入', async () => {
await panel.locator('[data-recipe-paste-toggle]').click()
await panel.locator('[data-recipe-paste] textarea').fill(JSON.stringify(FLOW_RECIPE))
await panel.locator('[data-recipe-paste] button', { hasText: '保存配方' }).click()
await until(async () => puts().length === 8, 4000)
check('「粘贴配方」发 PUT,body 就是粘贴的配方', JSON.stringify(puts()[7]?.body?.recipe) === JSON.stringify(FLOW_RECIPE))
await until(async () => puts().length === 10, 4000)
check('「粘贴配方」发 PUT,body 就是粘贴的配方', JSON.stringify(puts()[9]?.body?.recipe) === JSON.stringify(FLOW_RECIPE))
await page.waitForTimeout(300)
check('……问题清掉以后「试运行」可用', await trialBtn.isEnabled())
// 连续修改:上一次 PUT 还没回来就接着改。上一次回来时,排在后面的修改不能被服务端那份盖掉,