mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 01:21:59 +00:00
合入:交叉表支持按文字导入的指标(「高峰时段」一行写着「18-19」时落成文字列,规则起草按文字指标起草并要人确认,配方面板加「指标类型」);堵上占位符陷阱(拒收含数字的占位符,「不是数字」的下一步按格子情况写,整行文字给「按文字导入这一行」修复按钮)。验收:配方相关测试 2195 条、新增 15 条全过;check-copy 通过;check-manage 718 项全过;真实文件本机重跑起草完整、按日 31 行、时段 527 行;泄露扫描干净
This commit is contained in:
@@ -265,7 +265,7 @@ _FIELD_LABEL = {
|
||||
"meaning": "含义", "blank": "空格的处理", "text_number": "文本数字的处理", "formula": "公式的处理",
|
||||
"segments": "分段", "role": "分段类型", "table": "表", "locate": "定位方式", "by": "定位依据",
|
||||
"title": "分段标题", "segment": "紧跟的分段", "labels": "标签", "expect": "期望的标签",
|
||||
"measures": "指标列名对照", "dim": "维度", "derive": "派生列", "value": "值列", "const": "常量列",
|
||||
"measures": "指标列名对照", "text_measures": "按文字导入的指标", "dim": "维度", "derive": "派生列", "value": "值列", "const": "常量列",
|
||||
"pick": "取值", "stop_parser": "结束本段的标签", "labels_parser": "标签解析方式", "verify": "核对方式",
|
||||
"against_table": "核对的基表", "keep_as": "另存", "header_rows": "表头行数", "after_title": "表头上方的标题",
|
||||
"columns": "列", "header": "表头", "store": "存储写法", "extra_columns": "多出的列",
|
||||
@@ -815,6 +815,21 @@ class _Checker:
|
||||
if key in seen:
|
||||
self.add(lp, "label_duplicate", f"标签「{label}」与「{seen[key]}」是同一个标签")
|
||||
seen.setdefault(key, label)
|
||||
if isinstance(seg, MeasuresSegment):
|
||||
self.check_text_measures(sp, seg)
|
||||
|
||||
def check_text_measures(self, sp: tuple[Any, ...], seg: MeasuresSegment) -> None:
|
||||
"""文字指标的键与 measures 一样逐字对 labels.expect;标签末尾写着词表里的单位的是数字指标,不能按文字导入。"""
|
||||
for label in seg.text_measures:
|
||||
tp = sp + ("text_measures", label)
|
||||
if label not in seg.labels.expect:
|
||||
self.add(tp, "text_measure_unknown",
|
||||
f"按文字导入的「{label}」不在期望的标签中(必须与标签逐字相同)")
|
||||
continue
|
||||
unit = split_unit_suffix(label)[1]
|
||||
if unit is not None and unit in UNITS:
|
||||
self.add(tp, "text_measure_unit",
|
||||
f"标签「{label}」写着单位「{unit}」,是数字指标,不能按文字导入")
|
||||
|
||||
@staticmethod
|
||||
def label_key(seg: Any, label: str) -> str | None:
|
||||
@@ -880,13 +895,25 @@ class _Checker:
|
||||
# ---- 3. 占位符只能映射空值,不能像数 ----
|
||||
|
||||
def check_placeholders(self) -> None:
|
||||
"""占位符不能像数;也不能含数字(origin 不是 replay 时)。「18-19」「约3000」不像一个数,却是数据本身:
|
||||
当成占位符,整格的内容就静默存成空值,干跑照样通过。表示无数据的符号(「·」「—」「/」「无」)不含数字;
|
||||
已提交的配方里沿用下来的(同一块、canon 相同)放行,重放也不报,免得期 3 之前存下的配方在下一期被拒。"""
|
||||
literal = self.origin != "replay"
|
||||
carried = self.base_placeholders() if literal else {}
|
||||
for si, sheet in enumerate(self.r.sheets):
|
||||
for bi, block in enumerate(sheet.blocks):
|
||||
for n, ph in enumerate(block.values.placeholders):
|
||||
pp = ("sheets", si, "blocks", bi, "values", "placeholders", n, "text")
|
||||
if looks_numeric_text(ph.text):
|
||||
self.add(("sheets", si, "blocks", bi, "values", "placeholders", n, "text"),
|
||||
"placeholder_numeric",
|
||||
self.add(pp, "placeholder_numeric",
|
||||
f"占位符「{ph.text}」看起来是一个数,不能当作占位符(占位符一律存为空值)")
|
||||
elif (literal and any(ch.isdigit() for ch in ph.text)
|
||||
and canon(ph.text) not in carried.get(block.id, set())):
|
||||
what = (";如果这一行写的都是文字(如时段、名称),请把这个指标的「指标类型」改为「文字」或「时段」"
|
||||
if isinstance(block, CrosstabBlock) else "")
|
||||
self.add(pp, "placeholder_digits",
|
||||
f"占位符「{ph.text}」含数字:表示无数据的符号不含数字(如「·」「—」「无」),"
|
||||
f"含数字的通常是数据本身,当作占位符会整格存成空值{what}")
|
||||
for si, bi, block in self.lists:
|
||||
total = block.rows.total_row
|
||||
if total is not None and any(ch.isdigit() for ch in total.pick):
|
||||
@@ -897,6 +924,14 @@ class _Checker:
|
||||
self.add(("sheets", si, "blocks", bi, "rows", "subtotals", "pick"), "period_literal",
|
||||
f"小计行的开头词「{sub.pick}」不能含数字(配方里不能写死年份或期次)")
|
||||
|
||||
def base_placeholders(self) -> dict[str, set[str]]:
|
||||
"""现行配方(base)里每个块的占位符(canon 后):块 id → 集合。没有 base 时为空。"""
|
||||
out: dict[str, set[str]] = {}
|
||||
for sheet in self.base.sheets if self.base is not None else []:
|
||||
for block in sheet.blocks:
|
||||
out.setdefault(block.id, set()).update(canon(p.text) for p in block.values.placeholders)
|
||||
return out
|
||||
|
||||
# ---- 4、9、13. 单位:键是本表列名,值在词表里,与原表标签、表头里的单位一致 ----
|
||||
|
||||
def check_units(self) -> None:
|
||||
@@ -915,7 +950,12 @@ class _Checker:
|
||||
for si, bi, _gi, _block, seg in self.segs:
|
||||
if isinstance(seg, MeasuresSegment):
|
||||
for label in seg.labels.expect:
|
||||
if label in seg.measures:
|
||||
if label not in seg.measures:
|
||||
continue
|
||||
if label in seg.text_measures:
|
||||
# 标签写着单位的已报 text_measure_unit;这里只拦文字列上写的单位
|
||||
self.no_unit_on_text(seg.table, seg.measures[label])
|
||||
else:
|
||||
self.unit_against(seg.table, seg.measures[label], label, "标签")
|
||||
elif isinstance(seg, DimensionSegment) and seg.locate.by == "section_title" and seg.locate.title:
|
||||
self.unit_against(seg.table, seg.value, seg.locate.title, "分段标题")
|
||||
@@ -938,6 +978,12 @@ class _Checker:
|
||||
return next((s for sj, bj, _gi, _block, s in self.segs
|
||||
if (sj, bj) == (si, bi) and isinstance(s, DimensionSegment) and s.id == seg.locate.segment), None)
|
||||
|
||||
def no_unit_on_text(self, table: str, column: str) -> None:
|
||||
i = self.declared.get(table)
|
||||
if i is None or not column or column not in self.r.tables[i].units:
|
||||
return
|
||||
self.add(("tables", i, "units", column), "unit_on_text", f"列「{column}」按文字导入,不能写单位")
|
||||
|
||||
def unit_against(self, table: str, column: str, source: str, what: str) -> None:
|
||||
i = self.declared.get(table)
|
||||
if i is None or not column or (table in self.broken and column not in self.col_names(table)):
|
||||
|
||||
@@ -121,7 +121,8 @@ RECIPE_DRAFT_SYSTEM = (
|
||||
"2. 配方语言是封闭的:没有正则,不写单元格坐标;只能用 Schema 里有的字段和取值。\n"
|
||||
"3. measures 的键必须与 labels.expect 里的标签逐字相同、一一对应。\n"
|
||||
"4. const 的 pick 只能从「分段标题候选词」里选。\n"
|
||||
"5. 占位符只能表示空值(无数据或不适用)。\n"
|
||||
"5. 占位符只能表示空值(无数据或不适用),不含数字。指标行每格写的都是文字(如时段「18-19」)时,"
|
||||
"在该 measures 分段的 text_measures 里声明(时段用 hour_range,其余用 text),不要当成占位符。\n"
|
||||
"6. 单位只能取自「单位词表」;标签或表头末尾括号里的单位拆进 tables[].units,列名去掉单位,"
|
||||
"单位必须与括号里写的一致;括号里的单位不在词表里时,列名保留括号、不写单位。\n"
|
||||
"7. 合计、小计行用 role=derived 核对,并用 keep_as 另存成一张表。\n"
|
||||
|
||||
@@ -214,7 +214,8 @@ def _column_sources(recipe: Recipe) -> dict[tuple[str, str], str]:
|
||||
|
||||
|
||||
def _text_stores(recipe: Recipe) -> dict[tuple[str, str], tuple[str, bool]]:
|
||||
"""(表, 列表的 TEXT 列) → (实际生效的存法, 是否显式写了)。缺省时主键列 canonical、其余 raw。"""
|
||||
"""(表, 列表的 TEXT 列) → (实际生效的存法, 是否显式写了)。缺省时主键列 canonical、其余 raw。
|
||||
交叉表按文字导入的指标也算:按时段解析的存规范写法,按文字的存原文,都是显式写的。"""
|
||||
grains = {t.name: set(t.grain) for t in recipe.tables}
|
||||
out: dict[tuple[str, str], tuple[str, bool]] = {}
|
||||
for sheet in recipe.sheets:
|
||||
@@ -224,6 +225,13 @@ def _text_stores(recipe: Recipe) -> dict[tuple[str, str], tuple[str, bool]]:
|
||||
if c.type == "TEXT":
|
||||
eff = c.store or ("canonical" if c.name in grains.get(block.table, set()) else "raw")
|
||||
out[(block.table, c.name)] = (eff, c.store is not None)
|
||||
continue
|
||||
for seg in block.segments:
|
||||
if isinstance(seg, MeasuresSegment):
|
||||
for label, tm in seg.text_measures.items():
|
||||
if seg.measures.get(label):
|
||||
out[(seg.table, seg.measures[label])] = (
|
||||
"canonical" if tm.parser == "hour_range" else "raw", True)
|
||||
return out
|
||||
|
||||
|
||||
@@ -516,6 +524,9 @@ def _signatures(recipe: Recipe, sheet_name: Callable[[SheetRecipe], str]) -> dic
|
||||
if isinstance(seg, DerivedSegment):
|
||||
keep = seg.keep_as.table if seg.keep_as else None
|
||||
out[f"derived:{seg.id}"] = _sig(keep, sorted(match_key(x) for x in seg.labels.expect))
|
||||
elif isinstance(seg, MeasuresSegment):
|
||||
for label, tm in seg.text_measures.items():
|
||||
out[f"text_measure:{seg.id}:{label}"] = _sig(tm.parser, seg.measures.get(label))
|
||||
else:
|
||||
out[f"merged_fill:{block.id}"] = _sig(block.merged_data)
|
||||
out[f"ignored_columns:{block.id}"] = _sig(block.extra_columns, ignore_cols)
|
||||
@@ -1048,6 +1059,12 @@ def _recipe_items(bag: _Bag, recipe: Recipe, ex: dict[str, Any], facts: Any, *,
|
||||
prefix = rows_text(seg_rows.get(seg.id)) or f"分段「{seg.id}」的合计行"
|
||||
keep = f",原值另存表「{seg.keep_as.table}」" if seg.keep_as else ",原值不另存"
|
||||
bag.add(f"derived:{seg.id}", f"{prefix}改作核对{keep}")
|
||||
elif isinstance(seg, MeasuresSegment):
|
||||
# 文字指标:这一行不再按数字核对类型,写错成别的文字也照样入库,所以配置了就要人确认
|
||||
for label, tm in seg.text_measures.items():
|
||||
how = "按时段导入,存规范写法(如 18-19)" if tm.parser == "hour_range" else "按文字原样导入"
|
||||
bag.add(f"text_measure:{seg.id}:{label}",
|
||||
f"「{label}」{how},不当数字:表「{seg.table}」的列「{seg.measures.get(label, '')}」是文字列")
|
||||
derived_rows: dict[str, set[int]] = {}
|
||||
for d in ex.get("derived") or []:
|
||||
if isinstance(d, dict) and d.get("kind") == "column_sum":
|
||||
|
||||
@@ -67,10 +67,10 @@ from app.data import engine as _data_engine
|
||||
from app.data import tabular, xlsx_cells
|
||||
from app.data.names import canon
|
||||
from app.data.recipe_parsers import (
|
||||
TOTAL_WORDS, AxisDate, HourRange, HourTotal, Period, classify_outside, cn_date_range, cn_year_month,
|
||||
TOTAL_WORDS, UNITS, AxisDate, HourRange, HourTotal, Period, classify_outside, cn_date_range, cn_year_month,
|
||||
filename_period,
|
||||
hour_range, hour_range_total, is_pure_period_cell, looks_numeric_text, match_key, month_day_or_date,
|
||||
period_residue, text_label, thousands_number,
|
||||
period_residue, split_unit_suffix, text_label, thousands_number,
|
||||
)
|
||||
from app.data.recipe_types import (
|
||||
SUBTOTAL_KEY, AxisOut, CanonEntry, CheckResult, ColumnOut, ContextSpec, CrosstabBlock, DerivedItem,
|
||||
@@ -116,7 +116,7 @@ PROBLEM_CODES: dict[str, str] = {
|
||||
"value_not_integer": "structure", "value_text_number": "structure", "value_blank": "structure",
|
||||
"value_error": "structure", "value_not_number": "structure", "value_not_date": "structure",
|
||||
"value_formula": "structure", "formula_uncached": "structure", "formula_full_calc": "structure",
|
||||
"pk_duplicate": "structure",
|
||||
"pk_duplicate": "structure", "value_not_text": "structure", "value_not_hour_range": "structure",
|
||||
}
|
||||
|
||||
#: 修复按钮的种类(契约 Problem.fix,取值见 FIX_KINDS)。参数 fix_args 的形状见 P3-SPEC 3.2,由报问题的地方随问题
|
||||
@@ -520,6 +520,9 @@ class _Issue:
|
||||
row: int | None = None
|
||||
#: value_not_number:能当占位符提议的文字(canon 后)→ 格数,最多收 PLACEHOLDER_TEXTS_MAX + 1 种
|
||||
texts: Counter[str] = field(default_factory=Counter)
|
||||
#: value_not_number:measures 段里整行都是文字的指标(declare_text_measure 的参数):
|
||||
#: {"segment", "label"(期望标签原文), "cell"(标签格), "parsers"(能用的解析方式)}
|
||||
text_rows: list[dict[str, Any]] = field(default_factory=list)
|
||||
|
||||
|
||||
#: 合并类问题的说明:code → (原因({n} 是个数), 下一步)。下一步写配方面板上的「字段」和「选项」原文
|
||||
@@ -530,8 +533,13 @@ _ISSUE_TEXT: dict[str, tuple[str, str]] = {
|
||||
"。如需按数字导入,请在配方面板的「文本写的数字」中选择「千分位写法按数字保存」"),
|
||||
"value_blank": ("{n} 个单元格为空", "。如原表确实为空,请在配方面板的「空格」中选择「存为空值」"),
|
||||
"value_error": ("{n} 个单元格是错误值", "。请在 Excel 中修正后重新上传"),
|
||||
# 下一步按格子的情况另写(_not_number_tail):整行是文字的提示改「指标类型」,能当占位符的才提示加占位符
|
||||
"value_not_number": ("{n} 个单元格不是数字", "。如果它表示无数据,请在配方面板中添加占位符"),
|
||||
"value_not_date": ("{n} 个单元格不是日期", "。请检查原表,或在配方面板中把这一列的「类型」改为「文字」"),
|
||||
"value_not_text": ("{n} 个单元格不是文字",
|
||||
"。这个指标按文字导入:请检查原表,或在配方面板中把它的「指标类型」改为「数字」"),
|
||||
"value_not_hour_range": ("{n} 个单元格无法按时段解析",
|
||||
"。这个指标按时段导入:请检查原表,或在配方面板中把它的「指标类型」改为「文字」"),
|
||||
"value_formula": ("{n} 个数据格是公式",
|
||||
"。数据区的公式通常是没有识别出来的合计;如确为数据,请在配方面板的「公式」中选择「按保存值导入」"),
|
||||
"formula_uncached": ("{n} 个公式格没有保存计算结果", ",文件未经计算保存。请在 Excel 中打开并保存后重新上传"),
|
||||
@@ -548,6 +556,21 @@ _PK_PLACEHOLDER = ("主键列有 {n} 个单元格是占位符", "。占位符按
|
||||
_PK_NULL = ("有 {n} 处为空值", "。主键不能为空:请检查原表,或在配方面板的「主键」中去掉这一列")
|
||||
|
||||
|
||||
def _not_number_tail(it: _Issue) -> str:
|
||||
"""value_not_number 的下一步。整行写的都是文字的指标,提示改「指标类型」;有能当占位符的文字(不含数字的短符号)
|
||||
才提示加占位符:「18-19」这种写法当占位符会整格存成空值,干跑照样通过(静态校验 placeholder_digits 也拦)。"""
|
||||
hint: list[str] = []
|
||||
if it.text_rows:
|
||||
hint.append(f"{_quoted([x['label'] for x in it.text_rows])}这一行写的都是文字:如果本来就是文字(如时段、名称),"
|
||||
"请在配方面板中把它的「指标类型」改为「文字」或「时段」")
|
||||
if it.texts:
|
||||
hint.append("如果其中有表示无数据的符号,请在配方面板中添加占位符" if hint
|
||||
else "如果它表示无数据,请在配方面板中添加占位符")
|
||||
if not hint:
|
||||
return "。数值区只能是数字或表示无数据的符号(如「·」「—」,不含数字):请检查原表"
|
||||
return "。" + ";".join(hint)
|
||||
|
||||
|
||||
class _Problems:
|
||||
def __init__(self) -> None:
|
||||
self.items: list[tuple[str | None, Problem]] = []
|
||||
@@ -592,6 +615,12 @@ class _Problems:
|
||||
if sample is not None and (sample in it.texts or len(it.texts) <= PLACEHOLDER_TEXTS_MAX):
|
||||
it.texts[sample] += 1
|
||||
|
||||
def text_row(self, sheet: str, where: str, row: dict[str, Any]) -> None:
|
||||
"""把整行是文字的指标记到这一段的 value_not_number 上(那一行的格转换时已经各自报过)。"""
|
||||
it = self.issues.get(("value_not_number", sheet, where + "\x00"))
|
||||
if it is not None and all(x["label"] != row["label"] for x in it.text_rows):
|
||||
it.text_rows.append(row)
|
||||
|
||||
def finish(self, failed_sheets: set[str],
|
||||
args_for: Callable[[_Issue], dict[str, Any] | None] | None = None) -> list[Problem]:
|
||||
"""合并类问题出成 Problem;args_for 给有修复按钮的合并类问题(outside_number、value_not_number)算 fix_args。
|
||||
@@ -602,7 +631,8 @@ class _Problems:
|
||||
ex = f"(如 {';'.join(it.examples)})" if it.examples else f"({_join(shown, 5)})"
|
||||
model_ex = f"({_join(shown, 5)})"
|
||||
args = args_for(it) if args_for is not None and it.code in PROBLEM_FIX else None
|
||||
self.add(it.code, f"{it.where}:{reason}{ex}{it.tail}", f"{it.where}:{reason}{model_ex}", it.cells,
|
||||
tail = _not_number_tail(it) if it.code == "value_not_number" else it.tail
|
||||
self.add(it.code, f"{it.where}:{reason}{ex}{tail}", f"{it.where}:{reason}{model_ex}", it.cells,
|
||||
sheet=it.sheet, fix_args=args)
|
||||
for (code, sheet), n in self._over.items():
|
||||
where = f"工作表「{sheet}」" if sheet else "本文件"
|
||||
@@ -1482,10 +1512,22 @@ class _Crosstab:
|
||||
eng.headers[(seg.table, seg.measures.get(label, ""))] = _text(self._label(r).value) # type: ignore[union-attr]
|
||||
# 值保留 _FAIL(已报过问题),交给 _Sink 按空值存;没找到的标签(已报 label_missing)也按 _FAIL
|
||||
values: dict[tuple[str, int], Any] = {}
|
||||
pol = self.block.values
|
||||
for label, r in rows_by_label.items():
|
||||
excluded = self.exclude_hidden and sh.hidden_row(r)
|
||||
text = seg.text_measures.get(label)
|
||||
column = seg.measures.get(label, "")
|
||||
for c in self._axis_cols():
|
||||
values[(label, c)] = None if excluded else self._value(r, c, where)
|
||||
if excluded:
|
||||
values[(label, c)] = None
|
||||
elif text is not None:
|
||||
values[(label, c)] = eng.convert_text(self.grid.get(r, c), text.parser, pol, sh.name, where, r, c,
|
||||
seg.table, column)
|
||||
else:
|
||||
values[(label, c)] = self._value(r, c, where)
|
||||
if text is None and not excluded and (parsers := self._text_parsers(r, label)):
|
||||
eng.problems.text_row(sh.name, where, {"segment": seg.id, "label": label,
|
||||
"cell": sh.coord(r, self.label_c), "parsers": parsers})
|
||||
if self.dates is None:
|
||||
return
|
||||
for c in self._axis_cols():
|
||||
@@ -1498,6 +1540,31 @@ class _Crosstab:
|
||||
cells.append((seg.measures.get(label, ""), r, c))
|
||||
sink.add(row, sheet=sh.name, at=(self.axis_row, c), cells=cells, fails=True)
|
||||
|
||||
def _text_parsers(self, r: int, label: str) -> list[str] | None:
|
||||
"""这一行能不能改按文字指标导入(declare_text_measure 的提议):轴列上除了占位符和空格,全是文字(不是数字、
|
||||
日期、公式、错误值,也不是写成文字的数),至少一格。全都能按时段解析时给两种解析方式,时段在前。标签写着
|
||||
词表里的单位的是数字指标(静态校验 text_measure_unit 也拒),不提议。"""
|
||||
unit = split_unit_suffix(label)[1]
|
||||
if unit is not None and unit in UNITS:
|
||||
return None
|
||||
ph = self.eng._placeholders(self.block.values)
|
||||
texts: list[str] = []
|
||||
for c in self._axis_cols():
|
||||
cell = self.grid.get(r, c)
|
||||
if cell is None:
|
||||
continue
|
||||
if cell.formula is not None or not isinstance(cell.value, str):
|
||||
return None
|
||||
t = cell.value.strip()
|
||||
if not t or canon(t) in ph:
|
||||
continue
|
||||
if t in tabular.EXCEL_ERRORS or thousands_number(t) is not None or looks_numeric_text(t):
|
||||
return None
|
||||
texts.append(t)
|
||||
if not texts:
|
||||
return None
|
||||
return ["hour_range", "text"] if all(hour_range(t) is not None for t in texts) else ["text"]
|
||||
|
||||
def _derive_values(self, derive: dict[str, str], hr: Any) -> list[Any]:
|
||||
ordered = sorted(derive.items(), key=lambda kv: (kv[1] != "start", kv[0]))
|
||||
if not isinstance(hr, (HourRange, HourTotal)):
|
||||
@@ -2441,6 +2508,55 @@ class _Engine:
|
||||
return v if vtype == "REAL" else int(v)
|
||||
return self._number(v, vtype, pol, sheet, where, r, c, pk)
|
||||
|
||||
def convert_text(self, cell: GridCell | None, parser: str, pol: Any, sheet: str, where: str, r: int, c: int,
|
||||
table: str, column: str) -> Any:
|
||||
"""文字指标的一格(MeasuresSegment.text_measures)→ 入库值。空格、公式、占位符照块的取值规则;text 存原文
|
||||
(去首尾空白),hour_range 存时段的规范写法。数字、日期不是文字(不替人把 8-9 被 Excel 改成的日期认回去),
|
||||
解析不了的时段也是问题:都按 _FAIL 返回、已记问题。"""
|
||||
p = self.problems
|
||||
if cell is None:
|
||||
if pol.blank == "null":
|
||||
return None
|
||||
p.issue("value_blank", sheet, where, _coord(sheet, r, c))
|
||||
return _FAIL
|
||||
v = cell.value
|
||||
if cell.formula is not None:
|
||||
v = self._accept_formula(cell, pol, sheet, where, r, c)
|
||||
if v is _FAIL:
|
||||
return _FAIL
|
||||
if not isinstance(v, str):
|
||||
p.issue("value_not_text", sheet, where, _coord(sheet, r, c), f"{cell_ref(r, c)}「{_shown(v)}」")
|
||||
return _FAIL
|
||||
t = v.strip()
|
||||
ph = self._placeholders(pol)
|
||||
key = canon(t)
|
||||
if key in ph:
|
||||
name = ph[key]
|
||||
self.ex.placeholders[name] = self.ex.placeholders.get(name, 0) + 1
|
||||
return None
|
||||
if not t:
|
||||
if pol.blank == "null":
|
||||
return None
|
||||
p.issue("value_blank", sheet, where, _coord(sheet, r, c))
|
||||
return _FAIL
|
||||
if t in tabular.EXCEL_ERRORS:
|
||||
p.issue("value_error", sheet, where, _coord(sheet, r, c), f"{cell_ref(r, c)}「{_shown(v)}」")
|
||||
return _FAIL
|
||||
if parser == "text":
|
||||
return t
|
||||
hr = hour_range(t)
|
||||
if hr is None:
|
||||
p.issue("value_not_hour_range", sheet, where, _coord(sheet, r, c), f"{cell_ref(r, c)}「{_shown(v)}」")
|
||||
return _FAIL
|
||||
if hr.canonical != t:
|
||||
log = (table, column, t, hr.canonical)
|
||||
hit = self.canon_log.get(log)
|
||||
if hit is not None:
|
||||
hit.count += 1
|
||||
elif len(self.canon_log) < CANON_MAX:
|
||||
self.canon_log[log] = CanonEntry(table, column, t, hr.canonical, 1, _coord(sheet, r, c))
|
||||
return hr.canonical
|
||||
|
||||
def _accept_formula(self, cell: GridCell, pol: Any, sheet: str, where: str, r: int, c: int) -> Any:
|
||||
"""数据区的公式格:拒收、无缓存、打开时重算都是问题;否则按保存值导入并计数。"""
|
||||
p = self.problems
|
||||
@@ -2675,7 +2791,11 @@ class _Engine:
|
||||
if it.block is None:
|
||||
return None
|
||||
texts = list(it.texts)[:PLACEHOLDER_TEXTS_MAX]
|
||||
return {"block": it.block, "texts": texts, "other": it.count - sum(it.texts[t] for t in texts)}
|
||||
args = {"block": it.block, "texts": texts, "other": it.count - sum(it.texts[t] for t in texts)}
|
||||
if it.text_rows:
|
||||
# 只有标签原文、标签格坐标和解析方式,没有数据格的内容
|
||||
args["text_rows"] = [dict(x) for x in it.text_rows]
|
||||
return args
|
||||
if it.code == "outside_number" and it.row is not None:
|
||||
sh = next((x for x in self.sheets if x.name == it.sheet), None)
|
||||
if sh is None:
|
||||
|
||||
@@ -27,10 +27,12 @@ from pydantic import ValidationError
|
||||
from app.data.names import canon, collide_key
|
||||
from app.data.recipe import apply_patch, recipe_sha256
|
||||
from app.data.recipe_parsers import (
|
||||
UNITS,
|
||||
candidate_words,
|
||||
looks_numeric_text,
|
||||
match_key,
|
||||
month_day_or_date,
|
||||
split_unit_suffix,
|
||||
)
|
||||
from app.data.recipe_suggest import _measure_column
|
||||
from app.data.recipe_types import (
|
||||
@@ -255,6 +257,8 @@ def _key(kind: str, target: dict[str, Any]) -> str:
|
||||
return f"rename_sheet:{t['sheet_id']}"
|
||||
if kind == "declare_subtotals":
|
||||
return f"declare_subtotals:{t['block']}"
|
||||
if kind == "declare_text_measure":
|
||||
return f"declare_text_measure:{t['segment']}:{t['label']}"
|
||||
raise KeyError(kind)
|
||||
|
||||
|
||||
@@ -308,6 +312,8 @@ def _o_remove_label(recipe: dict, p: FixProposal, opt: FixOption, reason: str |
|
||||
cols = []
|
||||
for e in removed:
|
||||
col = seg["measures"].get(e)
|
||||
if e in (seg.get("text_measures") or {}):
|
||||
ops.append({"op": "remove", "path": ptr(*sp, "text_measures", e)})
|
||||
if col is None:
|
||||
continue
|
||||
cols.append(col)
|
||||
@@ -967,6 +973,67 @@ def _o_declare_placeholder(recipe: dict, p: FixProposal, opt: FixOption, reason:
|
||||
return ops, [f"块「{block['id']}」把{quoted(p.target['texts'])}声明为{meaning}(存为空值)"]
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# ⑪ 按文字导入整行是文字的指标(value_not_number 的另一个出口:「高峰时段」一行写的是「18-19」)
|
||||
# ==========================================================================
|
||||
|
||||
_TEXT_PARSER_OPTION = {
|
||||
"hour_range": ("按时段导入(存规范写法,如 18-19)", "以后各期这一行每格都要能按时段解析,否则拒收"),
|
||||
"text": ("按文字原样导入", "以后各期这一行按原文存,不再要求是数字"),
|
||||
}
|
||||
|
||||
|
||||
def _p_declare_text_measure(recipe: dict, prob: Any, args: dict, anchor: FixAnchor) -> list[FixProposal]:
|
||||
"""执行器在 fix_args.text_rows 里给出整行都是文字的指标(标签原文、标签格、能用的解析方式)。每个指标一条提议;
|
||||
标签写着词表里的单位的不提议(静态校验 text_measure_unit 会拒),已经按文字导入的不再提议。"""
|
||||
out: list[FixProposal] = []
|
||||
for row in args.get("text_rows") or []:
|
||||
found = find_segment(recipe, row.get("segment"))
|
||||
label = row.get("label")
|
||||
if found is None or not isinstance(label, str):
|
||||
continue
|
||||
seg = found[4]
|
||||
if (seg.get("role") != "measures" or label not in (seg.get("labels") or {}).get("expect", [])
|
||||
or label in (seg.get("text_measures") or {}) or not seg.get("measures", {}).get(label)):
|
||||
continue
|
||||
unit = split_unit_suffix(label)[1]
|
||||
if unit is not None and unit in UNITS:
|
||||
continue
|
||||
parsers = [p for p in row.get("parsers") or [] if p in _TEXT_PARSER_OPTION]
|
||||
if not parsers:
|
||||
continue
|
||||
target = {"segment": seg["id"], "label": label}
|
||||
col = seg["measures"][label]
|
||||
cell = row.get("cell")
|
||||
out.append(_proposal(
|
||||
"declare_text_measure", _key("declare_text_measure", target), code=_get(prob, "code"),
|
||||
title=f"「{label}」这一行写的都是文字,不是数字",
|
||||
cells=[cell] if isinstance(cell, str) else [], target=target,
|
||||
options=[FixOption(p, _TEXT_PARSER_OPTION[p][0],
|
||||
detail=f"表「{seg.get('table')}」的列「{col}」改成文字列;{_TEXT_PARSER_OPTION[p][1]}",
|
||||
breaking=True) for p in parsers],
|
||||
anchor=anchor))
|
||||
return out
|
||||
|
||||
|
||||
def _o_declare_text_measure(recipe: dict, p: FixProposal, opt: FixOption, reason: str | None) -> tuple[list, list[str]]:
|
||||
found = find_segment(recipe, p.target.get("segment"))
|
||||
label = p.target.get("label")
|
||||
if found is None or found[4].get("role") != "measures" or label not in found[4].get("measures", {}):
|
||||
raise _Stale
|
||||
si, bi, gi, _block, seg = found
|
||||
sp = ("sheets", si, "blocks", bi, "segments", gi)
|
||||
ops: list[dict[str, Any]] = []
|
||||
if "text_measures" not in seg:
|
||||
ops.append({"op": "add", "path": ptr(*sp, "text_measures"), "value": {}})
|
||||
ops.append({"op": "add", "path": ptr(*sp, "text_measures", label), "value": {"parser": opt.value}})
|
||||
col = seg["measures"][label]
|
||||
ti = table_index(recipe, seg.get("table"))
|
||||
if ti is not None and col in (recipe["tables"][ti].get("units") or {}):
|
||||
ops.append({"op": "remove", "path": ptr("tables", ti, "units", col)})
|
||||
return ops, [f"分段「{seg['id']}」的「{label}」{opt.label}:表「{seg.get('table')}」的列「{col}」是文字列"]
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# ⑧ 声明隐藏行列怎么处理
|
||||
# ==========================================================================
|
||||
@@ -1062,7 +1129,12 @@ _OPS: dict[str, Callable[[dict, FixProposal, FixOption, str | None], tuple[list,
|
||||
"remove_label": _o_remove_label, "add_label": _o_add_label, "edit_members": _o_edit_members,
|
||||
"rename_title": _o_rename_title, "declare_total": _o_declare_total, "ignore_cells": _o_ignore_cells,
|
||||
"declare_placeholder": _o_declare_placeholder, "declare_hidden": _o_declare_hidden, "rename_sheet": _o_rename_sheet,
|
||||
"declare_subtotals": _o_declare_subtotals,
|
||||
"declare_subtotals": _o_declare_subtotals, "declare_text_measure": _o_declare_text_measure,
|
||||
}
|
||||
#: 同一条问题的另一种出口(问题的 fix 字段只写一种,界面按 fix_ids 放按钮,所以能挂几个提议):value_not_number 的
|
||||
#: 格子是整行文字时,除了声明占位符,还可以把这个指标改按文字导入
|
||||
_EXTRA_BUILDERS: dict[str, Callable[[dict, Any, dict, FixAnchor], list[FixProposal]]] = {
|
||||
"value_not_number": _p_declare_text_measure,
|
||||
}
|
||||
|
||||
|
||||
@@ -1096,7 +1168,15 @@ def propose_fixes(recipe: dict[str, Any], problems: list[dict[str, Any]], recipe
|
||||
try:
|
||||
push(builder(recipe, prob, args, FixAnchor("problem", i)))
|
||||
except (KeyError, TypeError, ValueError, IndexError, AttributeError):
|
||||
continue # fix_args 的形状不对(执行器的契约问题):不出提议,不拖垮 staging_out
|
||||
pass # fix_args 的形状不对(执行器的契约问题):不出提议,不拖垮 staging_out
|
||||
extra = _EXTRA_BUILDERS.get(_get(prob, "code") or "")
|
||||
if extra is None:
|
||||
continue
|
||||
try:
|
||||
for p in extra(recipe, prob, args, FixAnchor("problem", i)):
|
||||
push(p)
|
||||
except (KeyError, TypeError, ValueError, IndexError, AttributeError):
|
||||
continue
|
||||
fl = _facts(facts)
|
||||
for i, rp in enumerate(recipe_problems or []):
|
||||
try:
|
||||
@@ -1125,6 +1205,8 @@ def _edit_title(p: FixProposal, opt: FixOption) -> str:
|
||||
return opt.label
|
||||
if p.kind == "declare_subtotals":
|
||||
return f"列表「{t.get('block')}」中间「{t.get('pick')}」开头的行{opt.label}"
|
||||
if p.kind == "declare_text_measure":
|
||||
return f"分段「{t.get('segment')}」的「{t.get('label')}」{opt.label}"
|
||||
return p.title
|
||||
|
||||
|
||||
|
||||
@@ -696,8 +696,9 @@ def _period_column_key(col: ColumnOut) -> str | None:
|
||||
|
||||
|
||||
def _null_eligible(ctx: _Ctx, w: _Writers, table: str, col: ColumnOut) -> bool:
|
||||
"""这一列要不要考虑空值说明:值列;主键不会是空值;表内合计表的空值已由表说明(未能核对)交代。"""
|
||||
value_col = col.role in ("measure", "value") and col.type in ("INTEGER", "REAL")
|
||||
"""这一列要不要考虑空值说明:值列(交叉表按文字导入的指标也是,它的占位符、空格同样存成空值);主键不会是空值;
|
||||
表内合计表的空值已由表说明(未能核对)交代。"""
|
||||
value_col = col.role == "measure" or (col.role == "value" and col.type in ("INTEGER", "REAL"))
|
||||
is_total = bool(w.kept_from or w.list_totals or w.list_subtotals)
|
||||
return value_col and not is_total and col.name not in set(ctx.grains.get(table, []))
|
||||
|
||||
|
||||
@@ -359,6 +359,7 @@ class _Seg:
|
||||
title: str | None = None
|
||||
title_row: int | None = None
|
||||
parser: str | None = None # dimension:hour_range / text
|
||||
text: dict[str, str] = field(default_factory=dict) # measures:整行是文字的标签原文 → 解析方式
|
||||
# 以下由命名阶段填
|
||||
id: str = ""
|
||||
table: str = ""
|
||||
@@ -417,6 +418,41 @@ def _sums_above(cell: GridCell | None, r: int) -> bool:
|
||||
return bool(m) and m.group(1) == m.group(3) and int(m.group(4)) == r - 1
|
||||
|
||||
|
||||
def _text_row(grid: Grid, r: int, c1: int, c2: int, label: str) -> str | None:
|
||||
"""指标行是不是整行文字(「高峰时段」每格写着「18-19」):是的话返回解析方式(全都能按时段解析的 hour_range,
|
||||
否则 text),不是返回 None。轴列上除了占位符候选和空格全是文字,至少一格;有一格是数、公式、错误值、写成文字的
|
||||
数就不是。标签末尾写着词表里的单位(「时段人数(人次)」)的是数字指标,不算。"""
|
||||
unit = P.split_unit_suffix(label)[1]
|
||||
if unit is not None and P.unit_known(unit):
|
||||
return None
|
||||
texts: list[str] = []
|
||||
for c in range(c1, c2 + 1):
|
||||
cell = grid.get(r, c)
|
||||
kind = _cell_kind(cell)
|
||||
if kind in ("blank", "ph"):
|
||||
continue
|
||||
if kind != "text":
|
||||
return None
|
||||
t = cell.value.strip() # type: ignore[union-attr]
|
||||
if P.looks_numeric_text(t) or P.thousands_number(t) is not None:
|
||||
return None
|
||||
texts.append(t)
|
||||
if not texts:
|
||||
return None
|
||||
return "hour_range" if all(P.hour_range(t) is not None for t in texts) else "text"
|
||||
|
||||
|
||||
def _value_failure(sheet: str, r: int, label: str, why: str, cells: list[tuple[int, str]]) -> _Failure:
|
||||
"""数值区一行里同一种不合格的格合成一条(一行 30 格不再报 30 条)。给模型的那条只有坐标,没有格子里的字。"""
|
||||
if len(cells) == 1:
|
||||
c, shown = cells[0]
|
||||
return _Failure(f"工作表「{sheet}」:{_a1(r, c)} 的「{shown}」{why}", f"工作表「{sheet}」:{_a1(r, c)} {why}")
|
||||
examples = "、".join(f"{_a1(r, c)}「{shown}」" for c, shown in cells[:3])
|
||||
where = _a1_list([(r, c) for c, _ in cells], 3)
|
||||
return _Failure(f"工作表「{sheet}」:第 {r} 行「{label}」有 {len(cells)} 个单元格{why}(如 {examples})",
|
||||
f"工作表「{sheet}」:第 {r} 行有 {len(cells)} 个单元格{why}({where})")
|
||||
|
||||
|
||||
def _analyze_crosstab(grid: Grid, axis_row: int, date_cols: list[int]) -> _Cross:
|
||||
sheet = grid.sheet
|
||||
c1, c2 = min(date_cols), max(date_cols)
|
||||
@@ -544,8 +580,13 @@ def _analyze_crosstab(grid: Grid, axis_row: int, date_cols: list[int]) -> _Cross
|
||||
# ---- 数值区:类型、占位符、空格、公式(P2-SPEC 6.1 第 9 条)
|
||||
ints = True
|
||||
for seg in x.segments:
|
||||
for r in seg.rows:
|
||||
for r, label in zip(seg.rows, seg.labels):
|
||||
# 指标行整行是文字(「高峰时段」每格写着「18-19」):按文字指标起草,不当数字报错;要人在问题里确认
|
||||
text = _text_row(grid, r, c1, c2, label) if seg.role == "measures" else None
|
||||
if text is not None:
|
||||
seg.text[label] = text
|
||||
row_vals: list[int | float | None] = []
|
||||
bad: dict[str, list[tuple[int, str]]] = {}
|
||||
for c in range(c1, c2 + 1):
|
||||
cell = grid.get(r, c)
|
||||
kind = _cell_kind(cell)
|
||||
@@ -565,13 +606,16 @@ def _analyze_crosstab(grid: Grid, axis_row: int, date_cols: list[int]) -> _Cross
|
||||
entry[1] += 1
|
||||
entry[2].append((r, c))
|
||||
x.nonnumeric[cell.value.strip()] = x.nonnumeric.get(cell.value.strip(), 0) + 1
|
||||
elif kind == "text" and text is not None:
|
||||
continue
|
||||
elif kind in ("text", "error", "other"):
|
||||
shown = _shown(cell.value)
|
||||
if isinstance(cell.value, str):
|
||||
x.nonnumeric[shown] = x.nonnumeric.get(shown, 0) + 1
|
||||
why = "是错误值" if kind == "error" else "不是数字,也不像表示无数据的符号"
|
||||
fail(_Failure(f"工作表「{sheet}」:{_a1(r, c)} 的「{shown}」{why}",
|
||||
f"工作表「{sheet}」:{_a1(r, c)} {why}"))
|
||||
bad.setdefault("是错误值" if kind == "error" else "不是数字,也不像表示无数据的符号",
|
||||
[]).append((c, shown))
|
||||
for why, cells in bad.items():
|
||||
fail(_value_failure(sheet, r, label, why, cells))
|
||||
x.values[r] = row_vals
|
||||
x.value_type = "INTEGER" if ints else "REAL"
|
||||
|
||||
@@ -1436,8 +1480,11 @@ def _prefer_prefix(period: list[tuple[int, int, str]]) -> str | None:
|
||||
|
||||
def _segment_dict(s: _Seg) -> dict[str, Any]:
|
||||
if s.role == "measures":
|
||||
return {"id": s.id, "role": "measures", "table": s.table, "locate": {"by": "labels"},
|
||||
"labels": {"expect": list(s.labels)}, "measures": dict(s.columns)}
|
||||
out = {"id": s.id, "role": "measures", "table": s.table, "locate": {"by": "labels"},
|
||||
"labels": {"expect": list(s.labels)}, "measures": dict(s.columns)}
|
||||
if s.text:
|
||||
out["text_measures"] = {label: {"parser": p} for label, p in s.text.items()}
|
||||
return out
|
||||
if s.role == "dimension":
|
||||
out: dict[str, Any] = {
|
||||
"id": s.id, "role": "dimension", "table": s.table,
|
||||
@@ -2427,6 +2474,18 @@ def _unit_cards(labels: list[tuple[str, str]], sheet_id: str) -> list[Card]:
|
||||
return out
|
||||
|
||||
|
||||
def _label_row(grid: Grid | None, loc: _CrossLoc | None, seg_id: str, label: str) -> int | None:
|
||||
"""分段里标签是 label 的那一行(按 match_key 比);定位不到时 None。"""
|
||||
if grid is None or loc is None:
|
||||
return None
|
||||
key = P.match_key(label)
|
||||
for r in loc.seg_rows.get(seg_id, []):
|
||||
cell = grid.get(r, loc.label_col)
|
||||
if cell is not None and isinstance(cell.value, str) and P.match_key(cell.value) == key:
|
||||
return r
|
||||
return None
|
||||
|
||||
|
||||
def _crosstab_cards(si: int, bi: int, sheet: Any, block: CrosstabBlock, loc: _CrossLoc | None, grid: Grid | None,
|
||||
name: str, recipe: dict[str, Any], parsed: Recipe, sheet_facts: list[Fact],
|
||||
relation_question: Callable[[Fact], Question], ext: Extraction | None,
|
||||
@@ -2539,6 +2598,32 @@ def _crosstab_cards(si: int, bi: int, sheet: Any, block: CrosstabBlock, loc: _Cr
|
||||
cards.extend(ph_cards)
|
||||
questions.extend(reversed(ph_qs))
|
||||
|
||||
# 6b 按文字导入的指标:要人确认。「不是」去掉这一条,这一行回到按块的数值类型导入,试运行会拒收
|
||||
for gi, s in enumerate(block.segments):
|
||||
if not isinstance(s, MeasuresSegment):
|
||||
continue
|
||||
for label, tm in s.text_measures.items():
|
||||
r = _label_row(grid, loc, s.id, label)
|
||||
spots: list[tuple[int, int]] = []
|
||||
if r is not None and loc is not None and grid is not None:
|
||||
spots = [(r, loc.label_col)] + [(r, cc) for cc in range(loc.c1, loc.c2 + 1)
|
||||
if grid.get(r, cc) is not None]
|
||||
where = f"第 {r} 行" if r is not None else f"分段「{s.id}」中"
|
||||
how = "时段" if tm.parser == "hour_range" else "文字"
|
||||
col = s.measures.get(label, "")
|
||||
qid = _qid(f"q_text_measure:{s.id}:{label}", block.id, used)
|
||||
reason = (f"这一行每格写的都是{'时段(如「18-19」)' if tm.parser == 'hour_range' else '文字'},不是数字:"
|
||||
f"确认后列「{col}」存成文字列"
|
||||
f"{'(时段存规范写法,「18–19」存成「18-19」)' if tm.parser == 'hour_range' else '(存原文)'},"
|
||||
"不参与数值核对;如果它应该是数字,这份文件无法按当前配方导入")
|
||||
cards.append(Card(f"text_measure:{s.id}:{label}", f"{where}「{label}」按{how}导入", reason,
|
||||
_coords(name, spots), qid))
|
||||
path = ["sheets", si, "blocks", bi, "segments", gi, "text_measures", label]
|
||||
questions.append(Question(qid, f"「{label}」这一行按{how}导入吗",
|
||||
[{"value": "text", "label": f"是,按{how}导入", "needs_reason": False},
|
||||
{"value": "reject", "label": "不是,不导入", "needs_reason": False}],
|
||||
None, {"text": [], "reject": [{"op": "remove", "path": _ptr(*path)}]}))
|
||||
|
||||
# 7 合计段
|
||||
for s in block.segments:
|
||||
if not isinstance(s, DerivedSegment):
|
||||
|
||||
@@ -154,6 +154,14 @@ class DimSpec(_M):
|
||||
derive: dict[str, Literal["start", "end"]] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class TextMeasure(_M):
|
||||
"""按文字导入的指标:这一行每格写的是文字(如「高峰时段」写着「18-19」),落成文字列。空格、公式、占位符照块的
|
||||
取值规则;块的数值类型、千分位规则对它不适用。"""
|
||||
|
||||
#: text:存原文(去首尾空白);hour_range:按时段解析,存规范写法(「18–19」存成「18-19」),解析不了的格是问题
|
||||
parser: Literal["text", "hour_range"] = "text"
|
||||
|
||||
|
||||
class MeasuresSegment(_M):
|
||||
"""每一行是一个指标:落成宽表,一个指标一列。"""
|
||||
|
||||
@@ -166,6 +174,8 @@ class MeasuresSegment(_M):
|
||||
#: derive_tables 按原文查,查不到报 measures_keys_mismatch,绝不产出空列名)。文件里标签的写法变化由
|
||||
#: 执行器按 match_key 认,与这里无关
|
||||
measures: dict[str, str]
|
||||
#: 按文字导入的指标:标签原文(与 measures 的键逐字相同)→ 解析方式。不在这里的指标按块的数值类型导入
|
||||
text_measures: dict[str, TextMeasure] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class DimensionSegment(_M):
|
||||
@@ -479,7 +489,7 @@ class Problem:
|
||||
#: 修复按钮的种类(P3-SPEC 3.2 ①–⑨;期 5 加 declare_subtotals:列表中间像小计的行改作小计核对)。Problem.fix、
|
||||
#: FixProposal.kind、确认项 fix:<目标键> 的 kind 都取自这里
|
||||
FIX_KINDS = ("remove_label", "add_label", "edit_members", "rename_title", "declare_total", "ignore_cells",
|
||||
"declare_placeholder", "declare_hidden", "rename_sheet", "declare_subtotals")
|
||||
"declare_placeholder", "declare_hidden", "rename_sheet", "declare_subtotals", "declare_text_measure")
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -1135,7 +1145,8 @@ def derive_tables(recipe: Recipe) -> tuple[dict[str, list[ColumnOut]], list[Reci
|
||||
problems.append(RecipeProblem(f"{path}/measures", "measures_keys_mismatch",
|
||||
f"分段「{seg.id}」:" + ";".join(parts)))
|
||||
cols = [ColumnOut(axis_name, "TEXT", role="axis")]
|
||||
cols += [ColumnOut(seg.measures.get(label, ""), vtype, header=label, role="measure")
|
||||
cols += [ColumnOut(seg.measures.get(label, ""), "TEXT" if label in seg.text_measures else vtype,
|
||||
header=label, role="measure")
|
||||
for label in seg.labels.expect]
|
||||
put(seg.table, cols, path)
|
||||
elif isinstance(seg, DimensionSegment):
|
||||
|
||||
@@ -488,9 +488,9 @@ def test_closed_value_sets():
|
||||
"header", "row_label", "section_title", "total_word", "axis", "after_title", "outside_text")
|
||||
assert _literal(T.EditResult, "kind") == ("fix", "selection")
|
||||
assert _literal(T.ChangeClass, "change") == ("same", "compatible", "retire", "semantic")
|
||||
assert T.FIX_KINDS[-1] == "declare_subtotals" # 期 5 加在末尾
|
||||
assert T.FIX_KINDS[:-1] == ("remove_label", "add_label", "edit_members", "rename_title", "declare_total",
|
||||
"ignore_cells", "declare_placeholder", "declare_hidden", "rename_sheet")
|
||||
assert T.FIX_KINDS[-2:] == ("declare_subtotals", "declare_text_measure") # 期 5、文字指标依次加在末尾
|
||||
assert T.FIX_KINDS[:-2] == ("remove_label", "add_label", "edit_members", "rename_title", "declare_total",
|
||||
"ignore_cells", "declare_placeholder", "declare_hidden", "rename_sheet")
|
||||
assert T.SELECTION_AS == ("list", "crosstab", "segment", "derived", "section_title", "ignore_rows",
|
||||
"ignore_columns", "ignore_outside")
|
||||
assert T.ACCUMULATE_ACTIONS == ("replace", "first", "append", "replace_period", "restart", "rejected")
|
||||
|
||||
@@ -362,6 +362,20 @@ def c_value_not_number(mp):
|
||||
return _run(_flow(_set(C11=f"约{N}")), FLOW), [f"约{N}", N]
|
||||
|
||||
|
||||
def _text_measure(parser: str) -> dict[str, Any]:
|
||||
data = recipe_dict()
|
||||
data["sheets"][0]["blocks"][0]["segments"][0]["text_measures"] = {"分区甲(人次)": {"parser": parser}}
|
||||
return data
|
||||
|
||||
|
||||
def c_value_not_text(mp):
|
||||
return _run(_flow(_set(C6=N)), _text_measure("text")), [N]
|
||||
|
||||
|
||||
def c_value_not_hour_range(mp):
|
||||
return _run(_flow(_set(C6=f"约{N}", D6="18-19", E6="19-20")), _text_measure("hour_range")), [f"约{N}", N]
|
||||
|
||||
|
||||
def c_value_not_date(mp):
|
||||
return _run(sales_book(edit=lambda ws: ws.cell(4, 3, f"{N}日")), LIST), [f"{N}日", N]
|
||||
|
||||
@@ -438,7 +452,10 @@ NEXT_STEPS = {
|
||||
"value_text_number": "在配方面板的「文本写的数字」中选择「千分位写法按数字保存」",
|
||||
"value_blank": "在配方面板的「空格」中选择「存为空值」",
|
||||
"value_formula": "在配方面板的「公式」中选择「按保存值导入」",
|
||||
"value_not_number": "在配方面板中添加占位符",
|
||||
# 「约987654」含数字、不是整行文字:既不提示加占位符,也不提示改指标类型(另见 test_recipe_text_measures)
|
||||
"value_not_number": "数值区只能是数字或表示无数据的符号",
|
||||
"value_not_text": "在配方面板中把它的「指标类型」改为「数字」",
|
||||
"value_not_hour_range": "在配方面板中把它的「指标类型」改为「文字」",
|
||||
"value_not_integer": "在配方面板中把「类型」改为「小数」",
|
||||
"value_not_date": "在配方面板中把这一列的「类型」改为「文字」",
|
||||
"sheet_unexpected": "在配方面板的「配方没有列出的其他工作表」中选择「记入回执,需确认」",
|
||||
|
||||
@@ -0,0 +1,395 @@
|
||||
"""交叉表里按文字导入的指标(MeasuresSegment.text_measures):契约、静态校验、执行、规则起草、修复按钮、确认项、说明。
|
||||
|
||||
起因:一份客流导出的顶部指标块里夹着「高峰时段」一行,每格写的是当天客流最多的时段(「18-19」),不是数字。
|
||||
以前起草逐格报「不是数字」(一行 30 条),提示去加占位符;照做的话「18-19」整格静默存成空值、干跑照样通过;
|
||||
从分段里去掉这一行再「忽略这些行」又不生效(按标签定位的分段认领连续的数据行)。
|
||||
|
||||
夹具在这里用 openpyxl 现造,名字全是假名(分区甲 / 分区乙):B2 统计期、B3 标题、第 4 行日期、第 5–9 行无标题的
|
||||
指标块(全日、分区甲、分区乙、高峰时段、时段人数)、空行、日间和夜间两段时段客流,没有合计行。第 4 天的时段全是「·」,
|
||||
那天的高峰时段、时段人数也是「·」;第 2 天的高峰时段用 en dash 写(「18–19」)。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import datetime as dt
|
||||
import io
|
||||
import random
|
||||
import sqlite3
|
||||
from typing import Any, Callable
|
||||
|
||||
import pytest
|
||||
from openpyxl import Workbook
|
||||
from openpyxl.utils import get_column_letter
|
||||
|
||||
from app.data import recipe_engine, xlsx_cells, xlsx_scan
|
||||
from app.data import recipe_fixes as F
|
||||
from app.data import recipe_suggest as S
|
||||
from app.data.recipe import apply_patch, canonical_recipe, validate_recipe
|
||||
from app.data.recipe_confirm import ConfirmContext, confirm_items, table_changes
|
||||
from app.data.recipe_types import Recipe, derive_tables
|
||||
|
||||
SHEET = "客流汇总"
|
||||
START = dt.date(2026, 7, 1)
|
||||
DAYS = 8 # 至少 tabular.DATE_HEADER_MIN(7)格日期才按交叉表起草
|
||||
DAY_HOURS = (8, 9, 10, 11)
|
||||
NIGHT_HOURS = (18, 19, 20)
|
||||
GAP = 3 # 第 4 天:时段全是「·」
|
||||
PEAK, PEAK_N = "高峰时段", "时段人数(人次)"
|
||||
|
||||
|
||||
def peak_book(edit: Callable[[Any], None] | None = None) -> tuple[bytes, str]:
|
||||
rnd = random.Random(7)
|
||||
wb = Workbook()
|
||||
ws = wb.active
|
||||
ws.title = SHEET
|
||||
end = START + dt.timedelta(days=DAYS - 1)
|
||||
last = get_column_letter(2 + DAYS)
|
||||
ws["B2"] = f"统计时间范围:{START.year}年{START.month}月{START.day}日至{end.year}年{end.month}月{end.day}日"
|
||||
ws.merge_cells(f"B2:{last}2")
|
||||
ws["B3"] = "某地区客流汇总表"
|
||||
ws.merge_cells(f"B3:{last}3")
|
||||
for j in range(DAYS):
|
||||
d = START + dt.timedelta(days=j)
|
||||
ws.cell(4, 3 + j, f"{d.month}月{d.day}日")
|
||||
hours: dict[int, list[Any]] = {h: [rnd.randint(100, 900) for _ in range(DAYS)] for h in DAY_HOURS + NIGHT_HOURS}
|
||||
zone_a = [rnd.randint(3000, 9000) for _ in range(DAYS)]
|
||||
zone_b = [rnd.randint(3000, 9000) for _ in range(DAYS)]
|
||||
peak: list[Any] = []
|
||||
peak_n: list[Any] = []
|
||||
for j in range(DAYS):
|
||||
if j == GAP:
|
||||
for h in hours:
|
||||
hours[h][j] = "·"
|
||||
peak.append("·")
|
||||
peak_n.append("·")
|
||||
continue
|
||||
h = max(hours, key=lambda k: hours[k][j])
|
||||
peak.append(f"{h}–{h + 1}" if j == 1 else f"{h}-{h + 1}")
|
||||
peak_n.append(hours[h][j])
|
||||
rows: list[tuple[str, list[Any]]] = [
|
||||
("全日客流(人次)", [a + b for a, b in zip(zone_a, zone_b)]), ("分区甲(人次)", zone_a),
|
||||
("分区乙(人次)", zone_b), (PEAK, peak), (PEAK_N, peak_n)]
|
||||
r = 5
|
||||
for label, values in rows:
|
||||
ws.cell(r, 2, label)
|
||||
for j, v in enumerate(values):
|
||||
ws.cell(r, 3 + j, v)
|
||||
r += 1
|
||||
ws.merge_cells(f"B{r}:{last}{r}") # 第 10 行空行
|
||||
r += 1
|
||||
for title, hs in (("日间时段客流(人次)", DAY_HOURS), ("夜间时段客流(人次)", NIGHT_HOURS)):
|
||||
ws.cell(r, 2, title)
|
||||
ws.merge_cells(f"B{r}:{last}{r}")
|
||||
r += 1
|
||||
for h in hs:
|
||||
ws.cell(r, 2, f"{h}-{h + 1}")
|
||||
for j, v in enumerate(hours[h]):
|
||||
ws.cell(r, 3 + j, v)
|
||||
r += 1
|
||||
if edit is not None:
|
||||
edit(ws)
|
||||
bio = io.BytesIO()
|
||||
wb.save(bio)
|
||||
return bio.getvalue(), f"月报导出_{START.isoformat()}_{end.isoformat()}.xlsx"
|
||||
|
||||
|
||||
class Book:
|
||||
def __init__(self, edit: Callable[[Any], None] | None = None):
|
||||
self.raw, self.filename = peak_book(edit)
|
||||
self.scan = xlsx_scan.scan(self.raw)
|
||||
self.grids = {s.name: xlsx_cells.read_grid(self.raw, self.scan, s.name, max_rows=S.DRAFT_MAX_ROWS)
|
||||
for s in self.scan.sheets if s.state == "visible" and s.bounds is not None and s.nonempty}
|
||||
|
||||
def run(self, recipe: dict[str, Any] | Recipe, db: str | None = None):
|
||||
parsed = recipe if isinstance(recipe, Recipe) else Recipe.model_validate(recipe)
|
||||
return recipe_engine.execute(parsed, self.raw, self.filename, self.scan, db)
|
||||
|
||||
def draft(self):
|
||||
return S.draft(self.scan, self.grids, self.filename,
|
||||
validate=lambda r, f, o: validate_recipe(r, facts=f, origin=o), dry_run=self.run)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def drafted():
|
||||
book = Book()
|
||||
d = book.draft()
|
||||
return book, d
|
||||
|
||||
|
||||
def measures_seg(recipe: dict[str, Any]) -> dict[str, Any]:
|
||||
return next(s for s in recipe["sheets"][0]["blocks"][0]["segments"] if s["role"] == "measures")
|
||||
|
||||
|
||||
def full(recipe: dict[str, Any]) -> dict[str, Any]:
|
||||
return Recipe.model_validate(recipe).model_dump(mode="json")
|
||||
|
||||
|
||||
def codes(problems: list[Any]) -> list[str]:
|
||||
return [p.code for p in problems]
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# 规则起草
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
def test_rule_draft_turns_the_text_row_into_a_text_measure(drafted):
|
||||
_book, d = drafted
|
||||
assert d.complete, d.failures
|
||||
assert d.failures == []
|
||||
seg = measures_seg(d.recipe)
|
||||
assert seg["text_measures"] == {PEAK: {"parser": "hour_range"}}
|
||||
assert PEAK in seg["labels"]["expect"] and PEAK_N in seg["labels"]["expect"]
|
||||
# 文字列不写单位;「时段人数(人次)」是数字指标,单位照常拆
|
||||
units = next(t["units"] for t in d.recipe["tables"] if t["name"] == seg["table"])
|
||||
assert PEAK not in units and units.get("时段人数") == "人次"
|
||||
|
||||
|
||||
def test_rule_draft_asks_to_confirm_the_text_measure(drafted):
|
||||
_book, d = drafted
|
||||
seg = measures_seg(d.recipe)
|
||||
card = next(c for c in d.cards if c.id == f"text_measure:{seg['id']}:{PEAK}")
|
||||
q = next(q for q in d.questions if q.id == card.question)
|
||||
assert card.title == f"第 8 行「{PEAK}」按时段导入"
|
||||
assert f"{SHEET}!B8" in card.cells and f"{SHEET}!C8" in card.cells
|
||||
assert [o["value"] for o in q.options] == ["text", "reject"]
|
||||
assert q.effects["text"] == []
|
||||
# 答「不是」:去掉这一条,这一行回到按数字导入,干跑拒收(不会静默存成空值)
|
||||
rejected = apply_patch(full(d.recipe), q.effects["reject"])
|
||||
assert PEAK not in measures_seg(rejected)["text_measures"]
|
||||
book = Book()
|
||||
ex = book.run(rejected)
|
||||
assert not ex.ok and "value_not_number" in codes(ex.problems)
|
||||
|
||||
|
||||
def test_rule_draft_merges_bad_cells_of_one_row_into_one_failure():
|
||||
# 时段人数一行混进两格文字:标签写着单位,是数字指标,照旧起草失败,但一行只报一条,给模型的那条没有格子里的字
|
||||
d = Book(lambda ws: (ws.cell(9, 3, "约3000"), ws.cell(9, 4, "约4000"))).draft()
|
||||
assert not d.complete
|
||||
lines = [f for f in d.failures if f.startswith(f"工作表「{SHEET}」:") and ("第 9 行" in f or "C9" in f)]
|
||||
assert lines == [f"工作表「{SHEET}」:第 9 行「{PEAK_N}」有 2 个单元格不是数字,也不像表示无数据的符号"
|
||||
"(如 C9「约3000」、D9「约4000」)"]
|
||||
assert not any("约3000" in f for f in d.failures_for_model)
|
||||
# 干跑另有一条按分段合并的,下一步不再叫人把「约3000」加成占位符
|
||||
dry = next(f for f in d.failures if "的分段" in f and "C9" in f)
|
||||
assert "添加占位符" not in dry and "请检查原表" in dry
|
||||
assert PEAK not in str(d.failures) # 高峰时段照样按文字指标起草,不报错
|
||||
single = Book(lambda ws: ws.cell(9, 3, "约3000")).draft()
|
||||
assert f"工作表「{SHEET}」:C9 的「约3000」不是数字,也不像表示无数据的符号" in single.failures
|
||||
|
||||
|
||||
def test_text_measure_lands_as_a_text_column(drafted, tmp_path):
|
||||
book, d = drafted
|
||||
db = str(tmp_path / "t.db")
|
||||
ex = book.run(d.recipe, db)
|
||||
assert ex.ok, [(p.code, p.message) for p in ex.problems]
|
||||
seg = measures_seg(d.recipe)
|
||||
table = next(t for t in ex.tables if t.name == seg["table"])
|
||||
types = {c.name: c.type for c in table.columns}
|
||||
assert types["高峰时段"] == "TEXT" and types["时段人数"] == "INTEGER" and types["全日客流"] == "INTEGER"
|
||||
con = sqlite3.connect(db)
|
||||
try:
|
||||
decl = {row[1]: row[2] for row in con.execute(f'PRAGMA table_info("{seg["table"]}")')}
|
||||
rows = con.execute(f'SELECT "日期", "高峰时段", "时段人数" FROM "{seg["table"]}" ORDER BY "日期"').fetchall()
|
||||
hour_rows = con.execute('SELECT COUNT(*) FROM "时段客流"').fetchone()[0]
|
||||
finally:
|
||||
con.close()
|
||||
assert decl["高峰时段"] == "TEXT"
|
||||
assert len(rows) == DAYS and hour_rows == DAYS * (len(DAY_HOURS) + len(NIGHT_HOURS))
|
||||
# en dash 写的存规范写法;整天没有时段数的那天两列都是空值
|
||||
assert rows[1][1] is not None and "–" not in rows[1][1] and rows[1][1].count("-") == 1
|
||||
assert rows[GAP][1:] == (None, None)
|
||||
assert all(isinstance(r[2], int) for i, r in enumerate(rows) if i != GAP)
|
||||
# 高峰时段能和时段客流按规范写法对上
|
||||
con = sqlite3.connect(db)
|
||||
try:
|
||||
hit = con.execute(f'SELECT COUNT(*) FROM "{seg["table"]}" d JOIN "时段客流" h ON h."日期" = d."日期" '
|
||||
'AND h."时段" = d."高峰时段" AND h."客流" = d."时段人数"').fetchone()[0]
|
||||
finally:
|
||||
con.close()
|
||||
assert hit == DAYS - 1
|
||||
assert any(c.column == "高峰时段" and "–" in c.raw for c in ex.canonicalized)
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# 执行器:文字指标的取值、报错
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
def test_number_in_a_text_measure_is_value_not_text(drafted):
|
||||
_, d = drafted
|
||||
book = Book(lambda ws: ws.cell(8, 3, 18))
|
||||
ex = book.run(d.recipe)
|
||||
p = next(p for p in ex.problems if p.code == "value_not_text")
|
||||
assert p.category == "structure" and f"{SHEET}!C8" in p.cells
|
||||
assert "「指标类型」改为「数字」" in p.message
|
||||
assert "18" not in p.model_message.replace(SHEET, "").replace("C8", "")
|
||||
|
||||
|
||||
def test_unparsable_hour_range_and_plain_text_parser(drafted):
|
||||
_, d = drafted
|
||||
book = Book(lambda ws: ws.cell(8, 3, "傍晚"))
|
||||
ex = book.run(d.recipe)
|
||||
p = next(p for p in ex.problems if p.code == "value_not_hour_range")
|
||||
assert "「指标类型」改为「文字」" in p.message and "傍晚" not in p.model_message
|
||||
as_text = full(d.recipe)
|
||||
measures_seg(as_text)["text_measures"][PEAK] = {"parser": "text"}
|
||||
ex2 = book.run(as_text)
|
||||
assert ex2.ok, [(p.code, p.message) for p in ex2.problems]
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# 没声明成文字指标时:报错的下一步和修复按钮
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
def _without_text_measure(d: Any) -> dict[str, Any]:
|
||||
r = full(d.recipe)
|
||||
measures_seg(r)["text_measures"] = {}
|
||||
return r
|
||||
|
||||
|
||||
def test_not_number_on_a_text_row_points_at_measure_kind_not_placeholder(drafted):
|
||||
book, d = drafted
|
||||
ex = book.run(_without_text_measure(d))
|
||||
p = next(p for p in ex.problems if p.code == "value_not_number")
|
||||
assert f"「{PEAK}」这一行写的都是文字" in p.message
|
||||
assert "「指标类型」改为「文字」或「时段」" in p.message
|
||||
assert "添加占位符" not in p.message
|
||||
seg = measures_seg(d.recipe)
|
||||
assert p.fix_args["texts"] == []
|
||||
assert p.fix_args["text_rows"] == [{"segment": seg["id"], "label": PEAK, "cell": f"{SHEET}!B8",
|
||||
"parsers": ["hour_range", "text"]}]
|
||||
|
||||
|
||||
def test_not_number_tail_depends_on_what_the_cells_are(drafted):
|
||||
_, d = drafted
|
||||
# 不含数字的短符号:照旧提示加占位符
|
||||
ex = Book(lambda ws: ws.cell(5, 3, "—")).run(d.recipe)
|
||||
p = next(p for p in ex.problems if p.code == "value_not_number")
|
||||
assert p.message.endswith("如果它表示无数据,请在配方面板中添加占位符")
|
||||
# 含数字、又不是整行文字:不再提示加占位符
|
||||
ex = Book(lambda ws: ws.cell(5, 3, "约3000")).run(d.recipe)
|
||||
p = next(p for p in ex.problems if p.code == "value_not_number")
|
||||
assert "占位符" not in p.message.split("。", 1)[1].replace("表示无数据的符号", "")
|
||||
assert "请检查原表" in p.message and "text_rows" not in p.fix_args
|
||||
# 标签写着单位的指标整行是文字:是数字指标,不提示改指标类型
|
||||
ex = Book(lambda ws: [ws.cell(9, c, "18-19") for c in range(3, 3 + DAYS)]).run(d.recipe)
|
||||
p = next(p for p in ex.problems if p.code == "value_not_number")
|
||||
assert "指标类型" not in p.message and "text_rows" not in p.fix_args
|
||||
|
||||
|
||||
def test_fix_button_declares_the_text_measure(drafted):
|
||||
book, d = drafted
|
||||
work = _without_text_measure(d)
|
||||
ex = book.run(work)
|
||||
problems = [p.__dict__ for p in ex.problems]
|
||||
props = F.propose_fixes(work, problems, [], d.facts)
|
||||
p = next(x for x in props if x.kind == "declare_text_measure")
|
||||
assert [o.value for o in p.options] == ["hour_range", "text"]
|
||||
assert all(o.breaking for o in p.options)
|
||||
assert p.anchor.index == next(i for i, x in enumerate(problems) if x["code"] == "value_not_number")
|
||||
res = F.fix_ops(work, p, "hour_range", None)
|
||||
assert res.ok and "按时段导入" in res.title
|
||||
new = apply_patch(work, res.ops)
|
||||
assert measures_seg(new)["text_measures"] == {PEAK: {"parser": "hour_range"}}
|
||||
parsed, probs = validate_recipe(new, facts=d.facts, origin="manual")
|
||||
assert probs == [] and parsed is not None
|
||||
assert book.run(parsed).ok
|
||||
# 已经按文字导入的不再提议
|
||||
assert not [x for x in F.propose_fixes(new, problems, [], d.facts) if x.kind == "declare_text_measure"]
|
||||
|
||||
|
||||
def test_remove_label_also_drops_the_text_measure(drafted):
|
||||
_, d = drafted
|
||||
work = full(d.recipe)
|
||||
seg = measures_seg(work)
|
||||
prob = {"code": "label_missing", "fix": "remove_label", "cells": [],
|
||||
"fix_args": {"segment": seg["id"], "labels": [PEAK], "role": "measures"}}
|
||||
p = next(x for x in F.propose_fixes(work, [prob], [], d.facts) if x.kind == "remove_label")
|
||||
new = apply_patch(work, F.fix_ops(work, p, p.options[0].value, None).ops)
|
||||
assert PEAK not in measures_seg(new)["text_measures"]
|
||||
_parsed, probs = validate_recipe(new, facts=d.facts, origin="manual")
|
||||
assert "text_measure_unknown" not in codes(probs)
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# 静态校验、契约
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
def test_static_checks_on_text_measures(drafted):
|
||||
_, d = drafted
|
||||
r = full(d.recipe)
|
||||
seg = measures_seg(r)
|
||||
seg["text_measures"]["不存在的指标"] = {"parser": "text"}
|
||||
seg["text_measures"][PEAK_N] = {"parser": "text"}
|
||||
table = next(t for t in r["tables"] if t["name"] == seg["table"])
|
||||
table["units"]["高峰时段"] = "人次"
|
||||
_parsed, probs = validate_recipe(r, facts=d.facts, origin="manual")
|
||||
got = {(p.code, p.path) for p in probs}
|
||||
sp = "/sheets/0/blocks/0/segments/0"
|
||||
assert ("text_measure_unknown", f"{sp}/text_measures/不存在的指标") in got
|
||||
assert ("text_measure_unit", f"{sp}/text_measures/{PEAK_N}") in got
|
||||
assert any(code == "unit_on_text" and path.endswith("/units/高峰时段") for code, path in got)
|
||||
|
||||
|
||||
def test_placeholder_with_digits_is_rejected_except_on_replay_or_when_carried(drafted):
|
||||
_, d = drafted
|
||||
r = full(d.recipe)
|
||||
measures_seg(r)["text_measures"] = {}
|
||||
r["sheets"][0]["blocks"][0]["values"]["placeholders"].append({"text": "18-19", "meaning": "无数据"})
|
||||
_parsed, probs = validate_recipe(r, facts=d.facts, origin="manual")
|
||||
p = next(p for p in probs if p.code == "placeholder_digits")
|
||||
assert p.path == "/sheets/0/blocks/0/values/placeholders/1/text"
|
||||
assert "「指标类型」改为「文字」或「时段」" in p.message
|
||||
assert "placeholder_digits" not in codes(validate_recipe(r, origin="replay")[1])
|
||||
base = Recipe.model_validate(copy.deepcopy(r))
|
||||
assert "placeholder_digits" not in codes(validate_recipe(r, facts=d.facts, origin="manual", base=base)[1])
|
||||
|
||||
|
||||
def test_contract_text_measures_are_text_columns_and_hash_compatible(drafted):
|
||||
_, d = drafted
|
||||
parsed = Recipe.model_validate(d.recipe)
|
||||
cols, problems = derive_tables(parsed)
|
||||
seg = measures_seg(d.recipe)
|
||||
assert problems == []
|
||||
assert {c.name: c.type for c in cols[seg["table"]]}["高峰时段"] == "TEXT"
|
||||
# 默认值(没有文字指标)不进规范形式:已存配方的哈希和构建 id 不变
|
||||
plain = full(d.recipe)
|
||||
measures_seg(plain)["text_measures"] = {}
|
||||
assert "text_measures" not in str(canonical_recipe(Recipe.model_validate(plain)))
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# 确认项、前后对照、说明
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
def test_confirm_item_and_breaking_changes(drafted):
|
||||
book, d = drafted
|
||||
ex = book.run(d.recipe)
|
||||
seg = measures_seg(d.recipe)
|
||||
items = {i.id: i for i in confirm_items(ConfirmContext(kind="first", recipe=d.recipe, extraction=ex, checks=[]))}
|
||||
item = items[f"text_measure:{seg['id']}:{PEAK}"]
|
||||
assert "按时段导入" in item.label and "文字列" in item.label
|
||||
as_text = full(d.recipe)
|
||||
measures_seg(as_text)["text_measures"][PEAK] = {"parser": "text"}
|
||||
kinds = {(c.kind, c.column) for c in table_changes(d.recipe, as_text)}
|
||||
assert ("store", "高峰时段") in kinds
|
||||
plain = full(d.recipe)
|
||||
measures_seg(plain)["text_measures"] = {}
|
||||
assert ("type", "高峰时段") in {(c.kind, c.column) for c in table_changes(d.recipe, plain)}
|
||||
|
||||
|
||||
def test_column_note_explains_nulls_of_the_text_measure(drafted, tmp_path):
|
||||
from app.data.recipe_checks import run_checks
|
||||
from app.data.recipe_notes import build_notes
|
||||
|
||||
book, d = drafted
|
||||
db = str(tmp_path / "n.db")
|
||||
ex = book.run(d.recipe, db)
|
||||
parsed = Recipe.model_validate(d.recipe)
|
||||
notes = build_notes(parsed, ex, run_checks(db, ex, parsed), [])
|
||||
seg = measures_seg(d.recipe)
|
||||
assert "空值" in notes.tables[seg["table"]].columns.get("高峰时段", "")
|
||||
@@ -1627,6 +1627,8 @@ export const RECIPE_TEXT = {
|
||||
derivedSegment: (id: string) => `合计段「${id}」`,
|
||||
tableName: '表名',
|
||||
columnName: '列名',
|
||||
/** measures 段每个指标的「指标类型」:数字按块的「类型」导入;文字、时段落成文字列(执行器的报错里写的就是这个字段名) */
|
||||
measureKind: '指标类型',
|
||||
unit: '单位',
|
||||
noUnit: '无单位',
|
||||
grain: '主键',
|
||||
@@ -1909,6 +1911,7 @@ export const FIX_KIND_LABEL: Record<string, string> = {
|
||||
rename_title: '改分段标题',
|
||||
declare_total: '改作合计核对',
|
||||
declare_subtotals: '改作小计核对',
|
||||
declare_text_measure: '按文字导入这一行',
|
||||
ignore_cells: '按文字忽略',
|
||||
declare_placeholder: '声明占位符',
|
||||
declare_hidden: '设置隐藏行处理',
|
||||
@@ -2152,6 +2155,7 @@ export const RECIPE_ORIGIN_LABEL: Record<string, string> = {
|
||||
export const RECIPE_CHOICE_LABEL = {
|
||||
locateBy: { labels: '按标签', section_title: '按分段标题' } as Record<string, string>,
|
||||
dimParser: { hour_range: '时段(如 7-8)', text: '文字' } as Record<string, string>,
|
||||
measureKind: { '': '数字', hour_range: '时段', text: '文字' } as Record<string, string>,
|
||||
valueType: { INTEGER: '整数', REAL: '小数' } as Record<string, string>,
|
||||
blankCross: { reject: '拒收', null: '存为空值(需确认)' } as Record<string, string>,
|
||||
blankList: { null: '存为空值', reject: '拒收' } as Record<string, string>,
|
||||
|
||||
@@ -130,8 +130,9 @@ export function tableColumns(recipe: Recipe | null): Map<string, RecipeColumn[]>
|
||||
for (const seg of arr(block.segments)) {
|
||||
if (!isObj(seg)) continue
|
||||
if (seg.role === 'measures') {
|
||||
const text = isObj(seg.text_measures) ? seg.text_measures : {}
|
||||
put(seg.table, [axis, ...arr(seg.labels?.expect).map((label: string) => ({
|
||||
name: String(seg.measures?.[label] ?? ''), type: vtype, role: 'measure', header: label,
|
||||
name: String(seg.measures?.[label] ?? ''), type: isObj(text[label]) ? 'TEXT' : vtype, role: 'measure', header: label,
|
||||
}))])
|
||||
} else if (seg.role === 'dimension') {
|
||||
put(seg.table, [
|
||||
@@ -568,6 +569,23 @@ function DeriveNames({ path, derive, onRename }: { path: string; derive: unknown
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* 指标改按文字(text / hour_range)或改回数字(空串)。文字列不能有单位:改成文字时去掉这一列的单位,改回数字时按标签
|
||||
* 末尾的括号补回(与加标签时一致)
|
||||
*/
|
||||
function setMeasureKind(r: Recipe, x: any, label: string, kind: string, units: string[]) {
|
||||
const col = x.measures?.[label]
|
||||
const table = arr(r.tables).find((t) => t?.name === x.table)
|
||||
if (kind) {
|
||||
x.text_measures = { ...(isObj(x.text_measures) ? x.text_measures : {}), [label]: { parser: kind } }
|
||||
if (table && col && isObj(table.units)) delete table.units[col]
|
||||
return
|
||||
}
|
||||
if (isObj(x.text_measures)) delete x.text_measures[label]
|
||||
const [, unit] = splitUnit(label, units)
|
||||
if (unit && table && col) table.units = { ...(isObj(table.units) ? table.units : {}), [col]: unit }
|
||||
}
|
||||
|
||||
function SegmentEditor({ s, b, g, seg, edit, candidates, labelOptions, units }: {
|
||||
s: number; b: number; g: number; seg: any; edit: Edit
|
||||
candidates: Record<string, string[]>; labelOptions: string[]; units: string[]
|
||||
@@ -596,6 +614,7 @@ function SegmentEditor({ s, b, g, seg, edit, candidates, labelOptions, units }:
|
||||
if (x.role === 'measures' && isObj(x.measures)) {
|
||||
const col = x.measures[label]
|
||||
delete x.measures[label]
|
||||
if (isObj(x.text_measures)) delete x.text_measures[label]
|
||||
const table = arr(r.tables).find((t) => t?.name === x.table)
|
||||
if (table && col) {
|
||||
if (isObj(table.units)) delete table.units[col]
|
||||
@@ -644,7 +663,14 @@ function SegmentEditor({ s, b, g, seg, edit, candidates, labelOptions, units }:
|
||||
if (old) renameColumn(r, x.table, old, v)
|
||||
else x.measures = { ...(isObj(x.measures) ? x.measures : {}), [label]: v }
|
||||
})} />
|
||||
<select className="field !w-20" data-field={`${base}/text_measures/${ptr(label)}`}
|
||||
value={String(seg.text_measures?.[label]?.parser ?? '')}
|
||||
aria-label={`${RECIPE_TEXT.measureKind}:${label}`} title={RECIPE_TEXT.measureKind}
|
||||
onChange={(e) => edit((r) => { setMeasureKind(r, at(r), label, e.target.value, units) })}>
|
||||
{Object.entries(RECIPE_CHOICE_LABEL.measureKind).map(([v, l]) => <option key={v} value={v}>{l}</option>)}
|
||||
</select>
|
||||
<FieldProblems path={`${base}/measures/${ptr(label)}`} />
|
||||
<FieldProblems path={`${base}/text_measures/${ptr(label)}`} />
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
|
||||
@@ -800,10 +800,11 @@ export interface CurrentRecipe {
|
||||
// 和接口的 JSON 同名(Selection 的 as 除外:后端 dataclass 叫 as_,JSON 里是 as)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** 修复按钮的九种(契约 FIX_KINDS) */
|
||||
/** 修复按钮的种类(契约 FIX_KINDS) */
|
||||
export type FixKind =
|
||||
| 'remove_label' | 'add_label' | 'edit_members' | 'rename_title' | 'declare_total' | 'ignore_cells'
|
||||
| 'declare_placeholder' | 'declare_hidden' | 'rename_sheet' | (string & {})
|
||||
| 'declare_placeholder' | 'declare_hidden' | 'rename_sheet' | 'declare_subtotals' | 'declare_text_measure'
|
||||
| (string & {})
|
||||
|
||||
/** 修复提议的一个封闭选项。效果由服务端算:界面只发 fix_id、value 和理由 */
|
||||
export interface FixOption {
|
||||
|
||||
@@ -2528,13 +2528,34 @@ await section('数据 · 表格:按配方导入', async () => {
|
||||
check('勾选「没有小计行」:发 PUT,no_subtotals 为 true、没有配小计', noSub?.no_subtotals === true && !noSub?.subtotals,
|
||||
JSON.stringify(noSub ?? {}))
|
||||
await page.waitForTimeout(300)
|
||||
// 指标类型:整行是文字的指标(如「高峰时段」写着「18-19」)改按时段 / 文字导入,落成文字列、去掉单位;改回数字时按标签补回单位
|
||||
const kindSel = panel.locator('[data-field="/sheets/0/blocks/0/segments/0/text_measures/分区乙(人次)"]')
|
||||
const kindOptions = await kindSel.locator('option').evaluateAll((os) => os.map((o) => `${o.value}:${o.textContent}`))
|
||||
check('指标类型是下拉框:数字、时段、文字', JSON.stringify(kindOptions) === JSON.stringify([':数字', 'hour_range:时段', 'text:文字']),
|
||||
kindOptions.join(','))
|
||||
replies.recipe.push(echo())
|
||||
await kindSel.selectOption('hour_range')
|
||||
await until(async () => puts().length === 7, 4000)
|
||||
const k1 = puts()[6]?.body?.recipe
|
||||
check('改成「时段」:发 PUT,text_measures 写上 hour_range,这一列的单位去掉',
|
||||
k1?.sheets?.[0]?.blocks?.[0]?.segments?.[0]?.text_measures?.['分区乙(人次)']?.parser === 'hour_range'
|
||||
&& !('分区乙' in (k1?.tables?.[0]?.units ?? {})), JSON.stringify(k1?.tables?.[0]?.units ?? {}))
|
||||
await page.waitForTimeout(300)
|
||||
replies.recipe.push(echo())
|
||||
await kindSel.selectOption('')
|
||||
await until(async () => puts().length === 8, 4000)
|
||||
const k2 = puts()[7]?.body?.recipe
|
||||
check('改回「数字」:发 PUT,text_measures 去掉这一项,单位按标签补回「人次」',
|
||||
!('分区乙(人次)' in (k2?.sheets?.[0]?.blocks?.[0]?.segments?.[0]?.text_measures ?? {}))
|
||||
&& k2?.tables?.[0]?.units?.['分区乙'] === '人次', JSON.stringify(k2?.tables?.[0]?.units ?? {}))
|
||||
await page.waitForTimeout(300)
|
||||
// 有问题的配方:一条对得上字段(单位),一条对不上(挂到面板顶部)
|
||||
replies.recipe.push(echo({ recipe_problems: [
|
||||
{ path: '/tables/0/units/分区甲', code: 'unit_label_conflict', message: '表「日客流量」:列「分区甲」的单位与标签「分区甲(人次)」不一致' },
|
||||
{ path: '/sheets/0/blocks/0/label_offset', code: 'schema', message: '第 1 个工作表的交叉表:标签列偏移只能是紧挨日期的左侧一列' },
|
||||
] }))
|
||||
await panel.locator('[data-field="/tables/0/units/分区甲"]').selectOption('人')
|
||||
await until(async () => puts().length === 7, 4000)
|
||||
await until(async () => puts().length === 9, 4000)
|
||||
await page.waitForTimeout(300)
|
||||
const trialBtn = wizard().locator('[data-trial-run]')
|
||||
check('配方有问题时「试运行」禁用', await trialBtn.isDisabled())
|
||||
@@ -2548,8 +2569,8 @@ await section('数据 · 表格:按配方导入', async () => {
|
||||
await panel.locator('[data-recipe-paste-toggle]').click()
|
||||
await panel.locator('[data-recipe-paste] textarea').fill(JSON.stringify(FLOW_RECIPE))
|
||||
await panel.locator('[data-recipe-paste] button', { hasText: '保存配方' }).click()
|
||||
await until(async () => puts().length === 8, 4000)
|
||||
check('「粘贴配方」发 PUT,body 就是粘贴的配方', JSON.stringify(puts()[7]?.body?.recipe) === JSON.stringify(FLOW_RECIPE))
|
||||
await until(async () => puts().length === 10, 4000)
|
||||
check('「粘贴配方」发 PUT,body 就是粘贴的配方', JSON.stringify(puts()[9]?.body?.recipe) === JSON.stringify(FLOW_RECIPE))
|
||||
await page.waitForTimeout(300)
|
||||
check('……问题清掉以后「试运行」可用', await trialBtn.isEnabled())
|
||||
// 连续修改:上一次 PUT 还没回来就接着改。上一次回来时,排在后面的修改不能被服务端那份盖掉,
|
||||
|
||||
Reference in New Issue
Block a user