From b988a5ae18ded443979eaf07a613814f82e9e567 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Tue, 13 Oct 2026 15:36:24 +0800 Subject: [PATCH] =?UTF-8?q?=E8=B5=B7=E8=8D=89=E5=BC=95=E6=93=8E=E5=92=8C?= =?UTF-8?q?=E5=90=8E=E5=8F=B0=E4=BB=BB=E5=8A=A1=E7=9A=84=E5=87=A0=E5=A4=84?= =?UTF-8?q?=E6=94=B6=E7=B4=A7=EF=BC=9A=E6=8F=90=E7=A4=BA=E8=AF=8D=E8=A6=81?= =?UTF-8?q?=E6=B1=82=E8=AF=B7=E5=86=99=E7=9A=84=E6=AF=8F=E4=B8=80=E5=88=97?= =?UTF-8?q?=E9=83=BD=E5=88=97=E5=87=BA=E6=9D=A5=EF=BC=88=E6=8B=BF=E4=B8=8D?= =?UTF-8?q?=E5=87=86=E7=9A=84=E5=AD=97=E6=AE=B5=E7=95=99=E7=A9=BA=E3=80=81?= =?UTF-8?q?=E5=88=97=E4=B8=8D=E6=BC=8F=EF=BC=89=EF=BC=8C=E5=85=8D=E5=BE=97?= =?UTF-8?q?=E6=A8=A1=E5=9E=8B=E7=95=99=E7=A9=BA=E7=9A=84=E5=88=97=E5=86=8D?= =?UTF-8?q?=E8=A2=AB=E8=A1=A5=E9=97=AE=EF=BC=9B=E7=BA=AF=E6=96=87=E6=9C=AC?= =?UTF-8?q?=E9=82=A3=E6=9D=A1=E8=B7=AF=E7=9A=84=E6=A0=BC=E5=BC=8F=E7=A4=BA?= =?UTF-8?q?=E4=BE=8B=E6=94=B9=E7=94=A8=E3=80=8C=E8=A1=A8=E7=B1=BB=E5=9E=8B?= =?UTF-8?q?=E3=80=8D=E3=80=8C=E5=BA=A6=E9=87=8F=E7=B1=BB=E5=9E=8B=E3=80=8D?= =?UTF-8?q?=E5=8D=A0=E4=BD=8D=EF=BC=8C=E4=B8=8D=E8=AE=A9=E6=A8=A1=E5=9E=8B?= =?UTF-8?q?=E7=85=A7=E6=8A=84=E7=A4=BA=E4=BE=8B=E5=80=BC=EF=BC=9B=E4=BB=8E?= =?UTF-8?q?=E6=AD=A3=E6=96=87=E5=8F=96=20JSON=20=E6=97=B6=E4=BE=9D?= =?UTF-8?q?=E6=AC=A1=E8=AF=95=E5=A4=9A=E4=B8=AA=E6=8B=AC=E5=8F=B7=E4=BD=9C?= =?UTF-8?q?=E5=BC=80=E5=A4=B4=EF=BC=8C=E8=AF=B4=E6=98=8E=E6=96=87=E5=AD=97?= =?UTF-8?q?=E9=87=8C=E5=B8=A6=E6=8B=AC=E5=8F=B7=E3=80=81=E5=A4=96=E9=9D=A2?= =?UTF-8?q?=E5=A4=9A=E5=8C=85=E4=B8=80=E5=B1=82=E9=83=BD=E8=AE=A4=E5=BE=97?= =?UTF-8?q?=EF=BC=9B=E7=94=A8=E9=87=8F=E8=AE=B0=E5=9C=A8=E5=8D=95=E6=AC=A1?= =?UTF-8?q?=E8=B0=83=E7=94=A8=E7=9A=84=E6=97=B6=E9=99=90=E4=B9=8B=E5=A4=96?= =?UTF-8?q?=EF=BC=8C=E8=90=BD=E5=BA=93=E6=85=A2=E4=BA=86=E4=B8=8D=E4=BC=9A?= =?UTF-8?q?=E6=8A=8A=E5=B7=B2=E7=BB=8F=E5=9B=9E=E6=9D=A5=E7=9A=84=E8=B0=83?= =?UTF-8?q?=E7=94=A8=E7=AE=97=E6=88=90=E8=B6=85=E6=97=B6=EF=BC=9B=E5=90=8E?= =?UTF-8?q?=E5=8F=B0=E4=BB=BB=E5=8A=A1=E6=94=B6=E5=B0=BE=E5=86=99=E5=BA=93?= =?UTF-8?q?=E5=87=BA=E9=94=99=E5=8F=AA=E8=AE=B0=E6=97=A5=E5=BF=97=EF=BC=9B?= =?UTF-8?q?=E8=A1=A5=E4=B8=8A=E4=BB=8E=E6=AD=A3=E6=96=87=E5=8F=96=20JSON?= =?UTF-8?q?=E3=80=81=E4=BB=BB=E5=8A=A1=E5=BC=80=E5=A7=8B=E6=97=B6=E6=A8=A1?= =?UTF-8?q?=E5=9E=8B=E5=B7=B2=E7=94=A8=E4=B8=8D=E4=BA=86=EF=BC=88=E6=95=B4?= =?UTF-8?q?=E4=BD=93=20failed=EF=BC=89=E7=9A=84=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/data/catalog.py | 7 +-- backend/app/data/catalog_draft.py | 61 +++++++++++++--------- backend/app/data/catalog_jobs.py | 24 ++++++--- backend/tests/test_catalog_draft_engine.py | 38 ++++++++++++++ backend/tests/test_catalog_draft_jobs.py | 23 ++++++++ 5 files changed, 117 insertions(+), 36 deletions(-) diff --git a/backend/app/data/catalog.py b/backend/app/data/catalog.py index c48b99bc..45eec1bb 100644 --- a/backend/app/data/catalog.py +++ b/backend/app/data/catalog.py @@ -1449,6 +1449,7 @@ status 状态;attribute 属性)。 规矩: - 只写数据事实,不写计算公式,不写指标怎么算。 - 拿不准的字段留空,不要猜。列名、表名照原样写,不要翻译或改写。 +- 请你写的每一列都在 columns 里写一项,拿不准的字段可以留空,但不要漏掉列:漏掉的列会被再问一次。 - 标着「系统说明」的内容是导入时按核对结果生成的,以它为准:不要重复,也不要与之矛盾。 - 标着「已确认」的目录项由人工确认过,以它为准。 - 表名下一行写着「这次只写……」的表,只写它说的那部分:只写表级的项时不写列,只写列出的几列时不写表级的项、\ @@ -1464,10 +1465,10 @@ CATALOG_DRAFT_BACKGROUND_PROMPT = "使用者补充的背景和要求:\n{lines} #: 纯文本那条路(结构化输出用不了时):没有 JSON Schema 约束,要把格式说清 CATALOG_DRAFT_TEXT_PROMPT = """只输出一个 JSON 对象,不要输出别的文字,格式如下(拿不准的字段不写): -{"tables": [{"table": "表名", "label": "中文名", "grain": "一行代表什么", "kind": "fact", "keys": ["列名"], \ +{"tables": [{"table": "表名", "label": "中文名", "grain": "一行代表什么", "kind": "表类型", "keys": ["列名"], \ "business_date": {"column": "列名", "rule": "一句规则", "timezone": "时区"}, \ -"columns": [{"name": "列名", "label": "中文名", "meaning": "含义", "unit": "单位", "measure": "flow"}]}]} -kind 取 fact、dimension、snapshot、log、config 之一;measure 取 flow、stock、ratio、identifier、status、attribute 之一。""" +"columns": [{"name": "列名", "label": "中文名", "meaning": "含义", "unit": "单位", "measure": "度量类型"}]}]} +表类型写 fact、dimension、snapshot、log、config 之一;度量类型写 flow、stock、ratio、identifier、status、attribute 之一。""" #: 拼表结构描述用的固定片段(写给模型) CATALOG_DRAFT_PROMPT_LABELS = { diff --git a/backend/app/data/catalog_draft.py b/backend/app/data/catalog_draft.py index 2b1d71bf..8833add9 100644 --- a/backend/app/data/catalog_draft.py +++ b/backend/app/data/catalog_draft.py @@ -347,15 +347,18 @@ def _array(text: str, i: int, depth: int) -> tuple[Any, int, bool]: out.append(value) -def _json_prefix(text: str) -> tuple[Any, bool] | None: - """正文里的第一个 JSON 值:(值, 写完了没有)。前面可以有一句话、围栏,后面可以有别的字;不是 JSON 返回 None。""" - for start in sorted(k for k in (text.find("{"), text.find("[")) if k >= 0): - try: - value, _, complete = _value(text, start, 0) - except (_Bad, RecursionError): - continue - return value, complete - return None +#: 正文里最多试几个 { 或 [ 作为 JSON 的开头(前面的说明文字里也可能有括号) +_MAX_STARTS = 16 + + +def _starts(text: str) -> list[int]: + out: list[int] = [] + for i, ch in enumerate(text): + if ch in "{[": + out.append(i) + if len(out) >= _MAX_STARTS: + break + return out def _entry(item: Any) -> dict[str, Any] | None: @@ -398,16 +401,22 @@ def _tables_of(root: Any) -> list[dict[str, Any]] | None: def _parse_text(text: str, cut: bool) -> _Reply | None: - found = _json_prefix(text) - if found is None: - return None - root, complete = found - entries = _tables_of(root) - if entries is None: - return None - if complete: - return _Reply(entries=entries) - return _Reply(entries=entries, truncated=True, exhausted=cut) + """正文里的表对象。前面可以有一句话、围栏,后面可以有别的字;JSON 没写完的(截断)只收已经写完的部分。 + 认不出约定的格式返回 None。""" + empty: _Reply | None = None + for start in _starts(text): + try: + root, _, complete = _value(text, start, 0) + except (_Bad, RecursionError): + continue + entries = _tables_of(root) + if entries is None: + continue + reply = _Reply(entries=entries) if complete else _Reply(entries=entries, truncated=True, exhausted=cut) + if entries: + return reply + empty = empty or reply # {} 或 {"tables": []}:后面还可能有真的那一份 + return empty def _cut_off(raw: Any) -> bool: @@ -738,7 +747,10 @@ class _Run: try: async with model_gate(): await self._hook(self.hooks.call_started, [p.table for p in pieces]) - return await asyncio.wait_for(self._call(pieces, mode), timeout=C.MODEL_TIMEOUT_S) + reply, raw = await asyncio.wait_for(self._call(pieces, mode), timeout=C.MODEL_TIMEOUT_S) + # 用量记在时限之外:落库慢了不能把已经回来的这次调用算成超时 + await self._usage(raw) + return reply except asyncio.CancelledError: raise except Exception as e: # noqa: BLE001 - 按原因分别处理 @@ -760,16 +772,15 @@ class _Run: continue return _Reply(error=self._call_error(e, kind, retried)) - async def _call(self, pieces: list[_Piece], mode: str) -> _Reply: + async def _call(self, pieces: list[_Piece], mode: str) -> tuple[_Reply, Any]: + """一次调用:(读好的回复, 原始回复)。原始回复给用量用(测试的假模型没有,是 None)。""" messages = self._messages(pieces, mode) if mode == "structured": runnable = self.model.with_structured_output(C.CATALOG_DRAFT_PROMPT_SCHEMA, include_raw=True) raw, parsed = _unpack(await runnable.ainvoke(messages)) - await self._usage(raw) - return _structured_reply(raw, parsed) + return _structured_reply(raw, parsed), raw raw = await self.model.ainvoke(messages) - await self._usage(raw) - return _text_reply(raw) + return _text_reply(raw), raw async def _length_reply(self, e: BaseException) -> _Reply: """LengthFinishReasonError:额度用完了。异常里带着那次的回复,能捡回来的照捡。""" diff --git a/backend/app/data/catalog_jobs.py b/backend/app/data/catalog_jobs.py index 4d1d244f..f38c08db 100644 --- a/backend/app/data/catalog_jobs.py +++ b/backend/app/data/catalog_jobs.py @@ -407,21 +407,29 @@ class _JobRun: current = asyncio.current_task() if current is not None: current.uncancel() - await self._settle() if self.stop_reason == "user": - await self._write(_close, self.job_id, "cancelled", None, "cancelled", None) + await self._end("cancelled", None, "cancelled", None) else: - await self._write(_close, self.job_id, "interrupted", INTERRUPTED_ERROR, "cancelled", INTERRUPTED_ERROR) + await self._end("interrupted", INTERRUPTED_ERROR, "cancelled", INTERRUPTED_ERROR) except _JobFailed as e: - await self._settle() - await self._write(_close, self.job_id, "failed", str(e), "failed", str(e)) + await self._end("failed", str(e), "failed", str(e)) except Exception as e: # noqa: BLE001 - 任务整体出错:照实记下,不留在「起草中」 from app.core.errors import describe_exception, first_line logger.exception("数据目录起草任务 %s 出错", self.job_id) - await self._settle() reason = JOB_ERROR.format(line=first_line(e) or describe_exception(e)) - await self._write(_close, self.job_id, "failed", reason, "failed", reason) + await self._end("failed", reason, "failed", reason) + + async def _end(self, job_status: str, job_error: str | None, task_status: str, task_error: str | None) -> None: + """停止、出错时收尾:等手上的库写入写完,再把没结束的表和任务记下来。收尾本身写不进去只记日志(下次启动时 + startup 会把它标成中断)。""" + try: + await self._settle() + await self._write(_close, self.job_id, job_status, job_error, task_status, task_error) + except asyncio.CancelledError: + raise + except Exception: # noqa: BLE001 + logger.exception("数据目录起草任务 %s 收尾时写库出错", self.job_id) async def _run(self) -> None: if not await self._write(self._claim): @@ -617,7 +625,7 @@ class DraftJobs: if run is not None: await asyncio.wait([run.task], timeout=timeout) # type: ignore[list-item] - async def shutdown(self, *, grace: float = 5.0) -> None: + async def shutdown(self, *, grace: float = 3.0) -> None: """关停:停掉进行中的任务(标中断),最多等 grace 秒收尾。""" runs = [run for job_id in list(self._runs) if (run := self._alive(job_id)) is not None] for run in runs: diff --git a/backend/tests/test_catalog_draft_engine.py b/backend/tests/test_catalog_draft_engine.py index a1ac2272..2818e566 100644 --- a/backend/tests/test_catalog_draft_engine.py +++ b/backend/tests/test_catalog_draft_engine.py @@ -553,3 +553,41 @@ def test_budget_words_match_the_other_drafts(): from app.data import recipe_ai assert D.BUDGET_WORDS == recipe_ai._BUDGET_WORDS + + +# -------------------------------------------------------------------------- +# 从正文里取 JSON +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("text, tables, truncated", [ + # 前面有一句话、带括号的说明、围栏 + ('按 {table} 的格式写好了:\n```json\n{"tables": [{"table": "a", "label": "甲"}]}\n```', ["a"], False), + ('[说明] 草稿:{"tables": [{"table": "a"}]} 以上', ["a"], False), + # 外面多包了一层 + ('{"result": {"tables": [{"table": "a"}, {"table": "b"}]}}', ["a", "b"], False), + # 截断在字符串、数字、字面量、键名中间:只收写完了的 + ('{"tables": [{"table": "a", "label": "甲"}, {"table": "b", "label": "乙', ["a", "b"], True), + ('{"tables": [{"table": "a", "columns": [{"name": "x", "label": "某"}, {"name": "y", "len": 12', ["a"], True), + ('{"tables": [{"table": "a", "keys": ["x", "y"', ["a"], True), + ('{"tables": [{"table": "a"}, {"tab', ["a"], True), + ('{"tables": [{"table": "a", "flag": tr', ["a"], True), +]) +def test_json_is_taken_from_the_text(text, tables, truncated): + reply = D._parse_text(text, cut=False) + assert reply is not None and [e["table"] for e in reply.entries] == tables + assert reply.truncated is truncated + + +def test_cut_values_are_dropped_and_complete_ones_kept(): + text = '{"tables": [{"table": "a", "label": "甲", "keys": ["x", "y"], "columns": [{"name": "x", "label": "某"}, ' \ + '{"name": "y", "label": "半' + entry = D._parse_text(text, cut=True).entries[0] + assert entry == {"table": "a", "label": "甲", "keys": ["x", "y"], "columns": [{"name": "x", "label": "某"}]} + cut_keys = D._parse_text('{"tables": [{"table": "a", "label": "甲", "keys": ["x", "y"', cut=True).entries[0] + assert cut_keys == {"table": "a", "label": "甲"}, "没写完的业务主键整项不要,不能只留前半" + + +@pytest.mark.parametrize("text", ["好的,我来写", "{坏的 json}", ""]) +def test_not_json_is_not_a_reply(text): + assert D._parse_text(text, cut=False) is None diff --git a/backend/tests/test_catalog_draft_jobs.py b/backend/tests/test_catalog_draft_jobs.py index 87b66072..27c14957 100644 --- a/backend/tests/test_catalog_draft_jobs.py +++ b/backend/tests/test_catalog_draft_jobs.py @@ -215,6 +215,29 @@ async def test_a_table_whose_model_part_failed_is_failed_but_structure_is_writte assert by["channels"]["status"] == "ok" +async def test_a_job_that_cannot_start_is_failed_with_the_reason(client, monkeypatch): + """建任务时起草模型还能用,开始起草时已经用不了(接入刚被停用):任务整体 failed,error 写原因,没开始的表也记 failed。""" + sid = await _source() + calls: list[int] = [] + + async def flaky(session): + calls.append(1) + if len(calls) > 1: + raise catalog.CatalogModelUnavailable("起草模型使用的模型接入「虚构接入」已停用") + return FakeModel(REPLIES), "fake-model" + + monkeypatch.setattr(catalog, "resolve_draft_model", flaky) + job = (await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["visits", "gates"], + "use_model": True})).json()["job"] + await finished(job["id"]) + got = await get_job(client, sid, job["id"]) + assert got["status"] == "failed" and got["error"] == "起草模型使用的模型接入「虚构接入」已停用" + assert {t["status"] for t in got["tasks"]} == {"failed"} and got["finished_at"] + assert all(t["error"] == got["error"] for t in got["tasks"]) + retry = await client.post(f"{_base(sid)}/draft-jobs/{job['id']}/retry") + assert retry.status_code == 409 and retry.json()["code"] == "catalog_draft_model_unavailable" + + async def test_busy_source_is_409_and_the_active_job_is_reachable(client, monkeypatch): sid = await _source() model = Held(REPLIES, hold={"visits"})