mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-18 00:11:59 +00:00
162 lines
8.2 KiB
Python
162 lines
8.2 KiB
Python
"""双盲复算测试用的虚构零售库和盲算模型的剧本。表名、数字、地名都是编的。
|
||
|
||
订单表埋了两个陷阱:退款行(status = 'refunded'),以及下单日和支付日不在同一周的单。2026-09-01 至 2026-09-07 这一周:
|
||
|
||
- 按支付日、排除退款(口径说明要的):1、2、5 号单,330.0
|
||
- 按下单日、排除退款:2、4、5 号单,310.0
|
||
- 按支付日、不排除退款:1、2、3、5 号单,380.0
|
||
- 按下单日、不排除退款:2、3、4、5 号单,360.0
|
||
- 付款客户数(按支付日、排除退款):按行计数 3,去重计数 2
|
||
|
||
另有渠道(channel)和测试单标记(is_test,都是 0)两列:写数据目录的测试拿它们凑「指标自己的筛选范围」和标记列。
|
||
|
||
下单日、支付日两列声明为 DATE(SQLite 照样存成文字,按字符串比的先后和按日期比的一样):双盲复算只在两边的时间列都
|
||
声明为带日期的类型时才把选择记成业务日期。build(date_type="TEXT") 建一份声明成文本的、build(date_type="") 建一份
|
||
不写类型的,看不记的那几种。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import re
|
||
import sqlite3
|
||
from pathlib import Path
|
||
|
||
from langchain_core.messages import AIMessage, ToolMessage
|
||
|
||
from app.data import introspect
|
||
from app.db.base import SessionLocal
|
||
from app.db.models import DataSource
|
||
from tests.fixtures.names import letters
|
||
|
||
ROWS = [
|
||
(1, "华东", 101, 100.0, "paid", "2026-08-31", "2026-09-01", "online", 0),
|
||
(2, "华东", 102, 200.0, "paid", "2026-09-02", "2026-09-02", "offline", 0),
|
||
(3, "华南", 101, 50.0, "refunded", "2026-09-03", "2026-09-03", "online", 0),
|
||
(4, "华南", 103, 80.0, "paid", "2026-09-07", "2026-09-08", "online", 0),
|
||
(5, "华北", 102, 30.0, "paid", "2026-09-05", "2026-09-05", "offline", 0),
|
||
]
|
||
WEEK = "BETWEEN '2026-09-01' AND '2026-09-07'"
|
||
#: 口径说明要的写法:按支付日、排除退款
|
||
RIGHT = f"SELECT SUM(amount) AS gmv FROM orders WHERE status <> 'refunded' AND paid_date {WEEK}"
|
||
#: 只错在时间列:按下单日
|
||
BY_ORDER_DATE = f"SELECT SUM(amount) AS gmv FROM orders WHERE status <> 'refunded' AND order_date {WEEK}"
|
||
#: 只错在没排除退款
|
||
WITH_REFUNDS = f"SELECT SUM(amount) AS gmv FROM orders WHERE paid_date {WEEK}"
|
||
#: 两处都错
|
||
BOTH_WRONG = f"SELECT SUM(amount) AS gmv FROM orders WHERE order_date {WEEK}"
|
||
#: 付款客户数:按行计数 / 去重计数
|
||
BUYERS = f"SELECT COUNT(customer_id) AS buyers FROM orders WHERE status <> 'refunded' AND paid_date {WEEK}"
|
||
BUYERS_DISTINCT = (f"SELECT COUNT(DISTINCT customer_id) AS buyers FROM orders "
|
||
f"WHERE status <> 'refunded' AND paid_date {WEEK}")
|
||
#: 按地区分组:华东一行是 300.0(1、2 号单)
|
||
BY_REGION = (f"SELECT region, SUM(amount) AS gmv FROM orders WHERE status <> 'refunded' AND paid_date {WEEK} "
|
||
"GROUP BY region ORDER BY region")
|
||
DEFINITION = "已支付订单的金额合计:按支付日期(paid_date)归属到当周,不含退款"
|
||
|
||
|
||
def build(path: Path, *, date_type: str = "DATE") -> Path:
|
||
db = sqlite3.connect(path)
|
||
db.executescript(
|
||
"CREATE TABLE orders (id INTEGER PRIMARY KEY, region TEXT, customer_id INTEGER, amount REAL, status TEXT, "
|
||
f"order_date {date_type}, paid_date {date_type}, channel TEXT, is_test INTEGER);")
|
||
db.executemany("INSERT INTO orders VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", ROWS)
|
||
db.commit()
|
||
db.close()
|
||
return path
|
||
|
||
|
||
async def add_source(path: Path, *, options: dict | None = None) -> tuple[str, str]:
|
||
"""登记成只读的手工源,探查好表结构。返回 (数据源名, id)。"""
|
||
name = f"dc_shop_{letters(8)}"
|
||
async with SessionLocal() as session:
|
||
row = DataSource(name=name, kind="sqlite", database=str(path), readonly=True, enabled=True,
|
||
options=dict(options or {}))
|
||
session.add(row)
|
||
await session.flush()
|
||
row.schema_cache = await introspect.introspect(row)
|
||
await session.commit()
|
||
return name, row.id
|
||
|
||
|
||
def _text(messages) -> str:
|
||
return "\n".join(m.content if isinstance(m.content, str) else json.dumps(m.content, ensure_ascii=False)
|
||
for m in messages)
|
||
|
||
|
||
class Blind:
|
||
"""盲算模型的剧本,也管图里别的模型调用。
|
||
|
||
plans:{任务里出现的指标名: (SQL, 要交的列) 或 [SQL, …]、列}。键写成「指标名|输入2」时只认计算式里那一项的盲算
|
||
(一个指标有几个输入来自查询时用)。盲算循环里依次查这几条 SQL,收尾时交最后一条查询的第 0 行那一列(row 可改)。
|
||
没对上指标名的盲算交 null。不是盲算的调用(报告撰写)返回 writer 的正文并记下收到的消息。
|
||
calls 记下每次盲算调用(step / final),tasks 记下盲算收到的任务原文(几个盲算同时进行时也是每个任务一条)。
|
||
on_query:盲算第一次查库之前调用一次(比如往库里插一行,造出「两次取数之间数据变了」)。
|
||
rationale:盲算交的理由,{ref} 换成交的那一格的引用(缺省「按口径说明统计,结果是 {ref}。」)。
|
||
"""
|
||
|
||
def __init__(self, monkeypatch, source: str, plans: dict, *, writer: str = "本周销售额见口径卡。", row: int = 0,
|
||
on_query=None, rationale: str = "按口径说明统计,结果是 {ref}。"):
|
||
from app.providers import mock_model
|
||
|
||
self.source = source
|
||
self.plans = plans
|
||
self.writer = writer
|
||
self.row = row
|
||
self.on_query = on_query
|
||
self.rationale = rationale
|
||
self.calls: list[str] = []
|
||
self.tasks: list[str] = []
|
||
self.writer_prompts: list[str] = []
|
||
blind = self
|
||
|
||
def decide(model, messages):
|
||
text = _text(messages)
|
||
plan = blind.plan_of(text)
|
||
if model.response_format:
|
||
blind.calls.append("final")
|
||
done = len(re.findall(r"【证据 Q\d+】", text))
|
||
if plan is None or not done:
|
||
return AIMessage(content=json.dumps({"value": None, "rationale": "没有查到。"}, ensure_ascii=False))
|
||
_, column = plan
|
||
alias = f"Q{len(blind.sqls(plan))}"
|
||
body = {"value": {"alias": alias, "row": blind.row, "column": column},
|
||
"rationale": blind.rationale.format(ref=f"[[v:{alias}.r{blind.row}.{column}]]")}
|
||
return AIMessage(content=json.dumps(body, ensure_ascii=False))
|
||
if model.tools:
|
||
blind.calls.append("step")
|
||
# 盲算的任务是第一条用户消息;同一次盲算的几轮收到的是同一条,只记一次
|
||
task_text = next((str(m.content) for m in messages if m.type == "human"), "")
|
||
if task_text not in blind.tasks:
|
||
blind.tasks.append(task_text)
|
||
done = sum(isinstance(m, ToolMessage) for m in messages)
|
||
sqls = blind.sqls(plan) if plan is not None else []
|
||
if done < len(sqls):
|
||
if blind.on_query is not None:
|
||
hook, blind.on_query = blind.on_query, None
|
||
hook()
|
||
call = {"name": f"db_query__{blind.source}", "args": {"sql": sqls[done]},
|
||
"id": f"blind-{letters(6)}"}
|
||
return AIMessage(content="查一下。", tool_calls=[call])
|
||
return AIMessage(content="查完了。")
|
||
blind.writer_prompts.append(text)
|
||
return AIMessage(content=blind.writer)
|
||
|
||
monkeypatch.setattr(mock_model.MockChatModel, "_decide", decide)
|
||
|
||
def plan_of(self, text: str):
|
||
for key, plan in self.plans.items():
|
||
name, _, label = key.partition("|")
|
||
if f"指标:{name}" in text and (not label or f"计算式里的 {label}" in text):
|
||
return plan
|
||
return None
|
||
|
||
@staticmethod
|
||
def sqls(plan) -> list[str]:
|
||
sql = plan[0]
|
||
return list(sql) if isinstance(sql, (list, tuple)) else [sql]
|
||
|
||
@property
|
||
def runs(self) -> int:
|
||
"""盲算 agent 一共执行了几次(每次收尾交一次结果)。"""
|
||
return self.calls.count("final")
|