import { useState, useRef, useMemo } from "react"; import { useSQLQuery, useDiveState } from "@motherduck/react-sql-query"; import { LineChart, Line, BarChart, Bar, XAxis, YAxis, CartesianGrid, Tooltip, ResponsiveContainer, LabelList, Legend, ScatterChart, Scatter, ZAxis, } from "recharts"; export const REQUIRED_DATABASES = [ { type: 'share', path: 'md:_share/dabstep_logs_alex/137bc813-94fe-4536-8532-f8602cf55449', alias: 'dabstep_logs' } ]; const N = (v: unknown): number => (v != null ? Number(v) : 0); /* Data access: dabstep_logs is read via a personally-owned share of dabstep_logs_alex (ACCESS UNRESTRICTED, UPDATE AUTOMATIC — see REQUIRED_DATABASES), not the org's read-only dabstep_logs share. This is the same MotherDuck database every asm evaluate run (local Qwen3.8-27B and OpenRouter) already writes into via controllog. */ /* ── MotherDuck brand tokens — cream canvas, ink strokes, hard offset shadows. Keys preserved from v2 so every reference re-skins at once; values remapped from the real MotherDuck brand/style guide (colors_and_type.css): md-paper family for surfaces, md-ink for text/strokes, the brand spectrum (yellow/orange/teal/coral/sky/blue) for accents, Aeonik Mono (display) + Inter (body) + JetBrains Mono (data) for type. Card borders/shadows are dialed down from the guide's full hard-shadow marketing treatment (2px ink border + 8px offset shadow on every element) to a lighter touch suited to a dense data dashboard — see the dive report for that judgment call. ── */ const T = { paper: "#F4EFEA", paper2: "#ECE5DD", paper3: "#E1D6CB", ink: "#383838", inkMuted: "#818181", faint: "rgba(56,56,56,0.45)", line: "#383838", white: "#FFFFFF", yellow: "#FFDE00", orange: "#FF9538", teal: "#16AA98", coral: "#FF7169", coralSoft: "#FFCBC8", sky: "#6FC2FF", blue: "#2BA5FF", purple: "#9F7AEA", mono: '"JetBrains Mono", "Source Code Pro", "SF Mono", Menlo, monospace', sans: '"Inter", -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif', display: '"Aeonik Mono", "Space Mono", Menlo, ui-monospace, monospace', // brand display face — H1 + eyebrows shadow: "5px 5px 0 0 #383838", // hard offset — floating surfaces (picker, drawer) shadowSm: "2px 2px 0 0 #383838", // hard offset — small lift (tooltips, mini-chart) }; // agent tools render as one blue family separated by lightness (style-guide composition bar) const TOOL_ORDER = ["fetch_context", "query", "list_tables", "list_columns", "submit_answer"]; const TOOL_COLOR: Record = { query: "#0B6FCC", fetch_context: "#2BA5FF", list_tables: "#6FC2FF", list_columns: "#97D4FF", submit_answer: "#C2E5FF", }; // Cost vs. Accuracy legend/series colors, one per model, assigned by sorted model name. const AC_PALETTE = ["#2BA5FF", "#9F7AEA", "#16AA98", "#FF7169", "#FF9538", "#6FC2FF", "#068475", "#EC4899"]; const sq = (s: string) => s.replace(/'/g, "''"); type Tab = "overview" | "matrix" | "tools" | "outliers"; type Sel = { qid: string; a: string; b: string | null }; export default function ControllogDashboard() { const [tab, setTab] = useDiveState("tab", "overview"); const [sel, setSel] = useDiveState("cell", null); const [picked, setPicked] = useDiveState("runs", []); // [] = all runs (no run-label filter) const [fullOnly, setFullOnly] = useDiveState("fullOnly", true); // Cost vs. Accuracy: full-419 runs only const [modelSel, setModelSel] = useDiveState("models", []); // [] = all models const [splitSel, setSplitSel] = useDiveState("splits", ["test"]); // [] = all splits (train/test) // outliers: clicked point (trace) + drag-box selection const [oSel, setOSel] = useDiveState<{ rl: string; qid: string } | null>("opt", null); const [box, setBox] = useState<{ x0: number; y0: number; x1: number; y1: number } | null>(null); const chartRef = useRef(null); const rectRef = useRef(null); // drag rectangle, updated imperatively (no re-render) const dragStartRef = useRef<{ x: number; y: number } | null>(null); const dragLastRef = useRef<{ x: number; y: number } | null>(null); // run-comparison picker (searchable dropdown) const [pickerOpen, setPickerOpen] = useState(false); const [pickQuery, setPickQuery] = useState(""); const MAX_RUNS = 10; // short, stable column label: test419 family → v1..vN; otherwise truncate const shortRun = (label: string): string => { if (label === "test419") return "v1"; const m = label.match(/^test419_v(\d+)$/); if (m) return `v${m[1]}`; return label.length > 11 ? label.slice(0, 10) + "…" : label; }; // short per-model label for bubble labels/legends: strip the "org/" prefix for hosted // OpenRouter models (e.g. "anthropic/claude-sonnet-5" → "sonnet-5"). Local Qwen runs // have no org prefix — they are bare LM Studio / GGUF identifiers — so surface the // QUANTIZATION, which is the thing that actually distinguishes them. // // There is now more than one local Qwen run at different bit-widths, so the label is // derived from the model string rather than hardcoded: // qwen3.8-27b-3bit-local → qwen-3bit (unsloth GGUF UD-IQ3_XXS, 10.9GB) // qwen3.8-27b-mlx → qwen-4bit (lmstudio-community/Qwen3.8-27B-MLX-4bit) // qwen3.8-27b → qwen (quantization not recorded for that run) const shortModelName = (model: string): string => { const s = String(model ?? ""); const slash = s.indexOf("/"); if (slash === -1) { if (s.toLowerCase().startsWith("qwen")) { const q = s.match(/(\d+)\s*-?\s*bit/i); if (q) return `qwen-${q[1]}bit`; if (/mlx/i.test(s)) return "qwen-4bit"; return "qwen"; } return s; } const rest = s.slice(slash + 1); // Anthropic's product line is branded "Claude" on top of the org name, so // "anthropic/claude-sonnet-5" carries the lab name twice; drop the leading // "claude-" too so it reads as "sonnet-5". Other orgs (openai/gpt-…, // google/gemini-…, moonshotai/kimi-…) don't repeat like this, so only this // one prefix needs the extra strip. return rest.replace(/^claude-/i, ""); }; // The controllog schema never records the reasoning level a run used — the only // place it survives is as a token baked into the run_label at eval time (e.g. // "test_kimik3_off_20260828T020515Z", "test_sonnet5_low_..."). Best-effort parse // so two runs of the same model at different reasoning levels don't render as // two identically-labeled bubbles with no visible explanation for the accuracy gap. const REASONING_TOKENS = ["xhigh", "medium", "default", "high", "low", "off", "max"]; const parseReasoning = (runLabel: string): string | null => { const segs = String(runLabel ?? "").toLowerCase().split(/[^a-z0-9]+/).filter(Boolean); for (const seg of segs) if (REASONING_TOKENS.includes(seg)) return seg; return null; }; // shared run filter — drives Overview, Tools, Matrix and Outliers. // empty run-label selection = all runs; model/split filters AND on top when non-empty. const runFilter = [ picked.length ? `payload_json.run_label IN (${picked.map((p) => `'${sq(p)}'`).join(",")})` : "TRUE", modelSel.length ? `payload_json.model IN (${modelSel.map((m) => `'${sq(m)}'`).join(",")})` : "TRUE", splitSel.length ? `payload_json.split IN (${splitSel.map((s) => `'${sq(s)}'`).join(",")})` : "TRUE", ].join(" AND "); // ── queries ────────────────────────────────────────────────────────── const kpi = useSQLQuery(` SELECT (SELECT count(DISTINCT payload_json.run_label) FROM "dabstep_logs"."main"."events") AS runs, (SELECT count(*) FROM "dabstep_logs"."main"."events") AS events, (SELECT round(sum(CASE WHEN delta_numeric>0 THEN delta_numeric ELSE 0 END),2) FROM "dabstep_logs"."main"."postings" WHERE account_type='truth.money') AS cost, (SELECT sum(CASE WHEN delta_numeric>0 THEN delta_numeric ELSE 0 END) FROM "dabstep_logs"."main"."postings" WHERE account_type='truth.utility') AS utility, (SELECT count(*) FROM (SELECT 1 FROM "dabstep_logs"."main"."postings" GROUP BY account_type, unit HAVING abs(sum(delta_numeric))>1e-4)) AS violations `); // the v1→v7 tuning arc: accuracy + the three things it spends (tools, tokens, $). // tokens come from the ledger (resource.tokens), joined to events by run_label. const trend = useSQLQuery(` WITH ev AS ( SELECT payload_json.run_label AS version, min(event_time) AS ord, count(*) AS tasks, round(avg(CASE WHEN payload_json.is_correct THEN 1.0 ELSE 0 END)*100,1) AS pct_correct, round(sum(payload_json.cost_usd)/count(*),4) AS cost_per_task, round(sum(payload_json.cost_usd),2) AS total_cost, round(avg(payload_json.tool_calls),1) AS tool_calls, round(avg(payload_json.duration_ms)/1000.0,1) AS avg_dur_s, any_value(payload_json.model) AS model, any_value(payload_json.split) AS split FROM "dabstep_logs"."main"."events" WHERE kind='evaluation_result' AND ${runFilter} GROUP BY 1 ), tok AS ( SELECT dims_json.run_label AS version, sum(delta_numeric) FILTER (delta_numeric>0) AS tokens FROM "dabstep_logs"."main"."postings" WHERE account_type='resource.tokens' GROUP BY 1 ) SELECT ev.*, tok.tokens, round(tok.tokens/ev.tasks/1000.0,1) AS ktok_per_task FROM ev LEFT JOIN tok USING (version) ORDER BY ev.ord `); // run list for the comparison picker — only runs with the full eval metadata the dive needs // (scored evaluation_result events). ~12 of 53 raw run_labels lack it and would break the views. const runs = useSQLQuery(` SELECT payload_json.run_label AS run_label, strftime(min(event_time),'%m-%d %H:%M') AS started, any_value(payload_json.model) AS model, any_value(payload_json.split) AS split FROM "dabstep_logs"."main"."events" WHERE kind='evaluation_result' AND payload_json.run_label IS NOT NULL AND payload_json.is_correct IS NOT NULL AND payload_json.cost_usd IS NOT NULL AND payload_json.tool_calls IS NOT NULL AND payload_json.question_id IS NOT NULL GROUP BY 1 ORDER BY min(event_time) DESC `); const matrix = useSQLQuery(` WITH dedup AS ( SELECT payload_json.run_label AS ver, payload_json.question_id AS qid, payload_json.is_correct AS ok, payload_json.level AS lvl, min(event_time) OVER (PARTITION BY payload_json.run_label) AS ord, row_number() OVER (PARTITION BY payload_json.run_label, payload_json.question_id ORDER BY event_time DESC) AS rn FROM "dabstep_logs"."main"."events" WHERE kind='evaluation_result' AND ${runFilter} ), d AS (SELECT * FROM dedup WHERE rn=1), -- "disagreement": only questions whose correctness is not unanimous across the chosen runs volatile AS (SELECT qid FROM d GROUP BY qid HAVING count(DISTINCT ok) > 1) SELECT d.ver, d.qid, d.ok, d.lvl, strftime(d.ord,'%Y%m%d%H%M%S') AS ordk FROM d JOIN volatile v USING (qid) ORDER BY d.ord `); // tool-call distribution across the arc: which tools the agent called, how often. // unnest the bundled conversation trace and count function_call steps by name. const toolsQ = useSQLQuery(` WITH flat AS ( SELECT payload_json.run_label AS rl, min(event_time) OVER (PARTITION BY payload_json.run_label) AS ord, unnest(payload_json.raw_response.messages) AS m FROM "dabstep_logs"."main"."events" WHERE kind='evaluation_result' AND ${runFilter} AND payload_json.raw_response.messages IS NOT NULL ) SELECT rl, any_value(ord) AS ord, m.name AS tool, count(*) AS calls FROM flat WHERE m.type='function_call' GROUP BY rl, m.name ORDER BY ord `); // outliers: every task's cost/latency/metadata; the comparison table is computed // client-side from whatever the brush box selects (or all points when unboxed). const dist = useSQLQuery(` SELECT payload_json.is_correct AS ok, round(payload_json.duration_ms/1000.0,1) AS dur_s, round(payload_json.cost_usd,4) AS cost, payload_json.tool_calls AS tools, payload_json.level AS lvl, coalesce(payload_json.sql_errors,0) AS sql_errors, payload_json.question_id AS qid, payload_json.run_label AS rl FROM "dabstep_logs"."main"."events" WHERE kind='evaluation_result' AND ${runFilter} AND payload_json.duration_ms>0 `); // outlier point → single (run, question) trace + meta const oPred = oSel ? `payload_json.run_label='${sq(oSel.rl)}' AND payload_json.question_id='${sq(oSel.qid)}'` : "FALSE"; const oMeta = useSQLQuery(` SELECT payload_json.is_correct AS ok, round(payload_json.cost_usd,4) AS cost, payload_json.duration_ms AS dur, payload_json.tool_calls AS tools, payload_json.level AS lvl, payload_json.question_text AS q, payload_json.predicted_result AS pred, payload_json.gold_result AS gold FROM "dabstep_logs"."main"."events" WHERE kind='evaluation_result' AND ${oPred} ORDER BY event_time DESC LIMIT 1 `, { enabled: !!oSel }); const oTrace = useSQLQuery(` WITH one AS ( SELECT payload_json.raw_response.messages AS msgs FROM "dabstep_logs"."main"."events" WHERE kind='evaluation_result' AND ${oPred} AND payload_json.raw_response.messages IS NOT NULL ORDER BY event_time DESC LIMIT 1 ), flat AS (SELECT unnest(msgs) AS m, generate_subscripts(msgs,1) AS step_no FROM one) SELECT step_no, coalesce(m.type,m.role) AS step, m.name AS fn_name, m.arguments AS args, left(coalesce(m.output, CAST(m.content AS VARCHAR), ''), 700) AS body FROM flat ORDER BY step_no `, { enabled: !!oSel }); // runs being diffed for the selected question (A, and optional B) const selRuns = sel ? [sel.a, sel.b].filter(Boolean) as string[] : []; const runInList = selRuns.map((r) => `'${sq(r)}'`).join(",") || "''"; const selPred = sel ? `payload_json.question_id='${sq(sel.qid)}' AND payload_json.run_label IN (${runInList})` : "FALSE"; const meta = useSQLQuery(` SELECT payload_json.run_label AS rl, payload_json.is_correct AS ok, round(payload_json.cost_usd,4) AS cost, payload_json.duration_ms AS dur, payload_json.tool_calls AS tools, payload_json.level AS lvl, payload_json.question_text AS q, payload_json.predicted_result AS pred, payload_json.gold_result AS gold, row_number() OVER (PARTITION BY payload_json.run_label ORDER BY event_time DESC) AS rn FROM "dabstep_logs"."main"."events" WHERE kind='evaluation_result' AND ${selPred} QUALIFY rn=1 `, { enabled: !!sel }); const trace = useSQLQuery(` WITH src AS ( SELECT payload_json.run_label AS rl, payload_json.raw_response.messages AS msgs, row_number() OVER (PARTITION BY payload_json.run_label ORDER BY event_time DESC) AS rn FROM "dabstep_logs"."main"."events" WHERE kind='evaluation_result' AND ${selPred} AND payload_json.raw_response.messages IS NOT NULL ), flat AS (SELECT rl, unnest(msgs) AS m, generate_subscripts(msgs,1) AS step_no FROM src WHERE rn=1) SELECT rl, step_no, coalesce(m.type,m.role) AS step, m.name AS fn_name, m.arguments AS args, left(coalesce(m.output, CAST(m.content AS VARCHAR), ''), 1800) AS body FROM flat ORDER BY rl, step_no `, { enabled: !!sel }); // ── derive ─────────────────────────────────────────────────────────── const k = (Array.isArray(kpi.data) ? kpi.data : [])[0] ?? {}; const tRows = Array.isArray(trend.data) ? trend.data : []; const trendData = tRows.map((r) => { const tools = N(r.tool_calls), ktok = N(r.ktok_per_task); return { v: shortRun(String(r.version)), rl: String(r.version), pct: N(r.pct_correct), cost: N(r.cost_per_task) * 1000, costTask: N(r.cost_per_task), tools, ktok, tokens: N(r.tokens), tasks: N(r.tasks), durS: N(r.avg_dur_s), model: String(r.model ?? "—"), split: String(r.split ?? ""), // K tokens per tool call = (tokens/task) ÷ (calls/task). aggregate ratio — dabstep has // no per-call token attribution, so this is total tokens ÷ total tool calls, not per-call. tokPerCall: tools > 0 ? Math.round((ktok / tools) * 100) / 100 : 0, }; }); // "full runs only" — a run is comparable only if it scored the largest task count // among the currently-picked/visible runs (a partial/dev run's cheaper $/task and // different accuracy aren't apples-to-apples against a full 419-question run). const maxTasksSeen = trendData.reduce((m, d) => Math.max(m, d.tasks), 0); const isFullRun = (d: { tasks: number }) => maxTasksSeen === 0 || d.tasks >= maxTasksSeen; const frontierAll = trendData.map((d) => ({ v: d.v, rl: d.rl, x: d.cost, y: d.pct, z: d.durS, costTask: d.costTask, tokens: d.tokens, tasks: d.tasks, durS: d.durS, model: d.model, split: d.split, full: isFullRun(d), reasoning: parseReasoning(d.rl), })); // per-version (cost/task, accuracy) points for the cost-vs-accuracy frontier; bubble = avg runtime/question const frontier = fullOnly ? frontierAll.filter((d) => d.full) : frontierAll; // group the frontier points by model for the Cost vs. Accuracy legend/series — colors assigned // by sorted model name so they stay stable as the filtered set changes. const acByModel = new Map(); for (const d of frontier) { const key = d.model || "—"; if (!acByModel.has(key)) acByModel.set(key, []); acByModel.get(key)!.push(d); } const acColor = new Map(); [...acByModel.keys()].sort().forEach((m, i) => acColor.set(m, AC_PALETTE[i % AC_PALETTE.length])); // dynamic axis domains so arbitrary run selections (not just the 97–100% arc) don't clip const pad = (lo: number, hi: number, f: number) => { const g = (hi - lo) || Math.abs(hi) || 1; return [lo - g * f, hi + g * f] as [number, number]; }; const pcts = trendData.map((d) => d.pct); const pctDomain: [number, number] = pcts.length ? [Math.max(0, Math.min(97, Math.floor(Math.min(...pcts)))), 100] : [97, 100]; const xs = frontier.map((d) => d.x), ys = frontier.map((d) => d.y); const xDomain: [number, number] = [0, xs.length ? Math.max(...xs) * 1.1 : 0.045]; const yDomain: [number, number] = [90, 100]; // same idea for the Runtime vs. Accuracy chart's x-axis (avg wall-clock seconds/question) const durXs = frontier.map((d) => d.durS); const durXDomain: [number, number] = [0, durXs.length ? Math.max(...durXs) * 1.1 : 60]; // tasks-per-version, to normalize raw tool-call counts into calls-per-task (v4 ran 779, not 419) const tasksByVer = new Map(); for (const r of tRows) tasksByVer.set(String(r.version), N(r.tasks)); // model/split filter chip options — sourced from the unfiltered `runs` query so every // available value is always pickable, not just the ones surviving the current filter. const runsRows = Array.isArray(runs.data) ? runs.data : []; const modelOptions = Array.from(new Set(runsRows.map((r) => String(r.model ?? "")))).filter(Boolean).sort(); const splitOptions = Array.from(new Set(runsRows.map((r) => String(r.split ?? "")))).filter(Boolean).sort(); const splitLabel = (s: string) => (s === "templates" ? "train" : s === "test" ? "test" : s); const { verOrder, qRows } = buildMatrix(Array.isArray(matrix.data) ? matrix.data : []); // tool-call distribution → calls-per-task per version, pivoted for a stacked bar. // version order follows the trend arc (v1..vN); tools follow TOOL_ORDER. const toolRows = Array.isArray(toolsQ.data) ? toolsQ.data : []; const toolVers = trendData.map((d) => d.rl); // arc order const toolNames = TOOL_ORDER.filter((t) => toolRows.some((r) => String(r.tool) === t)) .concat(Array.from(new Set(toolRows.map((r) => String(r.tool)))).filter((t) => !TOOL_ORDER.includes(t))); const toolData = toolVers.map((rl) => { const tasks = tasksByVer.get(rl) || 1; const o: Record = { v: shortRun(rl) }; for (const t of toolNames) o[t] = 0; for (const r of toolRows) if (String(r.rl) === rl) o[String(r.tool)] = Math.round((N(r.calls) / tasks) * 10) / 10; return o; }); // selected-question A/B detail: split meta + trace rows by run const metaByRun = new Map>(); for (const r of (Array.isArray(meta.data) ? meta.data : [])) metaByRun.set(String(r.rl), r); const stepsByRun = new Map[]>(); for (const s of (Array.isArray(trace.data) ? trace.data : [])) { const rl = String(s.rl); if (!stepsByRun.has(rl)) stepsByRun.set(rl, []); stepsByRun.get(rl)!.push(s); } const gold = sel ? String(metaByRun.get(sel.a)?.gold ?? metaByRun.get(sel.b ?? "")?.gold ?? "—") : ""; const qText = sel ? String(metaByRun.get(sel.a)?.q ?? metaByRun.get(sel.b ?? "")?.q ?? "") : ""; // outliers — memoized so clicking a point / opening the drawer doesn't rebuild 3k+ points type Pt = { ok: boolean; dur_s: number; cost: number; tools: number; hard: boolean; sqlErr: number; qid: string; rl: string }; const distPts: Pt[] = useMemo(() => (Array.isArray(dist.data) ? dist.data : []).map((r) => ({ ok: Boolean(r.ok), dur_s: N(r.dur_s), cost: N(r.cost), tools: N(r.tools), hard: String(r.lvl) === "hard", sqlErr: N(r.sql_errors), qid: String(r.qid), rl: String(r.rl), })), [dist.data]); const distOk = useMemo(() => distPts.filter((p) => p.ok), [distPts]); const distBad = useMemo(() => distPts.filter((p) => !p.ok), [distPts]); // the scatter chart itself — memoized so only data changes (not clicks/drawer/box) re-render its points const scatterEl = useMemo(() => { const xs = distPts.map((p) => p.dur_s).filter((v) => v > 0); const ys = distPts.map((p) => p.cost).filter((v) => v > 0); const xlo = (Math.min(...xs) || 1) * 0.85, xhi = (Math.max(...xs) || 10) * 1.15; const ylo = (Math.min(...ys) || 0.001) * 0.85, yhi = (Math.max(...ys) || 1) * 1.15; const M = { top: 10, right: 16, bottom: 22, left: 8 }, YW = 52, XH = 22, H = 320; const px2x = (px: number, W: number) => { const L = M.left + YW, R = W - M.right; const f = Math.min(1, Math.max(0, (px - L) / (R - L))); return Math.exp(Math.log(xlo) + f * (Math.log(xhi) - Math.log(xlo))); }; const px2y = (py: number) => { const Tp = M.top, B = H - M.bottom - XH; const f = Math.min(1, Math.max(0, (py - Tp) / (B - Tp))); return Math.exp(Math.log(yhi) - f * (Math.log(yhi) - Math.log(ylo))); }; const onPt = (d: any) => { if (d?.payload) setOSel({ rl: d.payload.rl, qid: String(d.payload.qid) }); }; return (

{ const r = chartRef.current?.getBoundingClientRect(); if (!r || !e) return; const x = e.clientX - r.left, y = e.clientY - r.top; dragStartRef.current = { x, y }; dragLastRef.current = { x, y }; }} onMouseMove={(_s: any, e: any) => { const st = dragStartRef.current, r = chartRef.current?.getBoundingClientRect(), rc = rectRef.current; if (!st || !r || !e || !rc) return; const x = e.clientX - r.left, y = e.clientY - r.top; dragLastRef.current = { x, y }; rc.style.display = "block"; rc.style.left = `${Math.min(st.x, x)}px`; rc.style.top = `${Math.min(st.y, y)}px`; rc.style.width = `${Math.abs(x - st.x)}px`; rc.style.height = `${Math.abs(y - st.y)}px`; }} onMouseUp={() => { const st = dragStartRef.current, last = dragLastRef.current; const W = chartRef.current?.clientWidth ?? 600; if (st && last && Math.abs(last.x - st.x) + Math.abs(last.y - st.y) > 4) { const x0 = px2x(Math.min(st.x, last.x), W), x1 = px2x(Math.max(st.x, last.x), W); const y1 = px2y(Math.min(st.y, last.y)), y0 = px2y(Math.max(st.y, last.y)); setBox({ x0, x1, y0, y1 }); setOSel(null); } dragStartRef.current = null; dragLastRef.current = null; }}> `${v}s`} /> `$${v}`} /> [n === "cost" ? `$${v}` : `${v}s`, n]} />

); }, [distPts, distOk, distBad]); // points inside the brush box (or all points if no box) const shownPts = box ? distPts.filter((p) => p.dur_s >= box.x0 && p.dur_s <= box.x1 && p.cost >= box.y0 && p.cost <= box.y1) : distPts; const agg = (pts: Pt[]) => pts.length === 0 ? null : { n: pts.length, tools: pts.reduce((s, p) => s + p.tools, 0) / pts.length, dur_s: pts.reduce((s, p) => s + p.dur_s, 0) / pts.length, cost: pts.reduce((s, p) => s + p.cost, 0) / pts.length, pct_hard: pts.reduce((s, p) => s + (p.hard ? 1 : 0), 0) / pts.length * 100, sql_errors: pts.reduce((s, p) => s + p.sqlErr, 0) / pts.length, }; const okRow = agg(shownPts.filter((p) => p.ok)); const badRow = agg(shownPts.filter((p) => !p.ok)); const oDetail = (Array.isArray(oMeta.data) ? oMeta.data : [])[0]; const oSteps = Array.isArray(oTrace.data) ? oTrace.data : []; // ── ui primitives ────────────────────────────────────────────────────── // brand card: white surface on the cream canvas, ink stroke + a small hard-offset shadow // (dialed down from the style guide's full 2px/8px marketing treatment to suit a dense grid). const card = (extra?: React.CSSProperties): React.CSSProperties => ({ background: T.white, border: `1px solid ${T.paper3}`, borderRadius: 12, boxShadow: T.shadowSm, padding: 16, ...extra, }); const Skel = ({ h = 40 }: { h?: number }) => (

); // Bubble + two-line label (short model name over accuracy%), shared by the Cost vs. // Accuracy and Runtime vs. Accuracy scatter charts. Built as a Scatter `shape` render // prop (not ``, which never fired here — Scatter doesn't forward // its `data` down to a LabelList child the way Bar/Line do, so the content callback // was silently never invoked and no labels rendered at all). `shape` is Scatter's own // per-point renderer, so it's guaranteed to receive cx/cy/r/payload for every point; // drawing the circle ourselves here replaces Scatter's default dot with an identical // one plus the label, in one guaranteed-to-render place. const bubbleShape = (fill: string) => (p: any) => { const d = p?.payload ?? {}; const cx = N(p.cx), cy = N(p.cy), r = N(p.r) || 8; // labels above the dot get clipped by the SVG's own bounds once a point sits near the // y-axis max (e.g. accuracy near the 100% ceiling) — flip below the dot in that zone instead const above = cy - r - 26 >= 4; const ly = above ? cy - r - 15 : cy + r + 15; return ( {shortModelName(String(d.model))}{d.reasoning ? ` · ${d.reasoning}` : ""} {N(d.y).toFixed(1)}% ); }; const eyebrow: React.CSSProperties = { fontFamily: T.display, fontSize: 11, fontWeight: 700, textTransform: "uppercase", letterSpacing: "0.1em", color: T.faint, margin: "0 0 8px", }; const Kpi = ({ label, value, accent }: { label: string; value: React.ReactNode; accent?: string }) => (

{value}

{label}

); const TabBtn = ({ id, children }: { id: Tab; children: string }) => { const active = tab === id; return ( ); }; // correctness cells: soft semantic tints (style-guide correctness grid), grey for missing const cellColor = (ok: boolean | null) => ok == null ? T.paper2 : ok ? "rgba(22,170,152,0.20)" : "rgba(255,113,105,0.28)"; // ── small-multiple trend panel ───────────────────────────────────────── const Mini = ({ title, dk, fmt, domain, color, suffix }: { title: string; dk: string; fmt: (v: number) => string; domain?: [number, number]; color: string; suffix?: string; }) => (

{title}

{trend.isLoading ? : ( [`${fmt(N(val))}${suffix ?? ""}`, title]} /> {/* only the first & last points carry a value label, in the same muted weight as the axis */} { const last = trendData.length - 1; if (p.index == null || (p.index !== 0 && p.index !== last)) return null; const isFirst = p.index === 0; return {fmt(N(p.value))}; }} /> )}

); // shared run-filter pane — searchable multi-select with chips (compares up to MAX_RUNS; // empty = all runs), plus model/split filter chips that scope every tab via runFilter. // Inlined via a function call (NOT a ) so it isn't remounted on every parent // render — that's what keeps dropdown scroll + input focus. const renderRunPicker = () => (

Comparing {picked.length === 0 ? ( all runs (default) ) : picked.map((rl) => ( {shortRun(rl)} ))} {picked.length > 0 && ( )}

{/* model / split filter chips — AND together with the run-label picker above via runFilter, scoping every tab (Overview/Matrix/Tools/Outliers), not just Cost vs. Accuracy. Options are sourced from the full (unfiltered) run universe so every value stays pickable. */}

Filter {modelOptions.length > 0 && ( model: {modelOptions.map((m) => { const on = modelSel.includes(m); return ( ); })} )} {splitOptions.length > 0 && ( split: {splitOptions.map((s) => { const on = splitSel.includes(s); return ( ); })} )} setFullOnly(e.target.checked)} /> Full Runs Only ({maxTasksSeen || "—"} tasks) {(modelSel.length > 0 || splitSel.length > 0 || fullOnly) && ( )}

{pickerOpen && ( <> {/* click-away backdrop */}

{ setPickerOpen(false); setPickQuery(""); }} style={{ position: "fixed", inset: 0, zIndex: 10 }} />

setPickQuery(e.target.value)} placeholder="search jobs…" style={{ fontFamily: T.mono, fontSize: 12, width: "100%", boxSizing: "border-box", padding: "9px 12px", border: "none", borderBottom: `1px solid ${T.paper3}`, background: T.paper, color: T.ink, outline: "none" }} /> {/* full list stays mounted as you toggle (selected items aren't removed) so scroll position holds */}

{runsRows .filter((r) => String(r.run_label).toLowerCase().includes(pickQuery.toLowerCase())) .map((r) => { const rl = String(r.run_label); const isSel = picked.includes(rl); const atMax = !isSel && picked.length >= MAX_RUNS; return ( ); })}

{picked.length}/{MAX_RUNS} selected

)}

); const td: React.CSSProperties = { padding: "7px 10px", fontFamily: T.mono, fontSize: 12, color: T.ink, borderBottom: `1px solid ${T.paper3}` }; const tdNum: React.CSSProperties = { ...td, textAlign: "right", fontVariantNumeric: "tabular-nums" }; // formatted renderers for trace content const codeBox: React.CSSProperties = { fontFamily: T.mono, fontSize: 10.5, lineHeight: 1.5, whiteSpace: "pre-wrap", wordBreak: "break-word", margin: 0, padding: "7px 9px", background: T.paper2, border: `1px solid ${T.paper3}`, borderRadius: 6, color: T.ink }; const Sql = ({ text }: { text: string }) => (

{text.trim()}

); const Json = ({ value }: { value: unknown }) => (

{JSON.stringify(value, null, 2)}

); const Plain = ({ text }: { text: string }) => (

{text.trim()}

); const TableView = ({ rows }: { rows: string[][] }) => (

{rows[0].map((c, i) => ( ))}{rows.slice(1).map((r, ri) => ( {r.map((c, ci) => ( ))} ))}

{c}
{c}

); // inline markdown: **bold** and `code` const mdInline = (text: string, kb: string): React.ReactNode[] => { const out: React.ReactNode[] = []; const re = /(\*\*([^*]+)\*\*|`([^`]+)`)/; let rest = text, i = 0, m: RegExpMatchArray | null; while ((m = rest.match(re)) && m.index != null) { if (m.index > 0) out.push(rest.slice(0, m.index)); if (m[2] != null) out.push({m[2]}); else out.push({m[3]}); rest = rest.slice(m.index + m[0].length); i++; } if (rest) out.push(rest); return out; }; const looksMd = (s: string) => /(^|\n)#{1,6}\s|\*\*[^*]+\*\*|`[^`]+`|(^|\n)\s*[-*]\s+|(^|\n)\s*\d+\.\s+/.test(s); // minimal block-level markdown renderer (headings, lists, tables, code fences, paragraphs) const Markdown = ({ text }: { text: string }) => { const lines = text.split("\n"); const blocks: React.ReactNode[] = []; let i = 0, k = 0; while (i < lines.length) { const line = lines[i]; if (line.trim().startsWith("```")) { const buf: string[] = []; i++; while (i < lines.length && !lines[i].trim().startsWith("```")) { buf.push(lines[i]); i++; } i++; blocks.push(

{buf.join("\n")}

); continue; } const h = line.match(/^(#{1,6})\s+(.*)/); if (h) { const lv = h[1].length; blocks.push(

{mdInline(h[2], `h${k}`)}

); i++; continue; } // markdown table (header row + |---| separator) if (line.includes("|") && i + 1 < lines.length && /^[\s|:-]+$/.test(lines[i + 1]) && lines[i + 1].includes("-")) { const pr = (l: string) => l.replace(/^\s*\|/, "").replace(/\|\s*$/, "").split("|").map((c) => c.trim()); const rows = [pr(line)]; i += 2; while (i < lines.length && lines[i].includes("|")) { rows.push(pr(lines[i])); i++; } blocks.push(

); continue; } if (/^\s*[-*]\s+/.test(line) || /^\s*\d+\.\s+/.test(line)) { const items: string[] = []; while (i < lines.length && (/^\s*[-*]\s+/.test(lines[i]) || /^\s*\d+\.\s+/.test(lines[i]))) { items.push(lines[i].replace(/^\s*([-*]|\d+\.)\s+/, "")); i++; } blocks.push(

{mdInline(it, `li${k}-${j}`)}

); continue; } if (!line.trim()) { i++; continue; } const para: string[] = []; while (i < lines.length && lines[i].trim() && !/^#{1,6}\s/.test(lines[i]) && !/^\s*[-*]\s+/.test(lines[i]) && !lines[i].trim().startsWith("```")) { para.push(lines[i]); i++; } blocks.push(

{mdInline(para.join(" "), `p${k}`)}

); } return

{blocks}

; }; // render one step's content, formatted by shape (SQL / JSON / markdown / table / text) const Content = ({ step }: { step: Record }) => { const kind = String(step.step); if (kind === "function_call") { const obj = tryJson(String(step.args ?? "")); if (obj && typeof obj === "object" && !Array.isArray(obj) && (obj.sql || obj.query)) { const rest = Object.fromEntries(Object.entries(obj).filter(([k]) => k !== "sql" && k !== "query")); return ( <> {Object.keys(rest).length > 0 &&

{JSON.stringify(rest)}

} ); } if (obj !== undefined) return ; return ; } let body = String(step.body ?? ""); const j = tryJson(body); if (Array.isArray(j)) { // final assistant message: [{text,...}] const txt = j.map((x: any) => x?.text).filter(Boolean).join("\n"); if (!txt) return ; body = txt; } else if (j && typeof j === "object") return ; if (looksMd(body)) return ; const tbl = asTable(body); if (tbl) return ; return ; }; // reusable trace step list (used by the A/B diff and the outlier drill-down) const StepList = ({ steps, loading }: { steps: Record[] | null; loading: boolean }) => { if (steps === null) return pick a job to compare →; if (loading) return ; if (steps.length === 0) return no stored trace; return ( <> {steps.map((s) => { const kind = String(s.step); const isCall = kind === "function_call", isOut = kind === "function_call_output", isUser = kind === "user"; const accent = isCall ? T.blue : isUser ? T.orange : T.inkMuted; const label = isCall ? `↳ ${String(s.fn_name ?? "call")}` : isUser ? "task" : isOut ? "result" : kind; return (

{N(s.step_no)} {label}

); })} ); }; // A/B diff for the selected question: predicted-answer delta + a unified TRACE DIFF // (identical steps collapsed, only −A / +B differences shown). const DiffPanel = () => { if (!sel) return null; const lvl = String(metaByRun.get(sel.a)?.lvl ?? (sel.b ? metaByRun.get(sel.b)?.lvl : "") ?? ""); const predA = String(metaByRun.get(sel.a)?.pred ?? "—"); const predB = sel.b ? String(metaByRun.get(sel.b)?.pred ?? "—") : null; const differ = sel.b != null && predA !== predB; const aSteps = stepsByRun.get(sel.a) ?? []; const bSteps = sel.b ? (stepsByRun.get(sel.b) ?? []) : []; const ops = sel.b ? traceDiff(aSteps, bSteps) : []; const nEq = ops.filter((o) => o.t === "eq").length; const nDiff = ops.length - nEq; const RunPick = ({ which, run, accent }: { which: "A" | "B"; run: string | null; accent: string }) => { const m = run ? metaByRun.get(run) : undefined; return (

{which}

{run && m ? (

{String(m.pred ?? "—")} ${(N(m.cost) * 1000).toFixed(2)}/1k · {(N(m.dur) / 1000).toFixed(0)}s · {N(m.tools)} tools

) : run ? ( not evaluated in this job ) : ( pick a job to diff → )}

); }; // split (side-by-side) diff: align ops into A|B rows, collapse identical runs const stepLabel = (s: Record) => { const k = String(s.step); return k === "function_call" ? `↳ ${String(s.fn_name ?? "call")}` : k === "function_call_output" ? "result" : k === "user" ? "task" : k; }; const cell = (step: Record | undefined, side: "A" | "B") => { if (!step) return

; const accent = side === "A" ? T.coral : T.teal; return (

{side === "A" ? "−" : "+"} {stepLabel(step)}

); }; const renderDiff = () => { type Row = { left?: Record; right?: Record; same?: boolean }; const rows: Row[] = []; let delBuf: Record[] = [], insBuf: Record[] = []; const flushBuf = () => { const n = Math.max(delBuf.length, insBuf.length); for (let k = 0; k < n; k++) rows.push({ left: delBuf[k], right: insBuf[k] }); delBuf = []; insBuf = []; }; for (const op of ops) { if (op.t === "eq") { flushBuf(); rows.push({ left: op.a, right: op.b, same: true }); } else if (op.t === "del") delBuf.push(op.a!); else insBuf.push(op.b!); } flushBuf(); const out: React.ReactNode[] = []; out.push(

A · {shortRun(sel.a)}

B · {sel.b ? shortRun(sel.b) : "—"}

); let same = 0, key = 0; const flushSame = () => { if (same > 0) out.push(

· {same} identical step{same > 1 ? "s" : ""} ·

); same = 0; }; for (const r of rows) { if (r.same) { same++; continue; } flushSame(); out.push(

{cell(r.left, "A")}{cell(r.right, "B")}

); } flushSame(); if (nDiff === 0) out.push(

Identical traces — both jobs solved it the same way.

); return out; }; return (

Task {sel.qid} {lvl} {sel.b != null && ( {differ ? "answers differ" : "same answer"} )} · gold: {gold}

{qText &&

{qText}

}

Trace diff {!sel.b ? "pick job B to diff" : `${nDiff} differing · ${nEq} identical step${nEq === 1 ? "" : "s"}`}

{trace.isLoading ? : !sel.b ? ( ) : renderDiff()}

); }; // reusable correctness matrix — the core "how things change over time" view. // interactive=true wires A/B cell selection (Matrix tab); false renders read-only (Events). const MatrixGrid = ({ verOrder, qRows, interactive }: { verOrder: string[]; qRows: QRow[]; interactive: boolean }) => (

))} {qRows.map((qr) => ( {verOrder.map((ver, i) => { const ok = qr.row[i], prev = i > 0 ? qr.row[i - 1] : null; const flipped = ok != null && prev != null && ok !== prev; const isA = interactive && sel?.qid === qr.qid && sel?.a === ver; const isB = interactive && sel?.qid === qr.qid && sel?.b === ver; return ( ); })} ))}

{verOrder.map((ver) => (	{shortRun(ver)}	↕
#{qr.qid} {qr.lvl}		1 ? T.orange : T.inkMuted, fontWeight: qr.flips > 1 ? 700 : 400 }}> {qr.flips}

); // right-hand slide-out drawer for reading traces vertically const Drawer = ({ open, onClose, children }: { open: boolean; onClose: () => void; children: React.ReactNode }) => { if (!open) return null; return ( <>

{children}

); }; return (

{/* header */}

Eval Explorer

accuracy, cost, tools & tokens across jobs

{/* KPI row */}

{kpi.isLoading ? Array.from({ length: 5 }).map((_, i) => ) : ( <> )}

{/* tabs */}

Overview Matrix Tools Outliers

{/* ── OVERVIEW ── */} {tab === "overview" && (

{renderRunPicker()}

{trendData.length} job{trendData.length === 1 ? "" : "s"}

{/* Cost vs. Accuracy — one dot per run, grouped/colored by model; bubble = avg runtime/question */}

Cost vs. Accuracy · one dot per run · bubble = avg runtime/question {!fullOnly && <> · tip: ‘Full Runs Only’ (in Filter, above) for comparable points}

{trend.isLoading ? : frontier.length === 0 ? (

no runs match the current filter

) : ( `$${v.toFixed(0)}`} tickLine={false} axisLine={{ stroke: T.paper3 }} tick={{ fontFamily: T.mono, fontSize: 10, fill: T.inkMuted }} label={{ value: "cost per 1k trials → (cheaper is left)", position: "insideBottom", offset: -14, fontFamily: T.mono, fontSize: 10, fill: T.inkMuted }} /> { const d = active && payload?.length ? payload[0]?.payload : null; if (!d) return null; return (

{d.rl}

cost/1k tasks: ${(N(d.costTask) * 1000).toFixed(2)}

accuracy: {N(d.y).toFixed(1)}%

tasks: {N(d.tasks)}

avg runtime/question: {N(d.durS).toFixed(1)}s

model: {d.model || "—"}

split: {splitLabel(String(d.split ?? "—"))}

reasoning: {d.reasoning ?? "—"}

); }} /> {/* one series per model — colored + legended, no connecting line between points */} {[...acByModel.entries()].map(([model, pts]) => ( ))} )}

{[...acByModel.keys()].sort().map((m) => )}

best is top-left: more correct for less money. Bubble size is avg wall-clock seconds per question — bigger is slower.

{/* Runtime vs. Accuracy — same runs/grouping/coloring/labels as Cost vs. Accuracy above, swapping cost for avg wall-clock runtime per question on the x-axis. Bubble size is held uniform here (rather than reusing runtime, which is now the x-axis itself and would just double-encode the same value) so the dot size doesn't mislead. */}

Runtime vs. Accuracy · one dot per run · avg wall-clock seconds/question {!fullOnly && <> · tip: ‘Full Runs Only’ (in Filter, above) for comparable points}

{trend.isLoading ? : frontier.length === 0 ? (

no runs match the current filter

) : ( `${v.toFixed(0)}s`} tickLine={false} axisLine={{ stroke: T.paper3 }} tick={{ fontFamily: T.mono, fontSize: 10, fill: T.inkMuted }} label={{ value: "avg runtime/question → (faster is left)", position: "insideBottom", offset: -14, fontFamily: T.mono, fontSize: 10, fill: T.inkMuted }} /> { const d = active && payload?.length ? payload[0]?.payload : null; if (!d) return null; return (

{d.rl}

avg runtime/question: {N(d.durS).toFixed(1)}s

accuracy: {N(d.y).toFixed(1)}%

tasks: {N(d.tasks)}

cost/1k tasks: ${(N(d.costTask) * 1000).toFixed(2)}

model: {d.model || "—"}

split: {splitLabel(String(d.split ?? "—"))}

reasoning: {d.reasoning ?? "—"}

); }} /> {/* one series per model — colored + legended, no connecting line between points */} {[...acByModel.entries()].map(([model, pts]) => ( ))} )}

{[...acByModel.keys()].sort().map((m) => )}

best is top-left: more correct in less time.

)} {/* ── MATRIX + drill-down ── */} {tab === "matrix" && (

{renderRunPicker()}

{qRows.length} tasks differ × {verOrder.length} jobs

▲ flipped to correct · ▼ to incorrect

{/* matrix grid — reusable component; selecting never resizes it */} {matrix.isLoading ? : verOrder.length === 0 ? (

Pick at least one job above.

) : qRows.length === 0 ? (

No disagreement — every task is unanimous across the {verOrder.length} selected job(s).

) : ( )} {verOrder.length > 0 && !matrix.isLoading && (

Pick two cells in the same row — A then B — to open the diff. Click a selected cell to deselect it.

)} setSel(null)}>

)} {/* ── TOOLS ── which tools the agent called, normalized per task across the arc */} {tab === "tools" && (

{renderRunPicker()} {/* tokens per tool call — aggregate ratio (total tokens ÷ total tool calls), same line idiom as Overview */}

v.toFixed(1)} suffix="k" />

Tool calls per trial, by tool

{toolsQ.isLoading ? : (

{toolNames.map((t) => ( ))}

)} {/* per-tool summary: first vs last version of the arc */} {!toolsQ.isLoading && toolData.length >= 2 && (

{toolNames.map((t) => { const a = N(toolData[0][t]), b = N(toolData[toolData.length - 1][t]); const d = Math.round((b - a) * 10) / 10; return ( ); })}

Tool	{toolData[0].v} calls/trial	{toolData[toolData.length - 1].v} calls/trial	Δ
{t}	{a.toFixed(1)}	{b.toFixed(1)}	0 ? T.coral : T.inkMuted }}>{d > 0 ? "+" : ""}{d.toFixed(1)}

)}

)} {/* ── OUTLIERS ── distribution + what's different about the misses */} {tab === "outliers" && (

{renderRunPicker()}

Cost vs. latency · click a point for its trace · drag a box to filter the table beside it →

{/* scatter (left) */} {dist.isLoading ? : (

{scatterEl}

{box && ( )}

)} {/* comparison table (right, same level) — transposed: metric rows × correct/incorrect cols */} {dist.isLoading ? : (

{box ? `In the box · ${shownPts.length} trials` : "All shown trials"}

{[ { lbl: "trials", f: (r: any) => N(r.n).toLocaleString() }, { lbl: "avg tool calls", f: (r: any) => N(r.tools).toFixed(1) }, { lbl: "avg duration", f: (r: any) => `${N(r.dur_s).toFixed(1)}s` }, { lbl: "avg cost / 1k tasks", f: (r: any) => `$${(N(r.cost) * 1000).toFixed(2)}` }, { lbl: "% hard", f: (r: any) => `${N(r.pct_hard).toFixed(0)}%` }, { lbl: "avg SQL errors", f: (r: any) => N(r.sql_errors).toFixed(2) }, ].map((m) => ( ))}


{m.lbl}	{okRow ? m.f(okRow) : "—"}	{badRow ? m.f(badRow) : "—"}

)}

{/* trace region — the clicked point's trace */} setOSel(null)}> {oSel && (

{shortRun(oSel.rl)} · Q{oSel.qid} {oDetail && } {oDetail && ${(N(oDetail.cost) * 1000).toFixed(2)}/1k · {(N(oDetail.dur) / 1000).toFixed(0)}s · {N(oDetail.tools)} tools }

{oDetail?.q &&

{String(oDetail.q)}

} {oDetail &&

predicted: {String(oDetail.pred ?? "—")} · gold: {String(oDetail.gold ?? "—")}

}

)}

); } function Lg({ c, t, border }: { c: string; t: string; border?: boolean }) { return ( {t} ); } function Cap({ c, t }: { c: string; t: string }) { return {t}; } function tryJson(s: string): any { const t = s.trim(); if (!t || (t[0] !== "{" && t[0] !== "[")) return undefined; try { return JSON.parse(t); } catch { return undefined; } } // detect a pipe- or whitespace-delimited result table → rows of cells (first row = header) function asTable(text: string): string[][] | null { const lines = text.trim().split("\n").filter((l) => l.trim()); if (lines.length < 2) return null; const pipe = lines.every((l) => l.includes(" | ")); if (!pipe) return null; const rows = lines.map((l) => l.split(" | ").map((c) => c.trim())); const w = rows[0].length; if (w < 2 || !rows.every((r) => r.length === w)) return null; return rows; } // normalized signature for a trace step, used to align two traces function stepSig(s: Record): string { const kind = String(s.step); return `${kind}::${kind === "function_call" ? String(s.args ?? "") : String(s.body ?? "")}`; } // classic LCS diff over two step arrays → ordered ops (eq / del=only-in-A / ins=only-in-B) type DiffOp = { t: "eq" | "del" | "ins"; a?: Record; b?: Record }; function traceDiff(a: Record[], b: Record[]): DiffOp[] { const n = a.length, m = b.length; const L: number[][] = Array.from({ length: n + 1 }, () => new Array(m + 1).fill(0)); for (let i = n - 1; i >= 0; i--) for (let j = m - 1; j >= 0; j--) L[i][j] = stepSig(a[i]) === stepSig(b[j]) ? L[i + 1][j + 1] + 1 : Math.max(L[i + 1][j], L[i][j + 1]); const ops: DiffOp[] = []; let i = 0, j = 0; while (i < n && j < m) { if (stepSig(a[i]) === stepSig(b[j])) { ops.push({ t: "eq", a: a[i], b: b[j] }); i++; j++; } else if (L[i + 1][j] >= L[i][j + 1]) { ops.push({ t: "del", a: a[i] }); i++; } else { ops.push({ t: "ins", b: b[j] }); j++; } } while (i < n) { ops.push({ t: "del", a: a[i] }); i++; } while (j < m) { ops.push({ t: "ins", b: b[j] }); j++; } return ops; } type QRow = { qid: string; lvl: string; row: (boolean | null)[]; flips: number }; // pivot eval rows {ver,qid,ok,lvl,ordk} → chronological run columns + per-question rows with flip count. function buildMatrix(rows: Record[]): { verOrder: string[]; qRows: QRow[] } { const ordOf = new Map(); for (const r of rows) ordOf.set(String(r.ver), String(r.ordk)); const verOrder = Array.from(ordOf.keys()).sort((a, b) => (ordOf.get(a) ?? "").localeCompare(ordOf.get(b) ?? "")); const grid = new Map>(); const levelOf = new Map(); for (const r of rows) { const qid = String(r.qid), ver = String(r.ver); if (!grid.has(qid)) grid.set(qid, new Map()); grid.get(qid)!.set(ver, r.ok == null ? null : Boolean(r.ok)); if (r.lvl) levelOf.set(qid, String(r.lvl)); } const qRows: QRow[] = Array.from(grid.keys()).map((qid) => { const row = verOrder.map((ver) => grid.get(qid)!.get(ver) ?? null); let flips = 0; for (let i = 1; i < row.length; i++) if (row[i] != null && row[i - 1] != null && row[i] !== row[i - 1]) flips++; return { qid, lvl: levelOf.get(qid) ?? "", row, flips }; }).sort((a, b) => b.flips - a.flips || Number(a.qid) - Number(b.qid)); return { verOrder, qRows }; }

Agentic SQL Benchmark Including Local Models

Comments