import { useState, useEffect, useCallback, useRef, useMemo } from "react"; import { useSQLQuery } from "@motherduck/react-sql-query"; import { BarChart, Bar, XAxis, YAxis, CartesianGrid, Tooltip, ResponsiveContainer, LineChart, Line, Cell, Legend, ReferenceLine } from "recharts"; import { Loader2 } from "lucide-react"; export const REQUIRED_DATABASES = [ { type: 'share', path: 'md:_share/agentic_sql_results_public_share/c448b0a6-0c59-4711-ac60-ed61ff1b773c', alias: 'agentic_sql_results_public' } ]; const N = (v: unknown): number => (v != null ? Number(v) : 0); // ── Constants ────────────────────────────────────────────────────── // Canonical run IDs const BASELINE_RUN = "dabstep_test_1774297813"; // 424 questions, 38.2% const V4_PLUS_RUN = "dabstep_test_1774311322"; // 352 questions, 93.2% // Hero question that fails badly in baseline (40 trace steps) const HERO_QUESTION = "1765"; // Slide build counts: [title, transition, hardQ, justTables, trace, comments, views, twitter, macros, scoreboard] // Trace slide: 1 build. Internal stepping handled by slide via traceAdvance callback. const TRACE_BUILDS = 1; // Twitter slide: each build reveals one cycle (>= 3 runs each) // 6 valid cycles. +1 title, +1 baseline ref line, +1 KPI = 9 builds const TWITTER_BUILDS = 9; // [title, transition, dabstep, hardQ, harness, justTables, trace, whatAreComments, comments, views, twitter, v4iteration, v4explain, scoreboard, openQ] const SLIDE_BUILD_COUNTS = [3, 3, 4, 2, 4, 4, TRACE_BUILDS, 4, 3, 5, TWITTER_BUILDS, 8, 4, 4, 1]; // ── Presentation shell ───────────────────────────────────────────── export default function Presentation() { const [slide, setSlide] = useState(0); const [build, setBuild] = useState(0); const maxBuilds = SLIDE_BUILD_COUNTS[slide] || 1; const advance = useCallback(() => { if (build < maxBuilds - 1) { setBuild(b => b + 1); } else if (slide < SLIDE_BUILD_COUNTS.length - 1) { setSlide(s => s + 1); setBuild(0); } }, [build, maxBuilds, slide]); const retreat = useCallback(() => { if (build > 0) { setBuild(b => b - 1); } else if (slide > 0) { const prevSlide = slide - 1; setSlide(prevSlide); setBuild(SLIDE_BUILD_COUNTS[prevSlide] - 1); } }, [build, slide]); useEffect(() => { const handler = (e: KeyboardEvent) => { if (e.key === "ArrowRight" || e.key === "ArrowDown" || e.key === " ") { e.preventDefault(); advance(); } if (e.key === "ArrowLeft" || e.key === "ArrowUp") { e.preventDefault(); retreat(); } }; window.addEventListener("keydown", handler); return () => window.removeEventListener("keydown", handler); }, [advance, retreat, slide]); return (

{/* Slide number */}

{slide + 1} / {SLIDE_BUILD_COUNTS.length}

{/* Render all slides, show only the active one. This keeps query caches alive across navigation so data never "pops in" on revisited slides. */} {[ , , , , , , , , , , , , , , , ].map((el, i) => (

{el}

))}

); } // ── Shared styles ────────────────────────────────────────────────── // Build-state reveal: hidden elements show a skeleton placeholder at fixed height. // When visible, real content fades in. No layout shift. // Children are always rendered (but hidden) to avoid remounting/resize flicker. function BuildReveal({ visible, children, height }: { visible: boolean; children: React.ReactNode; height?: number; }) { return (

{/* Skeleton shown when not visible */}

{/* Content always rendered, fades in */}

{children}

); } // Simple fade for elements that don't need skeleton (titles, text that won't shift) const fadeIn = (visible: boolean): React.CSSProperties => ({ opacity: visible ? 1 : 0, transition: "opacity 0.4s ease", pointerEvents: visible ? "auto" : "none", }); // ── MotherDuck Design Tokens ─────────────────────────────────────── const MD = { bg: "#F4EFEA", // warm cream background text: "#383838", // primary text muted: "#706860", // secondary text (darkened for legibility on cream) blue: "#6FC2FF", // primary accent blueHover: "#2BA5FF", // deeper blue yellow: "#FFDE00", // highlight teal: "#53DBC9", // success/positive red: "#FF7169", // error/negative border: "#383838", // borders white: "#FFFFFF", lightGray: "#F8F8F7", darkGray: "#D7D7D7", fontHeading: "'Space Mono', monospace", fontBody: "'Inter', sans-serif", fontMono: "'Space Mono', monospace", }; const slideContainer: React.CSSProperties = { width: "100%", maxWidth: 900, padding: "48px 64px", fontFamily: MD.fontBody, }; const h1Style: React.CSSProperties = { fontSize: 48, fontWeight: 700, color: MD.text, lineHeight: 1.15, margin: 0, fontFamily: MD.fontHeading, textTransform: "uppercase", letterSpacing: -0.5, }; const h2Style: React.CSSProperties = { fontSize: 36, fontWeight: 700, color: MD.text, lineHeight: 1.2, margin: 0, fontFamily: MD.fontHeading, }; const bodyStyle: React.CSSProperties = { fontSize: 20, color: MD.muted, lineHeight: 1.5, margin: 0, fontFamily: MD.fontBody, fontWeight: 400, }; const codeBlockStyle: React.CSSProperties = { background: MD.text, color: "#e8e8e8", padding: "16px 20px", borderRadius: 2, border: `2px solid ${MD.border}`, fontFamily: MD.fontMono, fontSize: 15, lineHeight: 1.6, overflowX: "auto", }; const kpiStyle: React.CSSProperties = { fontSize: 44, fontWeight: 700, color: MD.text, margin: 0, lineHeight: 1, fontFamily: MD.fontHeading, }; const kpiLabelStyle: React.CSSProperties = { fontSize: 14, color: MD.muted, marginTop: 4, fontFamily: MD.fontBody, fontWeight: 400, }; const inlineCode: React.CSSProperties = { background: MD.lightGray, padding: "2px 6px", borderRadius: 2, fontFamily: MD.fontMono, fontSize: "0.9em", border: `1px solid ${MD.darkGray}`, }; // ── Slide 1: Title ───────────────────────────────────────────────── function SlideTitle({ build }: { build: number }) { return (

= 0)}>

This Is a Talk About SQL...

= 1)}>

...and AI

= 2), marginTop: 48 }}>

Mike Xu and Jacob Matson · MotherDuck · Data Engineer Things

); } // ── Slide 2: Transition ──────────────────────────────────────────── function SlideTransition({ build }: { build: number }) { return (

= 0)}>

We spent a few months trying to make AI even better at writing SQL.

= 1), marginTop: 32 }}>

We ran thousands of benchmark questions across five different database configurations.

= 2), marginTop: 32 }}>

The answer: build a better data model. Not a better AI.

); } // ── Slide 3: What is DABstep ─────────────────────────────────────── function SlideWhatIsDABstep({ build }: { build: number }) { return (

= 0)}>

What is DABstep?

A benchmark from Adyen: 460 payment processing questions, answered programmatically.

= 1} height={60}>

Easy questions (72): single-table lookups. "Which country has the most transactions?"

Hard questions (352+): multi-table joins with domain logic, fee calculations, simulations.

= 2} height={80}>

How we evaluate:

Each question comes with guidelines — what format to return, what type of answer is expected. Agent writes SQL, executes it, returns an answer. Result matches guidelines = correct. No partial credit.

= 3} height={56}>

36.4%

easy · original paper (o4-mini)

14.55%

hard · original paper (o4-mini)

); } // ── Slide 4: A Hard Question ────────────────────────────────────── function SlideHardQuestion({ build }: { build: number }) { return (

{/* Easy question */}

= 0)}>

Easy question — one table

"What percentage of transactions are made using credit cards?"

SELECT ROUND(100.0 * SUM(CASE WHEN is_credit THEN 1 END) / COUNT(*), 1) FROM payments

{/* Hard question */} = 1} height={260}>

Hard question — 4 tables, 9-dimension join

"What were the applicable Fee IDs for Crossfit_Hanna in March 2023?"

{`...
JOIN fees f ON (f.card_scheme = p.card_scheme OR f.card_scheme IS NULL)
  AND (f.aci = p.aci OR f.aci IS NULL)
  AND (f.merchant_category_code = m.mcc OR f.merchant_category_code IS NULL)
  AND (f.is_credit = p.is_credit OR f.is_credit IS NULL)
  AND (f.intracountry = ... OR f.intracountry IS NULL)
  AND (f.capture_delay = m.capture_delay OR f.capture_delay IS NULL)
  AND (f.monthly_volume = ... OR f.monthly_volume IS NULL)
  AND (f.monthly_fraud_level = ... OR f.monthly_fraud_level IS NULL)
  AND (f.account_type = ... OR f.account_type IS NULL)
...`}

); } // ── Slide 5: Our Harness ────────────────────────────────────────── function SlideOurHarness({ build }: { build: number }) { return (

= 0)}>

Our Setup

Same model every time (Gemini 3 Flash). Same tools. The only variable is the agent context.

{/* Tables */}

= 1), marginTop: 24 }}>

5 tables

{["payments", "fees", "merchants", "merchant_category_codes", "acquirer_countries"].map(t => (

{t}

))}

= 2), marginTop: 16 }}>

2 metadata files

2 prompts

{/* Tools */}

= 3), marginTop: 20 }}>

Tools (via MCP)

{["query", "list_tables", "list_columns"].map(tool => (

{tool}

))}

{[ { name: "ask_docs", note: "baseline/v1/v2" }, { name: "search_fragments", note: "v3" }, ].map(tool => (

{tool.name} ({tool.note})

))}

); } // ── Slide 6: What Are Column Comments ───────────────────────────── function SlideWhatAreComments({ build }: { build: number }) { return (

= 0)}>

Column Comments

An old part of the SQL spec that nobody used... until now.

= 1} height={70}>

{`COMMENT ON COLUMN fees.monthly_volume IS
  'Bucketed: <100k < 100k-1m < 1m-5m < >5m. NULL = matches all.';`}

= 2} height={60}>

Invisible to humans. Nobody reads them. No BI tool surfaces them. They just sit in the catalog.

= 3} height={60}>

Perfect for AI. The agent calls{" "} list_columns and gets every comment. It's metadata that goes straight into the context window.

); } // ── Slide 4: Just the Tables ─────────────────────────────────────── function SlideJustTheTables({ build }: { build: number }) { const baselineSplit = useSQLQuery(` SELECT CASE WHEN question_id::INT <= 100 THEN 'easy' ELSE 'hard' END as difficulty, COUNT(*) as total, ROUND(AVG(CASE WHEN (evaluation::JSON->>'is_correct')::BOOLEAN THEN 1 ELSE 0 END) * 100, 1) as accuracy, ROUND(SUM(estimated_cost_usd), 2) as cost FROM "agentic_sql_results_public"."main"."results" WHERE run_id = '${BASELINE_RUN}' GROUP BY difficulty ORDER BY difficulty `); const heroQ = useSQLQuery(` SELECT question_id, question_text, (evaluation::JSON->>'is_correct')::BOOLEAN as correct, evaluation::JSON->>'predicted_result' as predicted, evaluation::JSON->>'gold_result' as gold, ROUND(duration_ms/1000, 1) as duration_sec, json_array_length(raw_response::JSON->'messages') as trace_steps FROM "agentic_sql_results_public"."main"."results" WHERE run_id = '${BASELINE_RUN}' AND question_id = '${HERO_QUESTION}' `); const splits = Array.isArray(baselineSplit.data) ? baselineSplit.data : []; const easy = splits.find(r => String(r.difficulty) === "easy"); const hard = splits.find(r => String(r.difficulty) === "hard"); const totalCost = N(easy?.cost) + N(hard?.cost); const hero = Array.isArray(heroQ.data) ? heroQ.data : []; return (

= 0)}>

Baseline: Just the Data

Raw schema. No comments, no views. Full manual via ask_docs.

{baselineSplit.isLoading ? (

) : ( <>

{N(easy?.accuracy)}%

+{(N(easy?.accuracy) - 36.4).toFixed(1)} vs paper )}

easy questions ({N(easy?.total)})

single-table, no joins

{baselineSplit.isLoading ? (

) : ( <>

{N(hard?.accuracy)}%

+{(N(hard?.accuracy) - 14.55).toFixed(1)} vs paper )}

hard questions ({N(hard?.total)})

multi-table joins, domain logic

{baselineSplit.isLoading ? (

) : (

${totalCost}

)}

total cost

= 1), marginTop: 32 }}> {hero.length > 0 && (

Our hero question

"{String(hero[0]?.question_text)}"

)}

= 2), marginTop: 16 }}> {hero.length > 0 && (

{N(hero[0]?.trace_steps)} trace steps

{N(hero[0]?.duration_sec)}s duration

Result: {String(hero[0]?.predicted)}

)}

= 3), marginTop: 16 }}> {hero.length > 0 && (

Expected: {String(hero[0]?.gold).slice(0, 80)}...

)}

); } // ── Slide 5: Trace Visualization ─────────────────────────────────── // Parse MCP tool output: unwraps [{"type":"input_text","text":"..."}] and pretty-prints JSON type TraceContent = { text: string; isJson: boolean; isSql: boolean; table?: { columns: string[]; rows: (string | null)[][] } }; // Parse MCP tool output. If it's a query result with columns/rows, extract table data. function formatTraceContent(raw: string): TraceContent { if (!raw) return { text: "", isJson: false, isSql: false }; let str = raw; // Unwrap MCP array wrapper try { const arr = JSON.parse(str); if (Array.isArray(arr) && arr[0]?.text) str = arr[0].text; } catch {} // Try to parse as JSON try { const obj = JSON.parse(str); // Check if it's a query result with columns and rows if (obj.columns && Array.isArray(obj.columns) && obj.rows && Array.isArray(obj.rows)) { const maxRows = 8; // limit rows shown return { text: "", isJson: true, isSql: false, table: { columns: obj.columns as string[], rows: (obj.rows as (string | null)[][]).slice(0, maxRows), }, }; } // Check for error responses if (obj.error) { return { text: obj.error, isJson: false, isSql: false }; } // Generic JSON return { text: JSON.stringify(obj, null, 2), isJson: true, isSql: false }; } catch {} return { text: str, isJson: false, isSql: false }; } // Render a mini table from query results function MiniTable({ columns, rows }: { columns: string[]; rows: (string | null)[][] }) { const cellSt: React.CSSProperties = { padding: "2px 8px", fontSize: 10, fontFamily: MD.fontMono, borderBottom: `1px solid ${MD.darkGray}`, whiteSpace: "nowrap", maxWidth: 140, overflow: "hidden", textOverflow: "ellipsis", }; return (

{columns.map((col, i) => ( ))} {rows.map((row, ri) => ( {row.map((val, ci) => ( ))} ))}

{col}
{val === null ? "NULL" : String(val)}

); } function SlideTrace({ build }: { build: number }) { const [selectedQuestion, setSelectedQuestion] = useState(HERO_QUESTION); const [visibleCount, setVisibleCount] = useState(1); const [isPlaying, setIsPlaying] = useState(false); const playIntervalRef = useRef | null>(null); // Get available questions with their metadata for the dropdown const questionList = useSQLQuery(` SELECT question_id, LEFT(question_text, 90) as question_short, (evaluation::JSON->>'is_correct')::BOOLEAN as correct, json_array_length(raw_response::JSON->'messages') as trace_steps, ROUND(duration_ms/1000, 1) as duration_sec FROM "agentic_sql_results_public"."main"."results" WHERE run_id = '${BASELINE_RUN}' ORDER BY json_array_length(raw_response::JSON->'messages') DESC LIMIT 30 `); const questions = (Array.isArray(questionList.data) ? questionList.data : []).map(r => ({ id: String(r.question_id), label: String(r.question_short), correct: Boolean(r.correct), steps: N(r.trace_steps), duration: N(r.duration_sec), })); const questionDetail = useSQLQuery(` SELECT question_text, (evaluation::JSON->>'is_correct')::BOOLEAN as correct, evaluation::JSON->>'predicted_result' as predicted, evaluation::JSON->>'gold_result' as gold, hit_turn_limit, evaluation::JSON->>'partial_reason' as partial_reason FROM "agentic_sql_results_public"."main"."results" WHERE run_id = '${BASELINE_RUN}' AND question_id = '${selectedQuestion}' `); const detail = Array.isArray(questionDetail.data) ? questionDetail.data : []; const traceData = useSQLQuery(` WITH msg_array AS ( SELECT unnest(from_json(raw_response::JSON->'messages', '["json"]')) as msg, generate_subscripts(from_json(raw_response::JSON->'messages', '["json"]'), 1) as idx FROM "agentic_sql_results_public"."main"."results" WHERE run_id = '${BASELINE_RUN}' AND question_id = '${selectedQuestion}' ) SELECT idx, COALESCE(msg->>'role', '') as role, COALESCE(msg->>'type', '') as type, COALESCE(msg->>'name', '') as name, COALESCE(msg->>'arguments', '') as arguments, COALESCE(msg->>'output', '') as output, COALESCE(msg->'content'->0->>'text', msg->>'content', '') as content_text FROM msg_array ORDER BY idx `); const scrollRef = useRef(null); const steps = (Array.isArray(traceData.data) ? traceData.data : []) .map(r => ({ idx: N(r.idx), role: String(r.role || ""), type: String(r.type || ""), name: String(r.name || ""), arguments: String(r.arguments || ""), output: String(r.output || ""), contentText: String(r.content_text || ""), })) .filter(s => !( (s.role === "assistant" || s.type === "message") && (!s.contentText || s.contentText === "thought:" || s.contentText.trim() === "") )); const allDone = visibleCount >= steps.length && steps.length > 0; const visibleSteps = steps.slice(0, Math.min(visibleCount, steps.length)); // Advance one step const advanceStep = useCallback(() => { setVisibleCount(c => { if (c >= steps.length) return c; return c + 1; }); }, [steps.length]); // Play/pause auto-advance const togglePlay = useCallback(() => { setIsPlaying(p => !p); }, []); // Stop playing when all steps are shown useEffect(() => { if (allDone && isPlaying) setIsPlaying(false); }, [allDone, isPlaying]); // Auto-play interval useEffect(() => { if (isPlaying) { playIntervalRef.current = setInterval(() => { setVisibleCount(c => c + 1); }, 750); } else { if (playIntervalRef.current) clearInterval(playIntervalRef.current); playIntervalRef.current = null; } return () => { if (playIntervalRef.current) clearInterval(playIntervalRef.current); }; }, [isPlaying]); // Scroll to bottom as steps appear useEffect(() => { if (scrollRef.current && visibleCount > 1) { scrollRef.current.scrollTo({ top: scrollRef.current.scrollHeight, behavior: "smooth" }); } }, [visibleCount]); // Reset on question change useEffect(() => { setVisibleCount(1); setIsPlaying(false); if (scrollRef.current) scrollRef.current.scrollTo({ top: 0 }); }, [selectedQuestion]); const getStepInfo = (step: typeof steps[0]) => { if (step.role === "user") return { label: "Question", color: MD.blue }; if (step.type === "function_call") return { label: step.name, color: MD.blueHover }; if (step.type === "function_call_output") { const hasError = step.output.includes("error") || step.output.includes("Error"); return { label: "response", color: hasError ? MD.red : MD.teal }; } if (step.role === "assistant" || step.type === "message") return { label: "thinking", color: MD.muted }; return { label: step.type || step.role || "?", color: MD.muted }; }; const getStepContent = (step: typeof steps[0]): TraceContent | null => { if (step.role === "user") { return { text: step.contentText, isJson: false, isSql: false }; } if (step.type === "function_call") { try { const args = JSON.parse(step.arguments); if (args.sql) return { text: args.sql, isJson: false, isSql: true }; return { text: JSON.stringify(args, null, 2), isJson: true, isSql: false }; } catch { return step.arguments ? { text: step.arguments, isJson: false, isSql: false } : null; } } if (step.type === "function_call_output") { return formatTraceContent(step.output); } if (step.contentText && step.contentText !== "thought:") { return { text: step.contentText.slice(0, 400), isJson: false, isSql: false }; } return null; }; const isCorrect = detail.length > 0 ? Boolean(detail[0]?.correct) : false; return (

{/* Header row */}

Baseline

{/* Play/pause button */}

▾

{Math.min(visibleCount, steps.length)}/{steps.length}

{/* Question as subtitle */} {detail.length > 0 && (

"{String(detail[0]?.question_text)}"

)} {/* Trace as chat-like messages */} {traceData.isLoading ? (

Loading trace...

) : (

{visibleSteps.map((step) => { const { label, color } = getStepInfo(step); const content = getStepContent(step); return (

{/* Chat header */}

{label} {step.type === "function_call" && ( calling tool )} {step.type === "function_call_output" && color === MD.red && ( ERROR )}

{/* Content */} {content && content.table ? ( ) : content && content.text ? (

{content.text}

) : null}

); })} {/* Typing indicator while playing */} {isPlaying && !allDone && (

thinking...

)} {/* Result banner */} {allDone && detail.length > 0 && (

{Boolean(detail[0]?.hit_turn_limit) ? `RAN OUT OF TURNS — gave up with "${String(detail[0]?.predicted).slice(0, 40)}"` : isCorrect ? "CORRECT" : `WRONG — Got: ${String(detail[0]?.predicted).slice(0, 40)}` } {!isCorrect && ` · Expected: ${String(detail[0]?.gold).slice(0, 50)}`}

)}

); } // ── Slide 6: Column Comments ─────────────────────────────────────── function SlideColumnComments({ build }: { build: number }) { // Split by easy/hard for baseline vs v1 const tierSplit = useSQLQuery(` SELECT db_config as tier, CASE WHEN question_id::INT <= 100 THEN 'easy' ELSE 'hard' END as difficulty, COUNT(*) as questions, ROUND(AVG(CASE WHEN (evaluation::JSON->>'is_correct')::BOOLEAN THEN 1 ELSE 0 END) * 100, 1) as accuracy FROM "agentic_sql_results_public"."main"."results" WHERE run_id IN ('dabstep_test_1774297813', 'dabstep_test_1774299226') GROUP BY db_config, difficulty ORDER BY db_config, difficulty `); // Trace steps for baseline vs v1, correct vs incorrect const turnsData = useSQLQuery(` SELECT db_config as tier, (evaluation::JSON->>'is_correct')::BOOLEAN as correct, ROUND(AVG(json_array_length(raw_response::JSON->'messages')), 1) as avg_steps FROM "agentic_sql_results_public"."main"."results" WHERE benchmark = 'dabstep' AND model = 'google/gemini-3-flash-preview' AND run_id IN ('dabstep_test_1774297813', 'dabstep_test_1774299226') GROUP BY tier, correct ORDER BY tier, correct `); const turnsRows = Array.isArray(turnsData.data) ? turnsData.data : []; const baseCorrectSteps = N(turnsRows.find(r => String(r.tier) === "baseline" && Boolean(r.correct))?.avg_steps); const baseWrongSteps = N(turnsRows.find(r => String(r.tier) === "baseline" && !Boolean(r.correct))?.avg_steps); const v1CorrectSteps = N(turnsRows.find(r => String(r.tier) === "v1" && Boolean(r.correct))?.avg_steps); const v1WrongSteps = N(turnsRows.find(r => String(r.tier) === "v1" && !Boolean(r.correct))?.avg_steps); const splits = Array.isArray(tierSplit.data) ? tierSplit.data : []; const baseEasy = splits.find(r => String(r.tier) === "baseline" && String(r.difficulty) === "easy"); const baseHard = splits.find(r => String(r.tier) === "baseline" && String(r.difficulty) === "hard"); const v1Easy = splits.find(r => String(r.tier) === "v1" && String(r.difficulty) === "easy"); const v1Hard = splits.find(r => String(r.tier) === "v1" && String(r.difficulty) === "hard"); const chartData = [ { label: "Easy (no joins)", baseline: N(baseEasy?.accuracy), comments: N(v1Easy?.accuracy) }, { label: "Hard (multi-table)", baseline: N(baseHard?.accuracy), comments: N(v1Hard?.accuracy) }, ]; return (

= 0)}>

V1: Add Column Comments

{`fees.monthly_volume: Bucketed (ordered):
  <100k < 100k-1m < 1m-5m < >5m.
  NULL = matches all.

fees.rate: Fee = fixed_amount + (rate * eur_amount / 10000)`}

{/* Accuracy rows */} = 1} height={100}>

{/* Easy row */}

Easy questions

single-table, no joins

{N(baseEasy?.accuracy)}%

→

{N(v1Easy?.accuracy)}%

{/* Hard row */}

Hard questions

multi-table joins, domain logic

{N(baseHard?.accuracy)}%

→

{N(v1Hard?.accuracy)}%

within confidence interval

= 2} height={32}>

Comments describe columns. The hard logic lives in the joins between tables. There's nowhere to put "these four tables connect across nine dimensions with wildcard matching."

); } // ── Shared: ERD box component ────────────────────────────────────── const erdBoxColors = { table: { bg: MD.lightGray, border: MD.border, header: MD.bg }, view: { bg: "#dff0ff", border: MD.blueHover, header: "white" }, prompt: { bg: "#eae6e2", border: "#5C534A", header: MD.bg }, tool: { bg: "#e0e0e0", border: MD.text, header: MD.bg }, newView: { bg: "#dff0ff", border: MD.blueHover, header: "white" }, }; function ErdBox({ title, items, type = "table", visible = true, isNew = false, isModified = false }: { title: string; items: string[]; type?: keyof typeof erdBoxColors; visible?: boolean; isNew?: boolean; isModified?: boolean; }) { const c = erdBoxColors[type]; return (

{title} {isNew && NEW} {isModified && MODIFIED}

{items.map((item, i) => (

{item}

))}

); } // ── Slide 7: Build Views (ERD visual) ────────────────────────────── function SlideBuildViews({ build }: { build: number }) { const v2Stats = useSQLQuery(` SELECT CASE WHEN question_id::INT <= 100 THEN 'easy' ELSE 'hard' END as difficulty, COUNT(*) as questions, ROUND(AVG(CASE WHEN (evaluation::JSON->>'is_correct')::BOOLEAN THEN 1 ELSE 0 END) * 100, 1) as accuracy, ROUND(SUM(estimated_cost_usd), 2) as cost FROM "agentic_sql_results_public"."main"."results" WHERE run_id = 'dabstep_test_1772234580' GROUP BY difficulty ORDER BY difficulty `); const splits = Array.isArray(v2Stats.data) ? v2Stats.data : []; const easy = splits.find(r => String(r.difficulty) === "easy"); const hard = splits.find(r => String(r.difficulty) === "hard"); const totalCost = N(easy?.cost) + N(hard?.cost); const baseTables = [ { title: "payments", items: ["psp_reference", "merchant", "card_scheme", "eur_amount", "has_fraudulent_dispute", "...12 more columns"] }, { title: "fees", items: ["ID", "card_scheme", "account_type", "fixed_amount", "rate", "...9 more columns"] }, { title: "merchants", items: ["merchant", "capture_delay", "acquirer_country", "account_type"] }, ]; const smallTables = [ { title: "merchant_category_codes", items: ["unnamed: 0", "mcc", "description"] }, { title: "acquirer_countries", items: ["acquirer_country"] }, ]; return (

= 0)}>

V2: Build Views + Enrich Everything

V1 had comments on the base tables. V2 added views, richer prompts, patterns, and a docs tool.

{/* ERD: Base tables (always visible) */}

= 0), marginTop: 16, display: "flex", gap: 10, flexWrap: "wrap" }}> {baseTables.map(t => )} {smallTables.map(t => )}

{/* ERD: Views (build >= 1) */} = 1} height={100}>

{/* Prompts + tools (build >= 2) */} = 2} height={80}>

{/* Result KPI (build >= 3) */} = 3} height={80}>

{v2Stats.isLoading ? (

) : (

{N(easy?.accuracy)}%

)}

easy ({N(easy?.questions)} questions)

{v2Stats.isLoading ? (

) : (

{N(hard?.accuracy)}%

)}

hard ({N(hard?.questions)} questions)

{v2Stats.isLoading ? (

) : (

${totalCost}

)}

total cost (down from $10+)

{/* Insight (build >= 4) */}

= 4), marginTop: 12 }}>

The model discovers views through list_tables. No new tools needed. We didn't make the AI smarter — we made the data model better.

); } // ── Slide 8: Twitter Detour ──────────────────────────────────────── function SlideTwitter({ build }: { build: number }) { // Query with cycle detection: gaps > 2 hours = new cycle (learn/maintain happened) const v3Runs = useSQLQuery(` WITH ordered AS ( SELECT run_id, CAST(split_part(run_id, '_', 3) AS BIGINT) as ts, ROUND(AVG(CASE WHEN (evaluation::JSON->>'is_correct')::BOOLEAN THEN 1 ELSE 0 END) * 100, 1) as accuracy, ROW_NUMBER() OVER (ORDER BY run_id) as rn FROM "agentic_sql_results_public"."main"."results" WHERE benchmark = 'dabstep' AND db_config = 'v3' AND model = 'google/gemini-3-flash-preview' AND run_id LIKE 'dabstep_train%' GROUP BY run_id HAVING COUNT(*) = 26 ORDER BY run_id LIMIT 40 ) SELECT rn, accuracy, CASE WHEN COALESCE(ts - LAG(ts) OVER (ORDER BY rn), 0) > 7200 THEN true ELSE false END as new_cycle FROM ordered ORDER BY rn `); const allPoints = (Array.isArray(v3Runs.data) ? v3Runs.data : []).map(r => ({ run: N(r.rn), accuracy: N(r.accuracy), newCycle: Boolean(r.new_cycle), })); // Compute cycle indices for each point let cycleId = 0; const cycleBreaks: number[] = [0]; // run indices where new cycles start const pointsWithCycle = allPoints.map((p, i) => { if (i > 0 && p.newCycle) { cycleId++; cycleBreaks.push(i); } return { ...p, cycle: cycleId }; }); const totalCycles = cycleId + 1; // Group points into cycles, filter out cycles with < 3 points const cycleGroups: { cycle: number; points: { step: number; accuracy: number }[] }[] = []; let currentGroup: typeof cycleGroups[0] | null = null; for (const p of pointsWithCycle) { if (!currentGroup || p.cycle !== currentGroup.cycle) { currentGroup = { cycle: p.cycle, points: [] }; cycleGroups.push(currentGroup); } currentGroup.points.push({ step: currentGroup.points.length + 1, accuracy: p.accuracy }); } const validCycles = cycleGroups.filter(g => g.points.length >= 3); const totalValidCycles = validCycles.length; // Build 0 = title + subtitle // Build 1 = empty chart axes + V2 baseline ref line // Build 2 = first cycle, build 3 = second cycle, etc. // After all cycles = KPI const showChart = build >= 1; const showBaseline = build >= 1; const visibleCycleCount = build >= 2 ? build - 1 : 0; const allRevealed = !v3Runs.isLoading && totalValidCycles > 0 && visibleCycleCount >= totalValidCycles; // Chart data: x-axis is step within cycle (1, 2, 3...), one column per cycle const maxSteps = Math.max(...validCycles.map(g => g.points.length), 0); const chartData = Array.from({ length: maxSteps }, (_, i) => { const row: Record = { step: i + 1 }; validCycles.forEach((g, c) => { row[`cycle_${c}`] = (c < visibleCycleCount && i < g.points.length) ? g.points[i].accuracy : null; }); return row; }); // Newest cycle is red, older ones fade to lighter grays const getCycleColor = (cycleIndex: number) => { if (cycleIndex === visibleCycleCount - 1) return MD.red; // newest = red // Older cycles: lighter grays the further back they are const stepsBack = visibleCycleCount - 1 - cycleIndex; const gray = Math.min(220, 160 + stepsBack * 15); return `rgb(${gray},${gray},${gray})`; }; return (

= 0)}>

V3: Self Learning

Fragment search · learning loops · trust scoring · quality gate

{v3Runs.isLoading ? (

) : ( `${v}%`} fontSize={11} /> `${v}%`} /> {showBaseline && ( )} {Array.from({ length: Math.min(visibleCycleCount, totalValidCycles) }).map((_, c) => ( ))} )}

{allRevealed && (

79.3%

V3 best (vs V2: 86.6%)

$10+

V3 cost/run (vs V2: $4-5)

)}

); } // ── Slide 9: V4 Dev Iteration ────────────────────────────────────── function SlideV4Iteration({ build }: { build: number }) { const v4DevRuns = useSQLQuery(` SELECT ROW_NUMBER() OVER (ORDER BY run_id) as run_num, ROUND(AVG(CASE WHEN (evaluation::JSON->>'is_correct')::BOOLEAN THEN 1 ELSE 0 END) * 100, 1) as accuracy, ROUND(SUM(estimated_cost_usd), 4) as total_cost FROM "agentic_sql_results_public"."main"."results" WHERE benchmark = 'dabstep' AND db_config = 'v4' AND model = 'google/gemini-3-flash-preview' AND run_id LIKE 'dabstep_train%' GROUP BY run_id HAVING COUNT(*) = 26 ORDER BY run_id `); const allPoints = useMemo(() => (Array.isArray(v4DevRuns.data) ? v4DevRuns.data : []).map(r => ({ run: N(r.run_num), accuracy: N(r.accuracy), cost: N(r.total_cost), })), [v4DevRuns.data]); const totalV4Cost = allPoints.reduce((sum, p) => sum + p.cost, 0); // Builds: 0=title, 1=point1, 2=point2, 3=point3, 4=point4 (labeled), 5=all remaining const visibleCount = build >= 5 ? allPoints.length : build >= 1 ? Math.min(build, 4) : 0; const visiblePoints = allPoints.slice(0, visibleCount); const showingAll = build >= 5; return (

= 0)}>

V4: Back to Basics

Human + Claude Code · analyze failures · fix · test · repeat

= 1} height={340}>

{v4DevRuns.isLoading ? (

) : ( `${v}%`} fontSize={11} /> `${v}%`} /> { const isLabeled = props.index === 3 && !showingAll; // 4th point const isLast = showingAll && props.index === visiblePoints.length - 1; return ( ); }} label={(props: any) => { const isLabeled = props.index === 3 && !showingAll; const isLast = showingAll && props.index === visiblePoints.length - 1; if (!isLabeled && !isLast) return null; return ( {`${props.value.toFixed(1)}%`} ); }} isAnimationActive={false} /> )}

{build >= 6 && (

= 6), marginTop: 12, display: "flex", gap: 48, justifyContent: "center", transition: "opacity 0.3s ease", opacity: build >= 7 ? 0.4 : 1 }}>

{allPoints.length}

iterations

${totalV4Cost.toFixed(2)}

total cost (26q each)

${(totalV4Cost / allPoints.length).toFixed(2)}

avg per iteration

)} {build >= 7 && (

= 7), marginTop: 16, display: "flex", justifyContent: "center" }}>

$5.06

full run (424 questions)

)}

); } // ── Slide 10: What V4 Did ────────────────────────────────────────── function SlideV4Explanation({ build }: { build: number }) { const fullComparison = useSQLQuery(` WITH best_runs AS ( SELECT db_config as tier, run_id, ROUND(AVG(CASE WHEN (evaluation::JSON->>'is_correct')::BOOLEAN THEN 1 ELSE 0 END) * 100, 1) as accuracy, ROUND(SUM(estimated_cost_usd), 2) as cost FROM "agentic_sql_results_public"."main"."results" WHERE benchmark = 'dabstep' AND model = 'google/gemini-3-flash-preview' AND run_id LIKE 'dabstep_test%' GROUP BY db_config, run_id HAVING COUNT(*) >= 352 ), ranked AS ( SELECT tier, accuracy, cost, ROW_NUMBER() OVER (PARTITION BY tier ORDER BY accuracy DESC) as rn FROM best_runs WHERE tier IN ('baseline', 'v1', 'v2', 'v3', 'v4') ) SELECT tier, CASE tier WHEN 'baseline' THEN 'Just Tables' WHEN 'v1' THEN 'Comments' WHEN 'v2' THEN 'Views' WHEN 'v3' THEN 'Self-learning' WHEN 'v4' THEN 'Prompt+Macros' END as label, CASE tier WHEN 'baseline' THEN 1 WHEN 'v1' THEN 2 WHEN 'v2' THEN 3 WHEN 'v3' THEN 4 WHEN 'v4' THEN 5 END as sort_order, accuracy, cost FROM ranked WHERE rn = 1 ORDER BY sort_order `); const bars = useMemo(() => (Array.isArray(fullComparison.data) ? fullComparison.data : []).map(r => ({ label: String(r.label), accuracy: N(r.accuracy), cost: N(r.cost), })), [fullComparison.data]); const barLabel = useMemo(() => ({ position: "top" as const, fontSize: 12, fill: MD.text, formatter: (v: number) => `${v.toFixed(1)}%`, }), []); return (

= 0)}>

What We Actually Did

No search. We removed ask_docs and search_fragments entirely.

We jammed everything into the prompt. Full manual + SQL patterns + term mappings.

We had Claude build macros. Gave them obvious names. Added to prompt.

= 1} height={80}>

{`SELECT * FROM merchant_fee_matches('Golfclub_Kansen')`}

Instead of a 30-line join. Named as the answer: cheapest_fraud_aci, merchants_affected_by_fee

= 2} height={224}>

{fullComparison.isLoading ? (

) : ( `${v}%`} fontSize={11} /> `${v}%`} /> )}

= 3} height={28}>

Better data model, fewer retries, less money

); } // ── Slide 10: The Scoreboard ─────────────────────────────────────── function SlideScoreboard({ build }: { build: number }) { const leaderboard = [ { agent: "NVIDIA KGMON", org: "NVIDIA", model: "Claude Haiku 4.5", easy: 87.5, hard: 90.0 }, { agent: "DataPilot", org: "AntGroup", model: "Qwen3", easy: 86.1, hard: 87.6 }, { agent: "CambioML DS Agent", org: "CambioML", model: "GPT-5", easy: 94.4, hard: 57.7 }, { agent: "DS-STAR", org: "Google Cloud", model: "Gemini 2.5 Pro", easy: 87.5, hard: 45.2 }, ]; const ourRow = { agent: "V4+", org: "MotherDuck", model: "Gemini 3 Flash", easy: 93.1, hard: 93.2 }; const showOurs = build >= 1; const allRows = showOurs ? [ourRow, ...leaderboard] : leaderboard; const cellStyle: React.CSSProperties = { padding: "10px 16px", fontSize: 15, borderBottom: "1px solid #eee", textAlign: "left", }; const headerStyle: React.CSSProperties = { ...cellStyle, fontWeight: 700, color: MD.text, fontSize: 12, textTransform: "uppercase", letterSpacing: 1, borderBottom: "2px solid #231f20", }; return (

= 0)}>

The Scoreboard

DABstep leaderboard

{allRows.map((row) => { const isOurs = row.agent === "V4+"; return ( ); })}

Agent	Organization	Model	Easy	Hard
{row.agent}	{row.org}	{row.model}	{row.easy.toFixed(1)}%	{row.hard.toFixed(1)}%

= 2), marginTop: 48, textAlign: "center" }}>

This was a talk about SQL. And a little bit about AI.

); } // ── Slide 12: The Open Question ──────────────────────────────────── function SlideOpenQuestion({ build }: { build: number }) { return (

= 0)}>

Questions?

); }

This Is a Talk About SQL... and AI (Data Engineer Things remix)

Comments